/// <summary>
/// Expect sas token has the write permission for the specified container.
/// </summary>
internal void ValidateContainerWriteableWithSasToken(CloudBlobContainer container, string sastoken)
{
Test.Info("Verify container write permission");
CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken);
CloudBlobClient sasBlobClient = sasAccount.CreateCloudBlobClient();
CloudBlobContainer sasContainer = sasBlobClient.GetContainerReference(container.Name);
string blobName = Utility.GenNameString("saspageblob");
CloudPageBlob pageblob = sasContainer.GetPageBlobReference(blobName);
long blobSize = 1024 * 1024;
pageblob.Create(blobSize);
long buffSize = 1024 * 1024;
byte[] buffer = new byte[buffSize];
random.NextBytes(buffer);
MemoryStream ms = new MemoryStream(buffer);
pageblob.UploadFromStream(ms);
CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName);
Test.Assert(retrievedBlob != null, "Page blob should exist on server");
TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type");
TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size");
}
/// <summary>
/// Create a page blob with many small ranges (filling the page blob randomly by 3MB each time)
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobName"></param>
/// <param name="blobSize"></param>
public void CreatePageBlobWithManySmallRanges(string containerName, string blobName, int blobSize)
{
const int Min_PageRageSizeinByte = 1024 * 1024 * 3;
CloudBlobContainer container = client.GetContainerReference(containerName);
CloudPageBlob pageblob = container.GetPageBlobReference(blobName);
pageblob.Create(blobSize);
//write small page ranges
Random rnd = new Random();
byte[] data = new byte[Min_PageRageSizeinByte];
rnd.NextBytes(data);
int pageno = 0;
for (long i = 0; i < blobSize - Min_PageRageSizeinByte; i = i + Min_PageRageSizeinByte)
{
if (Utility.GetRandomBool())
{
pageblob.WritePages(new MemoryStream(data), i);
}
pageno++;
}
Test.Info("Page No: {0} totally", pageno);
}
public static List <string> CreateBlobs(CloudBlobContainer container, int count, BlobType type)
{
string name;
List <string> blobs = new List <string>();
for (int i = 0; i < count; i++)
{
switch (type)
{
case BlobType.BlockBlob:
name = "bb" + Guid.NewGuid().ToString();
CloudBlockBlob blockBlob = container.GetBlockBlobReference(name);
blockBlob.PutBlockList(new string[] { });
blobs.Add(name);
break;
case BlobType.PageBlob:
name = "pb" + Guid.NewGuid().ToString();
CloudPageBlob pageBlob = container.GetPageBlobReference(name);
pageBlob.Create(0);
blobs.Add(name);
break;
}
}
return(blobs);
}
/// <summary>
/// Create a cloud page blob with the given name in the given container
/// </summary>
/// <param name="container">reference to the target cloud container</param>
/// <param name="name">name of the intended page blob</param>
/// <returns>reference to a created blob</returns>
public static CloudPageBlob CreateCloudPageBlob(CloudBlobContainer container, string name)
{
CloudPageBlob blob = container.GetPageBlobReference(name);
// TODO(Tianyu): Will there ever be a race on this?
blob.Create(MAX_BLOB_SIZE);
return(blob);
}
public void Init()
{
if (!_blob.Exists())
{
//var nextSize = NextSize(0);
_blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*"));
}
_size = _blob.Properties.Length;
_etag = _blob.Properties.ETag;
}
public static void KillJob(string account, string key, string containerName, string jobDirectory)
{
CloudBlobContainer container = InitContainer(account, key, containerName, false);
string blobName = jobDirectory.TrimEnd('/') + "/kill";
CloudPageBlob killBlob = container.GetPageBlobReference(blobName);
if (!killBlob.Exists())
{
killBlob.Create(512);
}
}
public void Upload(CloudStorageAccount account, string driveBlobUri)
{
FileInfo fileInfo = new FileInfo(this.fileName);
Console.WriteLine("Uploading virtual hard disk (VHD) file: {0}", fileInfo.Name);
Console.WriteLine();
CloudBlobClient blobClient = account.CreateCloudBlobClient();
blobClient.Timeout = new TimeSpan(0, 0, 180);
// create page blob to provide backing storage for the drive - pad length to be multiple of 512 bytes
CloudPageBlob driveBlob = blobClient.GetPageBlobReference(driveBlobUri);
long paddedLength = (fileInfo.Length + PAGEBLOB_PAGE_SIZE - 1) & ~(PAGEBLOB_PAGE_SIZE - 1);
Console.WriteLine("Creating Azure Drive");
Console.WriteLine(" URI: {0}", driveBlob.Uri.AbsoluteUri);
Console.WriteLine(" Size: {0:n3} MB", paddedLength / (1024 * 1024));
Console.WriteLine();
// first ensure that container exists and then create the blob
CloudBlobContainer container = blobClient.GetContainerReference(driveBlob.Container.Name);
container.CreateIfNotExist();
driveBlob.Create(paddedLength);
this.fileOffset = 0;
this.blockLength = 0;
Console.WriteLine("{0} - Uploading data...\r\n", DateTime.Now.ToLongTimeString());
using (stream = fileInfo.OpenRead())
{
int blockNumber = 0;
ProgressIndicator progress = new ProgressIndicator();
while (Read())
{
Console.Write("\r Writing block {0,3} at offset: {1,16:n0}, size: {2,16:n1} KB", blockNumber, this.blockOffset, Convert.ToDecimal(this.blockLength) / 1024);
progress.Enable();
driveBlob.WritePages(new MemoryStream(this.buffer, 0, this.blockLength, false), this.blockOffset);
Console.WriteLine("\r *");
progress.Disable();
blockNumber++;
}
Console.WriteLine();
Console.WriteLine("{0} - Upload complete.", DateTime.Now.ToLongTimeString());
}
}
/// <summary>
/// create a new page blob with random properties and metadata
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">blob name</param>
/// <returns>ICloudBlob object</returns>
public ICloudBlob CreatePageBlob(CloudBlobContainer container, string blobName)
{
CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
int size = random.Next(1, 10) * PageBlobUnitSize;
pageBlob.Create(size);
byte[] buffer = new byte[size];
string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));
pageBlob.Properties.ContentMD5 = md5sum;
GenerateBlobPropertiesAndMetaData(pageBlob);
Test.Info(string.Format("create page blob '{0}' in container '{1}'", blobName, container.Name));
return(pageBlob);
}
public long GetOrInitPosition()
{
// blob.Exists actually fetches metadata
if (!_blob.Exists())
{
_blob.Metadata[CloudSetup.CheckpointMetadataName] = "0";
_blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*"));
_etag = _blob.Properties.ETag;
return(0);
}
var position = _blob.Metadata[CloudSetup.CheckpointMetadataName];
_etag = _blob.Properties.ETag;
var result = long.Parse(position);
Ensure.ZeroOrGreater("position", result);
return(result);
}
public PageBlobAppendStream(CloudPageBlob blob)
{
_blob = blob;
if (blob == null)
{
throw new ArgumentNullException("blob");
}
_reader = new BinaryReader(_blob.OpenRead());
if (!blob.Exists())
{
_blob.Create(0);
_lastPageIndex = -1;
return;
}
_blobLength = _blob.Properties.Length;
_lastPageIndex = (_blobLength / PageSize) - 1;
}
// Put (create or update) a page blob.
// Return true on success, false if unable to create, throw exception on error.
public bool PutBlob(string containerName, string blobName, int pageBlobSize)
{
try
{
CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
CloudPageBlob blob = container.GetPageBlobReference(blobName);
blob.Create(pageBlobSize);
return(true);
}
catch (StorageClientException ex)
{
if ((int)ex.StatusCode == 404)
{
return(false);
}
throw;
}
}
/// <summary>
/// create a new page blob with random properties and metadata
/// </summary>
/// <param name="container">CloudBlobContainer object</param>
/// <param name="blobName">blob name</param>
/// <returns>CloudBlob object</returns>
public async Task <CloudBlob> CreatePageBlobAsync(CloudBlobContainer container, string blobName, CancellationToken cancellationToken, bool createBigBlob = false)
{
CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
int size = (createBigBlob ? random.Next(102400, 204800) : random.Next(1, 10)) * PageBlobUnitSize;
pageBlob.Create(size);
byte[] buffer = new byte[size];
// fill in random data
random.NextBytes(buffer);
using (MemoryStream ms = new MemoryStream(buffer))
{
await pageBlob.UploadFromStreamAsync(ms, cancellationToken);
}
string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));
pageBlob.Properties.ContentMD5 = md5sum;
GenerateBlobPropertiesAndMetaData(pageBlob);
Test.Info(string.Format("create page blob '{0}' in container '{1}', md5 = {2}", blobName, container.Name, md5sum));
return(pageBlob);
}
/// <summary>
/// Expect sas token has the Create permission for the specified container.
/// </summary>
internal void ValidateContainerCreateableWithSasToken(CloudBlobContainer container, string sastoken)
{
Test.Info("Verify container Create permission");
CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken);
CloudBlobClient sasBlobClient = sasAccount.CreateCloudBlobClient();
CloudBlobContainer sasContainer = sasBlobClient.GetContainerReference(container.Name);
if (!container.Exists())
{
sasContainer.Create();
Test.Assert(sasContainer.Exists(), "The container should exist after Creating with sas token");
}
string blobName = Utility.GenNameString("saspageblob");
CloudPageBlob pageblob = sasContainer.GetPageBlobReference(blobName);
long blobSize = 1024 * 1024;
pageblob.Create(blobSize);
CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName);
Test.Assert(retrievedBlob != null, "Page blob should exist on server");
TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type");
TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size");
}
public static void UploadVHDToCloud(string fileName, long fileSize,
CloudBlobClient blobStorage, string containerName, string blobName)
{
CloudBlobContainer container = blobStorage.GetContainerReference(containerName);
CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
long blobSize = RoundUpToPageBlobSize(fileSize);
pageBlob.Create(blobSize);
FileStream stream = new FileStream(blobName, FileMode.Open, FileAccess.Read);
BinaryReader reader = new BinaryReader(stream);
long totalUploaded = 0;
long vhdOffset = 0;
int offsetToTransfer = -1; //xw32: Based on the code below, this var indicates the starting index of a list of consecutive ranges (4Mb each) that are not all zeros.
while (vhdOffset < fileSize)
{
byte[] range = reader.ReadBytes(FourMegabytesAsBytes);
int offsetInRange = 0;
// Make sure end is page size aligned
if ((range.Length % PageBlobPageSize) > 0)
{
int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize));
Array.Resize(ref range, range.Length + grow);
}
// Upload groups of contiguous non-zero page blob pages.
while (offsetInRange <= range.Length)
{
if ((offsetInRange == range.Length) ||
IsAllZero(range, offsetInRange, PageBlobPageSize))
{
if (offsetToTransfer != -1)
{
// Transfer up to this point
int sizeToTransfer = offsetInRange - offsetToTransfer;
MemoryStream memoryStream = new MemoryStream(range,
offsetToTransfer, sizeToTransfer, false, false);
pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer);
Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset)
+ " + " + PrintSize(sizeToTransfer));
totalUploaded += sizeToTransfer;
offsetToTransfer = -1;
}
}
else
{
if (offsetToTransfer == -1)
{
offsetToTransfer = offsetInRange;
}
}
offsetInRange += PageBlobPageSize;
}
vhdOffset += range.Length;
}
Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize));
}
public static AzureEventStoreChunk CreateNewForWriting(CloudPageBlob blob)
{
blob.Create(ChunkSize);
return(new AzureEventStoreChunk(blob, 0, ChunkSize));
}
private static void UploadVHDToCloud(Config config)
{
StorageCredentialsAccountAndKey creds = new
StorageCredentialsAccountAndKey(config.Account, config.Key);
CloudBlobClient blobStorage = new CloudBlobClient(config.AccountUrl, creds);
CloudBlobContainer container = blobStorage.GetContainerReference(config.Container);
container.CreateIfNotExist();
CloudPageBlob pageBlob = container.GetPageBlobReference(config.Blob);
Console.WriteLine("Vhd size: " + Megabytes(config.Vhd.Length));
long blobSize = RoundUpToPageBlobSize(config.Vhd.Length);
pageBlob.Create(blobSize);
FileStream stream = new FileStream(config.Vhd.FullName, FileMode.Open, FileAccess.Read);
BinaryReader reader = new BinaryReader(stream);
long totalUploaded = 0;
long vhdOffset = 0;
int offsetToTransfer = -1;
while (vhdOffset < config.Vhd.Length)
{
byte[] range = reader.ReadBytes(FourMegabytesAsBytes);
int offsetInRange = 0;
// Make sure end is page size aligned
if ((range.Length % PageBlobPageSize) > 0)
{
int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize));
Array.Resize(ref range, range.Length + grow);
}
// Upload groups of contiguous non-zero page blob pages.
while (offsetInRange <= range.Length)
{
if ((offsetInRange == range.Length) ||
IsAllZero(range, offsetInRange, PageBlobPageSize))
{
if (offsetToTransfer != -1)
{
// Transfer up to this point
int sizeToTransfer = offsetInRange - offsetToTransfer;
MemoryStream memoryStream = new MemoryStream(range,
offsetToTransfer, sizeToTransfer, false, false);
pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer);
Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset)
+ " + " + PrintSize(sizeToTransfer));
totalUploaded += sizeToTransfer;
offsetToTransfer = -1;
}
}
else
{
if (offsetToTransfer == -1)
{
offsetToTransfer = offsetInRange;
}
}
offsetInRange += PageBlobPageSize;
}
vhdOffset += range.Length;
}
Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize));
}
private static async Task PageBlobProcessAsync()
{
CloudStorageAccount storageAccount = null;
CloudBlobContainer cloudBlobContainer = null;
// Retrieve the connection string for use with the application. The storage connection string is stored
// in an environment variable on the machine running the application called storageconnectionstring.
// If the environment variable is created after the application is launched in a console or with Visual
// Studio, the shell needs to be closed and reloaded to take the environment variable into account.
string storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"]; //Environment.GetEnvironmentVariable("StorageConnectionString");
//storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
// Check whether the connection string can be parsed.
if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
{
try
{
// Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
//127.0.0.1:10000/devstoreaccount1/testcont
// Create a container called 'quickstartblobs' and append a GUID value to it to make the name unique.
cloudBlobContainer = cloudBlobClient.GetContainerReference("testcontpageblob");// "quickstartblobs" + Guid.NewGuid().ToString());
await cloudBlobContainer.CreateIfNotExistsAsync();
Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name);
Console.WriteLine();
// Set the permissions so the blobs are public.
BlobContainerPermissions permissions = new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Blob
};
await cloudBlobContainer.SetPermissionsAsync(permissions);
byte[] data = new byte[512];
Random rnd = new Random();
rnd.NextBytes(data);
// Get a reference to the blob address, then upload the file to the blob.
// Use the value of localFileName for the blob name.
CloudPageBlob cloudPageBlob = cloudBlobContainer.GetPageBlobReference("SamplePage");
cloudPageBlob.Create(512);
cloudPageBlob.WritePages(new MemoryStream(data), 0);
// List the blobs in the container.
Console.WriteLine("Listing blobs in container.");
BlobContinuationToken blobContinuationToken = null;
do
{
var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);
// Get the value of the continuation token returned by the listing call.
blobContinuationToken = results.ContinuationToken;
foreach (IListBlobItem item in results.Results)
{
Console.WriteLine(item.Uri);
}
} while (blobContinuationToken != null); // Loop while the continuation token is not null.
Console.WriteLine("Read the PageRanges");
cloudPageBlob.FetchAttributes();
Console.WriteLine("Blob length = {0}", cloudPageBlob.Properties.Length);
IEnumerable <PageRange> ranges = cloudPageBlob.GetPageRanges();
Console.Write("{0}:<", "Writing Data From PageBlob");
foreach (PageRange range in ranges)
{
Console.Write("[{0}-{1}]", range.StartOffset, range.EndOffset);
}
Console.WriteLine(">");
Console.WriteLine("Delete the PageBlob");
await cloudPageBlob.DeleteAsync();
}
catch (StorageException ex)
{
Console.WriteLine("Error returned from the service: {0}", ex.Message);
}
finally
{
// Console.WriteLine("Press any key to delete the sample files and example container.");
Console.ReadLine();
// Clean up resources. This includes the container and the two temp files.
//Console.WriteLine("Deleting the container and any blobs it contains");
if (cloudBlobContainer != null)
{
// await cloudBlobContainer.DeleteIfExistsAsync();
}
}
}
else
{
Console.WriteLine(
"A connection string has not been defined in the system environment variables. " +
"Add a environment variable named 'storageconnectionstring' with your storage " +
"connection string as a value.");
}
}
public static AzureBlobDataSet CreateSetWithSmallData(string uri, SerializableDataSetSchema schema, IDictionary <string, Array> dataToPut)
{
SerializableDataSetSchema info = schema;
List <SerializableDimension> dimensions = schema.Dimensions.ToList();
List <SerializableVariableSchema> varsUnsorted = schema.Variables.ToList();
List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(varsUnsorted.Count);
//vars for which data is provided should go first
int varsWithDataCount = 0;
foreach (var v in varsUnsorted)
{
if (dataToPut.ContainsKey(v.Name))
{
vars.Add(v);
++varsWithDataCount;
}
}
foreach (var v in varsUnsorted)
{
if (!dataToPut.ContainsKey(v.Name))
{
vars.Add(v);
}
}
Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);
foreach (var i in dimensions)
{
dimLengthDictionary.Add(i.Name, i.Length);
//System.Diagnostics.Trace.WriteLine(string.Format("ABDS: dimension added {0}[{1}]", i.Name, i.Length));
}
long estimatedBlobSize = 512;//only scheme size on 1st page
long[] varOffsets = new long[vars.Count];
AzureBlobDataSetUri azureUri = null;
if (DataSetUri.IsDataSetUri(uri))
{
azureUri = new AzureBlobDataSetUri(uri);
}
else
{
azureUri = AzureBlobDataSetUri.ToUri(uri);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container
CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);
container.CreateIfNotExist();
CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);
blob.DeleteIfExists();
int schemeSize;
using (MemoryStream bufferStream = new MemoryStream())
{
using (MemoryStream memStream = new MemoryStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
serializer.WriteObject(memStream, info);
schemeSize = (int)memStream.Length;
estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data
for (int i = 0; i < vars.Count; ++i)
{
varOffsets[i] = estimatedBlobSize;
if (vars[i].Dimensions.Length == 1)
{
//System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
}
else
{
int rowSize = 1;
for (int j = 1; j < vars[i].Dimensions.Length; ++j)
{
//System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[j], vars[i].Name));
rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
}
//System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
}
}
blob.Create(estimatedBlobSize);
//writing scheme size into the 1st page
UTF8Encoding utf8 = new UTF8Encoding();
using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
{
byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
sizeStream.Seek(0, SeekOrigin.Begin);
//blob.WritePages(sizeStream, 0);
//writing scheme starting with 2nd page
int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
byte[] scheme = new byte[sizeAligned];
sizeStream.Seek(0, SeekOrigin.Begin);
sizeStream.Read(scheme, 0, 512);
memStream.Seek(0, SeekOrigin.Begin);
memStream.Read(scheme, 512, schemeSize);
bufferStream.Write(scheme, 0, sizeAligned);
//for (int i = 0; i < sizeAligned; i += maxBlobChunk)
// blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
}
}
for (int i = 0; i < varsWithDataCount; ++i)
{
if (vars[i].Dimensions.Length == 1)
{
int len = dimLengthDictionary[vars[i].Dimensions[0]];
var data = dataToPut[vars[i].Name];
if (vars[i].Type == typeof(DateTime))
{
var temp = new Int64[data.Length];
for (int j = 0; j < temp.Length; ++j)
{
temp[j] = ((DateTime)data.GetValue(j)).Ticks;
}
data = temp;
}
int bufferSize = 512 * ((len * vars[i].ValueSize + 511) / 512);
byte[] buffer = new byte[bufferSize];
Buffer.BlockCopy(data, 0, buffer, 0, len * vars[i].ValueSize);
bufferStream.Write(buffer, 0, bufferSize);
//for (int j = 0; j < bufferSize; j += maxBlobChunk)
// blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
}
else
{
int outerDimLen = dimLengthDictionary[vars[i].Dimensions[0]];
int rowLen = vars[i].ValueSize;
for (int j = 1; j < vars[i].Dimensions.Length; ++j)
{
rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
}
int rowLenUnaligned = rowLen;
rowLen = 512 * ((rowLen + 511) / 512);
byte[] buffer = new byte[rowLen];
Array data = dataToPut[vars[i].Name];
if (vars[i].Type == typeof(DateTime))
{
int[] shapeTemp = new int[data.Rank];
for (int k = 0; k < shapeTemp.Length; ++k)
{
shapeTemp[k] = data.GetUpperBound(k) + 1;
}
Array temp = Array.CreateInstance(typeof(Int64), shapeTemp);
int[] resPos = new int[shapeTemp.Length];
for (int k = 0; k < resPos.Length; ++k)
{
resPos[k] = 0;
}
do
{
temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
}while (Move(resPos, shapeTemp));
data = temp;
}
for (int j = 0; j < outerDimLen; ++j)
{
Buffer.BlockCopy(data, j * rowLenUnaligned, buffer, 0, rowLenUnaligned);
bufferStream.Write(buffer, 0, rowLen);
}
}
}
int bufferStreamSize = (int)bufferStream.Length;
int bufferStreamSizeAligned = ((bufferStreamSize + 511) / 512) * 512;
byte[] bufferAligned = new byte[bufferStreamSizeAligned + 512];
bufferStream.Seek(0, SeekOrigin.Begin);
bufferStream.Read(bufferAligned, 0, bufferStreamSize);
for (int i = 0; i < bufferStreamSizeAligned; i += maxBlobChunk)
{
blob.WritePages(new MemoryStream(bufferAligned, i, Math.Min(maxBlobChunk, bufferStreamSizeAligned - i)), i);
}
}
return(new AzureBlobDataSet(uri, schemeSize, info));
}
public static AzureBlobDataSet CreateEmptySet(string uri, SerializableDataSetSchema schema)
{
SerializableDataSetSchema info = schema;
List <SerializableDimension> dimensions = schema.Dimensions.ToList();
List <SerializableVariableSchema> vars = schema.Variables.ToList();
Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);
foreach (var i in dimensions)
{
dimLengthDictionary.Add(i.Name, i.Length);
}
long estimatedBlobSize = 512;//only scheme size on 1st page
long[] varOffsets = new long[vars.Count];
AzureBlobDataSetUri azureUri = null;
if (DataSetUri.IsDataSetUri(uri))
{
azureUri = new AzureBlobDataSetUri(uri);
}
else
{
azureUri = AzureBlobDataSetUri.ToUri(uri);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container
CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);
container.CreateIfNotExist();
CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);
blob.DeleteIfExists();
int schemeSize;
using (MemoryStream memStream = new MemoryStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
serializer.WriteObject(memStream, info);
schemeSize = (int)memStream.Length;
estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data
for (int i = 0; i < vars.Count; ++i)
{
varOffsets[i] = estimatedBlobSize;
if (vars[i].Dimensions.Length == 1)
{
estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
}
else
{
int rowSize = 1;
for (int j = 1; j < vars[i].Dimensions.Length; ++j)
{
rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
}
estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
}
}
blob.Create(estimatedBlobSize);
//writing scheme size into the 1st page
UTF8Encoding utf8 = new UTF8Encoding();
using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
{
byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
sizeStream.Seek(0, SeekOrigin.Begin);
//blob.WritePages(sizeStream, 0);
//writing scheme starting with 2nd page
int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
byte[] scheme = new byte[sizeAligned];
sizeStream.Seek(0, SeekOrigin.Begin);
sizeStream.Read(scheme, 0, 512);
memStream.Seek(0, SeekOrigin.Begin);
memStream.Read(scheme, 512, schemeSize);
for (int i = 0; i < sizeAligned; i += maxBlobChunk)
{
blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
}
}
}
return(new AzureBlobDataSet(uri, schemeSize, info));
}
public static AzureBlobDataSet ArrangeData(string uri, DataSet source, SerializableVariableSchema[] emptyVariables)
{
List <SerializableDimension> dimensions = new List <SerializableDimension>();
foreach (var i in source.Dimensions)
{
dimensions.Add(new SerializableDimension(i.Name, i.Length));
}
List <SerializableVariableSchema> oldVars = source.Variables.Select <Variable, SerializableVariableSchema>(x => x.GetSchema().AsSerializable()).ToList();
List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(oldVars);
vars.AddRange(emptyVariables);
SerializableDataSetSchema info = new SerializableDataSetSchema(dimensions.ToArray(), vars.ToArray(), source.Metadata.AsDictionary());
Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);
foreach (var i in dimensions)
{
dimLengthDictionary.Add(i.Name, i.Length);
}
long estimatedBlobSize = 512;//only scheme size on 1st page
long[] varOffsets = new long[vars.Count];
AzureBlobDataSetUri azureUri = null;
if (DataSetUri.IsDataSetUri(uri))
{
azureUri = new AzureBlobDataSetUri(uri);
}
else
{
azureUri = AzureBlobDataSetUri.ToUri(uri);
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Retrieve a reference to a container
CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);
container.CreateIfNotExist();
container.SetPermissions(new BlobContainerPermissions
{
PublicAccess = BlobContainerPublicAccessType.Container
});
CloudPageBlob blob;
int schemeSize;
using (MemoryStream memStream = new MemoryStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
serializer.WriteObject(memStream, info);
schemeSize = (int)memStream.Length;
estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data
for (int i = 0; i < vars.Count; ++i)
{
varOffsets[i] = estimatedBlobSize;
if (vars[i].Dimensions.Length == 1)
{
estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
}
else
{
int rowSize = 1;
for (int j = 1; j < vars[i].Dimensions.Length; ++j)
{
rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
}
estimatedBlobSize += dimLengthDictionary[vars[i].Dimensions[0]] * ((rowSize * vars[i].ValueSize + 511) / 512) * 512;
}
}
blob = container.GetPageBlobReference(azureUri.Blob);
blob.DeleteIfExists(); // CRITICAL: some may interfere between calls
blob.Create(estimatedBlobSize);
//writing scheme size into the 1st page
UTF8Encoding utf8 = new UTF8Encoding();
using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
{
byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
sizeStream.Seek(0, SeekOrigin.Begin);
blob.WritePages(sizeStream, 0);
}
//writing scheme starting with 2nd page
int sizeAligned = ((schemeSize + 511) / 512) * 512;
byte[] scheme = new byte[sizeAligned];
memStream.Seek(0, SeekOrigin.Begin);
memStream.Read(scheme, 0, schemeSize);
for (int i = 0; i < sizeAligned; i += maxBlobChunk)
{
blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), 512 + i);
}
}
//populating blob with values from source
for (int i = 0; i < oldVars.Count; ++i)
{
if (oldVars[i].Dimensions.Length == 1)
{
int len = dimLengthDictionary[oldVars[i].Dimensions[0]];
var data = source[oldVars[i].Name].GetData();
if (oldVars[i].Type == typeof(DateTime))
{
var temp = new Int64[data.Length];
for (int j = 0; j < temp.Length; ++j)
{
temp[j] = ((DateTime)data.GetValue(j)).Ticks;
}
data = temp;
}
int bufferSize = 512 * ((len * oldVars[i].ValueSize + 511) / 512);
byte[] buffer = new byte[bufferSize];
Buffer.BlockCopy(data, 0, buffer, 0, len * oldVars[i].ValueSize);
for (int j = 0; j < bufferSize; j += maxBlobChunk)
{
blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
}
}
else
{
int outerDimLen = dimLengthDictionary[oldVars[i].Dimensions[0]];
int rowLen = vars[i].ValueSize;
for (int j = 1; j < vars[i].Dimensions.Length; ++j)
{
rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
}
int rowLenUnaligned = rowLen;
rowLen = 512 * ((rowLen + 511) / 512);
int[] origin = new int[oldVars[i].Dimensions.Length];
for (int j = 0; j < origin.Length; ++j)
{
origin[j] = 0;
}
int[] shape = new int[oldVars[i].Dimensions.Length];
shape[0] = 1;
for (int j = 1; j < origin.Length; ++j)
{
shape[j] = dimLengthDictionary[oldVars[i].Dimensions[j]];
}
byte[] buffer = new byte[rowLen];
for (int j = 0; j < outerDimLen; ++j)
{
origin[0] = j;
Array data = source[oldVars[i].Name].GetData(origin, shape);
if (oldVars[i].Type == typeof(DateTime))
{
int[] shapeTemp = new int[data.Rank];
for (int k = 0; k < shapeTemp.Length; ++k)
{
shapeTemp[k] = data.GetUpperBound(k) + 1;
}
Array temp = Array.CreateInstance(typeof(Int64), shapeTemp);
int[] resPos = new int[shapeTemp.Length];
for (int k = 0; k < resPos.Length; ++k)
{
resPos[k] = 0;
}
do
{
temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
}while (Move(resPos, shapeTemp));
data = temp;
}
Buffer.BlockCopy(data, 0, buffer, 0, rowLenUnaligned);
for (int k = 0; k < rowLen; k += maxBlobChunk)
{
blob.WritePages(new MemoryStream(buffer, k, Math.Min(maxBlobChunk, rowLen - k)), varOffsets[i] + (long)rowLen * (long)j + (long)k);
}
}
}
}
//blob is prepared: values are where they gotta be, trash is everwhere else!
return(new AzureBlobDataSet(uri, schemeSize, info));
}
