/// <summary> /// Expect sas token has the write permission for the specified container. /// </summary> internal void ValidateContainerWriteableWithSasToken(CloudBlobContainer container, string sastoken) { Test.Info("Verify container write permission"); CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken); CloudBlobClient sasBlobClient = sasAccount.CreateCloudBlobClient(); CloudBlobContainer sasContainer = sasBlobClient.GetContainerReference(container.Name); string blobName = Utility.GenNameString("saspageblob"); CloudPageBlob pageblob = sasContainer.GetPageBlobReference(blobName); long blobSize = 1024 * 1024; pageblob.Create(blobSize); long buffSize = 1024 * 1024; byte[] buffer = new byte[buffSize]; random.NextBytes(buffer); MemoryStream ms = new MemoryStream(buffer); pageblob.UploadFromStream(ms); CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName); Test.Assert(retrievedBlob != null, "Page blob should exist on server"); TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type"); TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size"); }
/// <summary> /// Create a page blob with many small ranges (filling the page blob randomly by 3MB each time) /// </summary> /// <param name="containerName"></param> /// <param name="blobName"></param> /// <param name="blobSize"></param> public void CreatePageBlobWithManySmallRanges(string containerName, string blobName, int blobSize) { const int Min_PageRageSizeinByte = 1024 * 1024 * 3; CloudBlobContainer container = client.GetContainerReference(containerName); CloudPageBlob pageblob = container.GetPageBlobReference(blobName); pageblob.Create(blobSize); //write small page ranges Random rnd = new Random(); byte[] data = new byte[Min_PageRageSizeinByte]; rnd.NextBytes(data); int pageno = 0; for (long i = 0; i < blobSize - Min_PageRageSizeinByte; i = i + Min_PageRageSizeinByte) { if (Utility.GetRandomBool()) { pageblob.WritePages(new MemoryStream(data), i); } pageno++; } Test.Info("Page No: {0} totally", pageno); }
public static List <string> CreateBlobs(CloudBlobContainer container, int count, BlobType type) { string name; List <string> blobs = new List <string>(); for (int i = 0; i < count; i++) { switch (type) { case BlobType.BlockBlob: name = "bb" + Guid.NewGuid().ToString(); CloudBlockBlob blockBlob = container.GetBlockBlobReference(name); blockBlob.PutBlockList(new string[] { }); blobs.Add(name); break; case BlobType.PageBlob: name = "pb" + Guid.NewGuid().ToString(); CloudPageBlob pageBlob = container.GetPageBlobReference(name); pageBlob.Create(0); blobs.Add(name); break; } } return(blobs); }
/// <summary> /// Create a cloud page blob with the given name in the given container /// </summary> /// <param name="container">reference to the target cloud container</param> /// <param name="name">name of the intended page blob</param> /// <returns>reference to a created blob</returns> public static CloudPageBlob CreateCloudPageBlob(CloudBlobContainer container, string name) { CloudPageBlob blob = container.GetPageBlobReference(name); // TODO(Tianyu): Will there ever be a race on this? blob.Create(MAX_BLOB_SIZE); return(blob); }
public void Init() { if (!_blob.Exists()) { //var nextSize = NextSize(0); _blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*")); } _size = _blob.Properties.Length; _etag = _blob.Properties.ETag; }
public static void KillJob(string account, string key, string containerName, string jobDirectory) { CloudBlobContainer container = InitContainer(account, key, containerName, false); string blobName = jobDirectory.TrimEnd('/') + "/kill"; CloudPageBlob killBlob = container.GetPageBlobReference(blobName); if (!killBlob.Exists()) { killBlob.Create(512); } }
public void Upload(CloudStorageAccount account, string driveBlobUri) { FileInfo fileInfo = new FileInfo(this.fileName); Console.WriteLine("Uploading virtual hard disk (VHD) file: {0}", fileInfo.Name); Console.WriteLine(); CloudBlobClient blobClient = account.CreateCloudBlobClient(); blobClient.Timeout = new TimeSpan(0, 0, 180); // create page blob to provide backing storage for the drive - pad length to be multiple of 512 bytes CloudPageBlob driveBlob = blobClient.GetPageBlobReference(driveBlobUri); long paddedLength = (fileInfo.Length + PAGEBLOB_PAGE_SIZE - 1) & ~(PAGEBLOB_PAGE_SIZE - 1); Console.WriteLine("Creating Azure Drive"); Console.WriteLine(" URI: {0}", driveBlob.Uri.AbsoluteUri); Console.WriteLine(" Size: {0:n3} MB", paddedLength / (1024 * 1024)); Console.WriteLine(); // first ensure that container exists and then create the blob CloudBlobContainer container = blobClient.GetContainerReference(driveBlob.Container.Name); container.CreateIfNotExist(); driveBlob.Create(paddedLength); this.fileOffset = 0; this.blockLength = 0; Console.WriteLine("{0} - Uploading data...\r\n", DateTime.Now.ToLongTimeString()); using (stream = fileInfo.OpenRead()) { int blockNumber = 0; ProgressIndicator progress = new ProgressIndicator(); while (Read()) { Console.Write("\r Writing block {0,3} at offset: {1,16:n0}, size: {2,16:n1} KB", blockNumber, this.blockOffset, Convert.ToDecimal(this.blockLength) / 1024); progress.Enable(); driveBlob.WritePages(new MemoryStream(this.buffer, 0, this.blockLength, false), this.blockOffset); Console.WriteLine("\r *"); progress.Disable(); blockNumber++; } Console.WriteLine(); Console.WriteLine("{0} - Upload complete.", DateTime.Now.ToLongTimeString()); } }
/// <summary> /// create a new page blob with random properties and metadata /// </summary> /// <param name="container">CloudBlobContainer object</param> /// <param name="blobName">blob name</param> /// <returns>ICloudBlob object</returns> public ICloudBlob CreatePageBlob(CloudBlobContainer container, string blobName) { CloudPageBlob pageBlob = container.GetPageBlobReference(blobName); int size = random.Next(1, 10) * PageBlobUnitSize; pageBlob.Create(size); byte[] buffer = new byte[size]; string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer)); pageBlob.Properties.ContentMD5 = md5sum; GenerateBlobPropertiesAndMetaData(pageBlob); Test.Info(string.Format("create page blob '{0}' in container '{1}'", blobName, container.Name)); return(pageBlob); }
public long GetOrInitPosition() { // blob.Exists actually fetches metadata if (!_blob.Exists()) { _blob.Metadata[CloudSetup.CheckpointMetadataName] = "0"; _blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*")); _etag = _blob.Properties.ETag; return(0); } var position = _blob.Metadata[CloudSetup.CheckpointMetadataName]; _etag = _blob.Properties.ETag; var result = long.Parse(position); Ensure.ZeroOrGreater("position", result); return(result); }
public PageBlobAppendStream(CloudPageBlob blob) { _blob = blob; if (blob == null) { throw new ArgumentNullException("blob"); } _reader = new BinaryReader(_blob.OpenRead()); if (!blob.Exists()) { _blob.Create(0); _lastPageIndex = -1; return; } _blobLength = _blob.Properties.Length; _lastPageIndex = (_blobLength / PageSize) - 1; }
// Put (create or update) a page blob. // Return true on success, false if unable to create, throw exception on error. public bool PutBlob(string containerName, string blobName, int pageBlobSize) { try { CloudBlobContainer container = BlobClient.GetContainerReference(containerName); CloudPageBlob blob = container.GetPageBlobReference(blobName); blob.Create(pageBlobSize); return(true); } catch (StorageClientException ex) { if ((int)ex.StatusCode == 404) { return(false); } throw; } }
/// <summary> /// create a new page blob with random properties and metadata /// </summary> /// <param name="container">CloudBlobContainer object</param> /// <param name="blobName">blob name</param> /// <returns>CloudBlob object</returns> public async Task <CloudBlob> CreatePageBlobAsync(CloudBlobContainer container, string blobName, CancellationToken cancellationToken, bool createBigBlob = false) { CloudPageBlob pageBlob = container.GetPageBlobReference(blobName); int size = (createBigBlob ? random.Next(102400, 204800) : random.Next(1, 10)) * PageBlobUnitSize; pageBlob.Create(size); byte[] buffer = new byte[size]; // fill in random data random.NextBytes(buffer); using (MemoryStream ms = new MemoryStream(buffer)) { await pageBlob.UploadFromStreamAsync(ms, cancellationToken); } string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer)); pageBlob.Properties.ContentMD5 = md5sum; GenerateBlobPropertiesAndMetaData(pageBlob); Test.Info(string.Format("create page blob '{0}' in container '{1}', md5 = {2}", blobName, container.Name, md5sum)); return(pageBlob); }
/// <summary> /// Expect sas token has the Create permission for the specified container. /// </summary> internal void ValidateContainerCreateableWithSasToken(CloudBlobContainer container, string sastoken) { Test.Info("Verify container Create permission"); CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken); CloudBlobClient sasBlobClient = sasAccount.CreateCloudBlobClient(); CloudBlobContainer sasContainer = sasBlobClient.GetContainerReference(container.Name); if (!container.Exists()) { sasContainer.Create(); Test.Assert(sasContainer.Exists(), "The container should exist after Creating with sas token"); } string blobName = Utility.GenNameString("saspageblob"); CloudPageBlob pageblob = sasContainer.GetPageBlobReference(blobName); long blobSize = 1024 * 1024; pageblob.Create(blobSize); CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName); Test.Assert(retrievedBlob != null, "Page blob should exist on server"); TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type"); TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size"); }
public static void UploadVHDToCloud(string fileName, long fileSize, CloudBlobClient blobStorage, string containerName, string blobName) { CloudBlobContainer container = blobStorage.GetContainerReference(containerName); CloudPageBlob pageBlob = container.GetPageBlobReference(blobName); long blobSize = RoundUpToPageBlobSize(fileSize); pageBlob.Create(blobSize); FileStream stream = new FileStream(blobName, FileMode.Open, FileAccess.Read); BinaryReader reader = new BinaryReader(stream); long totalUploaded = 0; long vhdOffset = 0; int offsetToTransfer = -1; //xw32: Based on the code below, this var indicates the starting index of a list of consecutive ranges (4Mb each) that are not all zeros. while (vhdOffset < fileSize) { byte[] range = reader.ReadBytes(FourMegabytesAsBytes); int offsetInRange = 0; // Make sure end is page size aligned if ((range.Length % PageBlobPageSize) > 0) { int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize)); Array.Resize(ref range, range.Length + grow); } // Upload groups of contiguous non-zero page blob pages. while (offsetInRange <= range.Length) { if ((offsetInRange == range.Length) || IsAllZero(range, offsetInRange, PageBlobPageSize)) { if (offsetToTransfer != -1) { // Transfer up to this point int sizeToTransfer = offsetInRange - offsetToTransfer; MemoryStream memoryStream = new MemoryStream(range, offsetToTransfer, sizeToTransfer, false, false); pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer); Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset) + " + " + PrintSize(sizeToTransfer)); totalUploaded += sizeToTransfer; offsetToTransfer = -1; } } else { if (offsetToTransfer == -1) { offsetToTransfer = offsetInRange; } } offsetInRange += PageBlobPageSize; } vhdOffset += range.Length; } Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize)); }
public static AzureEventStoreChunk CreateNewForWriting(CloudPageBlob blob) { blob.Create(ChunkSize); return(new AzureEventStoreChunk(blob, 0, ChunkSize)); }
private static void UploadVHDToCloud(Config config) { StorageCredentialsAccountAndKey creds = new StorageCredentialsAccountAndKey(config.Account, config.Key); CloudBlobClient blobStorage = new CloudBlobClient(config.AccountUrl, creds); CloudBlobContainer container = blobStorage.GetContainerReference(config.Container); container.CreateIfNotExist(); CloudPageBlob pageBlob = container.GetPageBlobReference(config.Blob); Console.WriteLine("Vhd size: " + Megabytes(config.Vhd.Length)); long blobSize = RoundUpToPageBlobSize(config.Vhd.Length); pageBlob.Create(blobSize); FileStream stream = new FileStream(config.Vhd.FullName, FileMode.Open, FileAccess.Read); BinaryReader reader = new BinaryReader(stream); long totalUploaded = 0; long vhdOffset = 0; int offsetToTransfer = -1; while (vhdOffset < config.Vhd.Length) { byte[] range = reader.ReadBytes(FourMegabytesAsBytes); int offsetInRange = 0; // Make sure end is page size aligned if ((range.Length % PageBlobPageSize) > 0) { int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize)); Array.Resize(ref range, range.Length + grow); } // Upload groups of contiguous non-zero page blob pages. while (offsetInRange <= range.Length) { if ((offsetInRange == range.Length) || IsAllZero(range, offsetInRange, PageBlobPageSize)) { if (offsetToTransfer != -1) { // Transfer up to this point int sizeToTransfer = offsetInRange - offsetToTransfer; MemoryStream memoryStream = new MemoryStream(range, offsetToTransfer, sizeToTransfer, false, false); pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer); Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset) + " + " + PrintSize(sizeToTransfer)); totalUploaded += sizeToTransfer; offsetToTransfer = -1; } } else { if (offsetToTransfer == -1) { offsetToTransfer = offsetInRange; } } offsetInRange += PageBlobPageSize; } vhdOffset += range.Length; } Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize)); }
private static async Task PageBlobProcessAsync() { CloudStorageAccount storageAccount = null; CloudBlobContainer cloudBlobContainer = null; // Retrieve the connection string for use with the application. The storage connection string is stored // in an environment variable on the machine running the application called storageconnectionstring. // If the environment variable is created after the application is launched in a console or with Visual // Studio, the shell needs to be closed and reloaded to take the environment variable into account. string storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"]; //Environment.GetEnvironmentVariable("StorageConnectionString"); //storageAccount = CloudStorageAccount.DevelopmentStorageAccount; // Check whether the connection string can be parsed. if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount)) { try { // Create the CloudBlobClient that represents the Blob storage endpoint for the storage account. CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); //127.0.0.1:10000/devstoreaccount1/testcont // Create a container called 'quickstartblobs' and append a GUID value to it to make the name unique. cloudBlobContainer = cloudBlobClient.GetContainerReference("testcontpageblob");// "quickstartblobs" + Guid.NewGuid().ToString()); await cloudBlobContainer.CreateIfNotExistsAsync(); Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name); Console.WriteLine(); // Set the permissions so the blobs are public. BlobContainerPermissions permissions = new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }; await cloudBlobContainer.SetPermissionsAsync(permissions); byte[] data = new byte[512]; Random rnd = new Random(); rnd.NextBytes(data); // Get a reference to the blob address, then upload the file to the blob. // Use the value of localFileName for the blob name. CloudPageBlob cloudPageBlob = cloudBlobContainer.GetPageBlobReference("SamplePage"); cloudPageBlob.Create(512); cloudPageBlob.WritePages(new MemoryStream(data), 0); // List the blobs in the container. Console.WriteLine("Listing blobs in container."); BlobContinuationToken blobContinuationToken = null; do { var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken); // Get the value of the continuation token returned by the listing call. blobContinuationToken = results.ContinuationToken; foreach (IListBlobItem item in results.Results) { Console.WriteLine(item.Uri); } } while (blobContinuationToken != null); // Loop while the continuation token is not null. Console.WriteLine("Read the PageRanges"); cloudPageBlob.FetchAttributes(); Console.WriteLine("Blob length = {0}", cloudPageBlob.Properties.Length); IEnumerable <PageRange> ranges = cloudPageBlob.GetPageRanges(); Console.Write("{0}:<", "Writing Data From PageBlob"); foreach (PageRange range in ranges) { Console.Write("[{0}-{1}]", range.StartOffset, range.EndOffset); } Console.WriteLine(">"); Console.WriteLine("Delete the PageBlob"); await cloudPageBlob.DeleteAsync(); } catch (StorageException ex) { Console.WriteLine("Error returned from the service: {0}", ex.Message); } finally { // Console.WriteLine("Press any key to delete the sample files and example container."); Console.ReadLine(); // Clean up resources. This includes the container and the two temp files. //Console.WriteLine("Deleting the container and any blobs it contains"); if (cloudBlobContainer != null) { // await cloudBlobContainer.DeleteIfExistsAsync(); } } } else { Console.WriteLine( "A connection string has not been defined in the system environment variables. " + "Add a environment variable named 'storageconnectionstring' with your storage " + "connection string as a value."); } }
public static AzureBlobDataSet CreateSetWithSmallData(string uri, SerializableDataSetSchema schema, IDictionary <string, Array> dataToPut) { SerializableDataSetSchema info = schema; List <SerializableDimension> dimensions = schema.Dimensions.ToList(); List <SerializableVariableSchema> varsUnsorted = schema.Variables.ToList(); List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(varsUnsorted.Count); //vars for which data is provided should go first int varsWithDataCount = 0; foreach (var v in varsUnsorted) { if (dataToPut.ContainsKey(v.Name)) { vars.Add(v); ++varsWithDataCount; } } foreach (var v in varsUnsorted) { if (!dataToPut.ContainsKey(v.Name)) { vars.Add(v); } } Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: dimension added {0}[{1}]", i.Name, i.Length)); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); int schemeSize; using (MemoryStream bufferStream = new MemoryStream()) { using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name)); estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[j], vars[i].Name)); rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name)); estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512); } } blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); //blob.WritePages(sizeStream, 0); //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512; byte[] scheme = new byte[sizeAligned]; sizeStream.Seek(0, SeekOrigin.Begin); sizeStream.Read(scheme, 0, 512); memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 512, schemeSize); bufferStream.Write(scheme, 0, sizeAligned); //for (int i = 0; i < sizeAligned; i += maxBlobChunk) // blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i); } } for (int i = 0; i < varsWithDataCount; ++i) { if (vars[i].Dimensions.Length == 1) { int len = dimLengthDictionary[vars[i].Dimensions[0]]; var data = dataToPut[vars[i].Name]; if (vars[i].Type == typeof(DateTime)) { var temp = new Int64[data.Length]; for (int j = 0; j < temp.Length; ++j) { temp[j] = ((DateTime)data.GetValue(j)).Ticks; } data = temp; } int bufferSize = 512 * ((len * vars[i].ValueSize + 511) / 512); byte[] buffer = new byte[bufferSize]; Buffer.BlockCopy(data, 0, buffer, 0, len * vars[i].ValueSize); bufferStream.Write(buffer, 0, bufferSize); //for (int j = 0; j < bufferSize; j += maxBlobChunk) // blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j); } else { int outerDimLen = dimLengthDictionary[vars[i].Dimensions[0]]; int rowLen = vars[i].ValueSize; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowLen *= dimLengthDictionary[vars[i].Dimensions[j]]; } int rowLenUnaligned = rowLen; rowLen = 512 * ((rowLen + 511) / 512); byte[] buffer = new byte[rowLen]; Array data = dataToPut[vars[i].Name]; if (vars[i].Type == typeof(DateTime)) { int[] shapeTemp = new int[data.Rank]; for (int k = 0; k < shapeTemp.Length; ++k) { shapeTemp[k] = data.GetUpperBound(k) + 1; } Array temp = Array.CreateInstance(typeof(Int64), shapeTemp); int[] resPos = new int[shapeTemp.Length]; for (int k = 0; k < resPos.Length; ++k) { resPos[k] = 0; } do { temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos); }while (Move(resPos, shapeTemp)); data = temp; } for (int j = 0; j < outerDimLen; ++j) { Buffer.BlockCopy(data, j * rowLenUnaligned, buffer, 0, rowLenUnaligned); bufferStream.Write(buffer, 0, rowLen); } } } int bufferStreamSize = (int)bufferStream.Length; int bufferStreamSizeAligned = ((bufferStreamSize + 511) / 512) * 512; byte[] bufferAligned = new byte[bufferStreamSizeAligned + 512]; bufferStream.Seek(0, SeekOrigin.Begin); bufferStream.Read(bufferAligned, 0, bufferStreamSize); for (int i = 0; i < bufferStreamSizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(bufferAligned, i, Math.Min(maxBlobChunk, bufferStreamSizeAligned - i)), i); } } return(new AzureBlobDataSet(uri, schemeSize, info)); }
public static AzureBlobDataSet CreateEmptySet(string uri, SerializableDataSetSchema schema) { SerializableDataSetSchema info = schema; List <SerializableDimension> dimensions = schema.Dimensions.ToList(); List <SerializableVariableSchema> vars = schema.Variables.ToList(); Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); int schemeSize; using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512); } } blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); //blob.WritePages(sizeStream, 0); //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512; byte[] scheme = new byte[sizeAligned]; sizeStream.Seek(0, SeekOrigin.Begin); sizeStream.Read(scheme, 0, 512); memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 512, schemeSize); for (int i = 0; i < sizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i); } } } return(new AzureBlobDataSet(uri, schemeSize, info)); }
public static AzureBlobDataSet ArrangeData(string uri, DataSet source, SerializableVariableSchema[] emptyVariables) { List <SerializableDimension> dimensions = new List <SerializableDimension>(); foreach (var i in source.Dimensions) { dimensions.Add(new SerializableDimension(i.Name, i.Length)); } List <SerializableVariableSchema> oldVars = source.Variables.Select <Variable, SerializableVariableSchema>(x => x.GetSchema().AsSerializable()).ToList(); List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(oldVars); vars.AddRange(emptyVariables); SerializableDataSetSchema info = new SerializableDataSetSchema(dimensions.ToArray(), vars.ToArray(), source.Metadata.AsDictionary()); Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Container }); CloudPageBlob blob; int schemeSize; using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } estimatedBlobSize += dimLengthDictionary[vars[i].Dimensions[0]] * ((rowSize * vars[i].ValueSize + 511) / 512) * 512; } } blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); // CRITICAL: some may interfere between calls blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); blob.WritePages(sizeStream, 0); } //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512; byte[] scheme = new byte[sizeAligned]; memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 0, schemeSize); for (int i = 0; i < sizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), 512 + i); } } //populating blob with values from source for (int i = 0; i < oldVars.Count; ++i) { if (oldVars[i].Dimensions.Length == 1) { int len = dimLengthDictionary[oldVars[i].Dimensions[0]]; var data = source[oldVars[i].Name].GetData(); if (oldVars[i].Type == typeof(DateTime)) { var temp = new Int64[data.Length]; for (int j = 0; j < temp.Length; ++j) { temp[j] = ((DateTime)data.GetValue(j)).Ticks; } data = temp; } int bufferSize = 512 * ((len * oldVars[i].ValueSize + 511) / 512); byte[] buffer = new byte[bufferSize]; Buffer.BlockCopy(data, 0, buffer, 0, len * oldVars[i].ValueSize); for (int j = 0; j < bufferSize; j += maxBlobChunk) { blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j); } } else { int outerDimLen = dimLengthDictionary[oldVars[i].Dimensions[0]]; int rowLen = vars[i].ValueSize; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowLen *= dimLengthDictionary[vars[i].Dimensions[j]]; } int rowLenUnaligned = rowLen; rowLen = 512 * ((rowLen + 511) / 512); int[] origin = new int[oldVars[i].Dimensions.Length]; for (int j = 0; j < origin.Length; ++j) { origin[j] = 0; } int[] shape = new int[oldVars[i].Dimensions.Length]; shape[0] = 1; for (int j = 1; j < origin.Length; ++j) { shape[j] = dimLengthDictionary[oldVars[i].Dimensions[j]]; } byte[] buffer = new byte[rowLen]; for (int j = 0; j < outerDimLen; ++j) { origin[0] = j; Array data = source[oldVars[i].Name].GetData(origin, shape); if (oldVars[i].Type == typeof(DateTime)) { int[] shapeTemp = new int[data.Rank]; for (int k = 0; k < shapeTemp.Length; ++k) { shapeTemp[k] = data.GetUpperBound(k) + 1; } Array temp = Array.CreateInstance(typeof(Int64), shapeTemp); int[] resPos = new int[shapeTemp.Length]; for (int k = 0; k < resPos.Length; ++k) { resPos[k] = 0; } do { temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos); }while (Move(resPos, shapeTemp)); data = temp; } Buffer.BlockCopy(data, 0, buffer, 0, rowLenUnaligned); for (int k = 0; k < rowLen; k += maxBlobChunk) { blob.WritePages(new MemoryStream(buffer, k, Math.Min(maxBlobChunk, rowLen - k)), varOffsets[i] + (long)rowLen * (long)j + (long)k); } } } } //blob is prepared: values are where they gotta be, trash is everwhere else! return(new AzureBlobDataSet(uri, schemeSize, info)); }