示例#1
0
        /// <summary>
        /// Expect sas token has the write permission for the specified container.
        /// </summary>
        internal void ValidateContainerWriteableWithSasToken(CloudBlobContainer container, string sastoken)
        {
            Test.Info("Verify container write permission");
            CloudStorageAccount sasAccount    = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken);
            CloudBlobClient     sasBlobClient = sasAccount.CreateCloudBlobClient();
            CloudBlobContainer  sasContainer  = sasBlobClient.GetContainerReference(container.Name);
            string        blobName            = Utility.GenNameString("saspageblob");
            CloudPageBlob pageblob            = sasContainer.GetPageBlobReference(blobName);
            long          blobSize            = 1024 * 1024;

            pageblob.Create(blobSize);

            long buffSize = 1024 * 1024;

            byte[] buffer = new byte[buffSize];
            random.NextBytes(buffer);
            MemoryStream ms = new MemoryStream(buffer);

            pageblob.UploadFromStream(ms);

            CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName);

            Test.Assert(retrievedBlob != null, "Page blob should exist on server");
            TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type");
            TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size");
        }
示例#2
0
        /// <summary>
        /// Create a page blob with many small ranges (filling the page blob randomly by 3MB each time)
        /// </summary>
        /// <param name="containerName"></param>
        /// <param name="blobName"></param>
        /// <param name="blobSize"></param>
        public void CreatePageBlobWithManySmallRanges(string containerName, string blobName, int blobSize)
        {
            const int Min_PageRageSizeinByte = 1024 * 1024 * 3;

            CloudBlobContainer container = client.GetContainerReference(containerName);
            CloudPageBlob      pageblob  = container.GetPageBlobReference(blobName);

            pageblob.Create(blobSize);

            //write small page ranges
            Random rnd = new Random();

            byte[] data = new byte[Min_PageRageSizeinByte];
            rnd.NextBytes(data);

            int pageno = 0;

            for (long i = 0; i < blobSize - Min_PageRageSizeinByte; i = i + Min_PageRageSizeinByte)
            {
                if (Utility.GetRandomBool())
                {
                    pageblob.WritePages(new MemoryStream(data), i);
                }
                pageno++;
            }

            Test.Info("Page No: {0} totally", pageno);
        }
示例#3
0
        public static List <string> CreateBlobs(CloudBlobContainer container, int count, BlobType type)
        {
            string        name;
            List <string> blobs = new List <string>();

            for (int i = 0; i < count; i++)
            {
                switch (type)
                {
                case BlobType.BlockBlob:
                    name = "bb" + Guid.NewGuid().ToString();
                    CloudBlockBlob blockBlob = container.GetBlockBlobReference(name);
                    blockBlob.PutBlockList(new string[] { });
                    blobs.Add(name);
                    break;

                case BlobType.PageBlob:
                    name = "pb" + Guid.NewGuid().ToString();
                    CloudPageBlob pageBlob = container.GetPageBlobReference(name);
                    pageBlob.Create(0);
                    blobs.Add(name);
                    break;
                }
            }
            return(blobs);
        }
        /// <summary>
        /// Create a cloud page blob with the given name in the given container
        /// </summary>
        /// <param name="container">reference to the target cloud container</param>
        /// <param name="name">name of the intended page blob</param>
        /// <returns>reference to a created blob</returns>
        public static CloudPageBlob CreateCloudPageBlob(CloudBlobContainer container, string name)
        {
            CloudPageBlob blob = container.GetPageBlobReference(name);

            // TODO(Tianyu): Will there ever be a race on this?
            blob.Create(MAX_BLOB_SIZE);
            return(blob);
        }
示例#5
0
        public void Init()
        {
            if (!_blob.Exists())
            {
                //var nextSize = NextSize(0);
                _blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*"));
            }

            _size = _blob.Properties.Length;
            _etag = _blob.Properties.ETag;
        }
        public static void KillJob(string account, string key, string containerName, string jobDirectory)
        {
            CloudBlobContainer container = InitContainer(account, key, containerName, false);
            string             blobName  = jobDirectory.TrimEnd('/') + "/kill";
            CloudPageBlob      killBlob  = container.GetPageBlobReference(blobName);

            if (!killBlob.Exists())
            {
                killBlob.Create(512);
            }
        }
        public void Upload(CloudStorageAccount account, string driveBlobUri)
        {
            FileInfo fileInfo = new FileInfo(this.fileName);

            Console.WriteLine("Uploading virtual hard disk (VHD) file: {0}", fileInfo.Name);
            Console.WriteLine();

            CloudBlobClient blobClient = account.CreateCloudBlobClient();

            blobClient.Timeout = new TimeSpan(0, 0, 180);

            // create page blob to provide backing storage for the drive - pad length to be multiple of 512 bytes
            CloudPageBlob driveBlob    = blobClient.GetPageBlobReference(driveBlobUri);
            long          paddedLength = (fileInfo.Length + PAGEBLOB_PAGE_SIZE - 1) & ~(PAGEBLOB_PAGE_SIZE - 1);

            Console.WriteLine("Creating Azure Drive");
            Console.WriteLine("  URI: {0}", driveBlob.Uri.AbsoluteUri);
            Console.WriteLine("  Size: {0:n3} MB", paddedLength / (1024 * 1024));
            Console.WriteLine();

            // first ensure that container exists and then create the blob
            CloudBlobContainer container = blobClient.GetContainerReference(driveBlob.Container.Name);

            container.CreateIfNotExist();
            driveBlob.Create(paddedLength);

            this.fileOffset  = 0;
            this.blockLength = 0;

            Console.WriteLine("{0} - Uploading data...\r\n", DateTime.Now.ToLongTimeString());
            using (stream = fileInfo.OpenRead())
            {
                int blockNumber            = 0;
                ProgressIndicator progress = new ProgressIndicator();

                while (Read())
                {
                    Console.Write("\r   Writing block {0,3} at offset: {1,16:n0}, size: {2,16:n1} KB", blockNumber, this.blockOffset, Convert.ToDecimal(this.blockLength) / 1024);
                    progress.Enable();

                    driveBlob.WritePages(new MemoryStream(this.buffer, 0, this.blockLength, false), this.blockOffset);
                    Console.WriteLine("\r *");
                    progress.Disable();
                    blockNumber++;
                }

                Console.WriteLine();
                Console.WriteLine("{0} - Upload complete.", DateTime.Now.ToLongTimeString());
            }
        }
示例#8
0
        /// <summary>
        /// create a new page blob with random properties and metadata
        /// </summary>
        /// <param name="container">CloudBlobContainer object</param>
        /// <param name="blobName">blob name</param>
        /// <returns>ICloudBlob object</returns>
        public ICloudBlob CreatePageBlob(CloudBlobContainer container, string blobName)
        {
            CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
            int           size     = random.Next(1, 10) * PageBlobUnitSize;

            pageBlob.Create(size);
            byte[] buffer = new byte[size];
            string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));

            pageBlob.Properties.ContentMD5 = md5sum;
            GenerateBlobPropertiesAndMetaData(pageBlob);
            Test.Info(string.Format("create page blob '{0}' in container '{1}'", blobName, container.Name));
            return(pageBlob);
        }
        public long GetOrInitPosition()
        {
            // blob.Exists actually fetches metadata
            if (!_blob.Exists())
            {
                _blob.Metadata[CloudSetup.CheckpointMetadataName] = "0";
                _blob.Create(0, AccessCondition.GenerateIfNoneMatchCondition("*"));
                _etag = _blob.Properties.ETag;
                return(0);
            }
            var position = _blob.Metadata[CloudSetup.CheckpointMetadataName];

            _etag = _blob.Properties.ETag;
            var result = long.Parse(position);

            Ensure.ZeroOrGreater("position", result);
            return(result);
        }
示例#10
0
        public PageBlobAppendStream(CloudPageBlob blob)
        {
            _blob = blob;
            if (blob == null)
            {
                throw new ArgumentNullException("blob");
            }

            _reader = new BinaryReader(_blob.OpenRead());
            if (!blob.Exists())
            {
                _blob.Create(0);
                _lastPageIndex = -1;
                return;
            }

            _blobLength    = _blob.Properties.Length;
            _lastPageIndex = (_blobLength / PageSize) - 1;
        }
示例#11
0
        // Put (create or update) a page blob.
        // Return true on success, false if unable to create, throw exception on error.

        public bool PutBlob(string containerName, string blobName, int pageBlobSize)
        {
            try
            {
                CloudBlobContainer container = BlobClient.GetContainerReference(containerName);
                CloudPageBlob      blob      = container.GetPageBlobReference(blobName);
                blob.Create(pageBlobSize);
                return(true);
            }
            catch (StorageClientException ex)
            {
                if ((int)ex.StatusCode == 404)
                {
                    return(false);
                }

                throw;
            }
        }
示例#12
0
        /// <summary>
        /// create a new page blob with random properties and metadata
        /// </summary>
        /// <param name="container">CloudBlobContainer object</param>
        /// <param name="blobName">blob name</param>
        /// <returns>CloudBlob object</returns>
        public async Task <CloudBlob> CreatePageBlobAsync(CloudBlobContainer container, string blobName, CancellationToken cancellationToken, bool createBigBlob = false)
        {
            CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);
            int           size     = (createBigBlob ? random.Next(102400, 204800) : random.Next(1, 10)) * PageBlobUnitSize;

            pageBlob.Create(size);

            byte[] buffer = new byte[size];
            // fill in random data
            random.NextBytes(buffer);
            using (MemoryStream ms = new MemoryStream(buffer))
            {
                await pageBlob.UploadFromStreamAsync(ms, cancellationToken);
            }

            string md5sum = Convert.ToBase64String(Helper.GetMD5(buffer));

            pageBlob.Properties.ContentMD5 = md5sum;
            GenerateBlobPropertiesAndMetaData(pageBlob);
            Test.Info(string.Format("create page blob '{0}' in container '{1}', md5 = {2}", blobName, container.Name, md5sum));
            return(pageBlob);
        }
示例#13
0
        /// <summary>
        /// Expect sas token has the Create permission for the specified container.
        /// </summary>
        internal void ValidateContainerCreateableWithSasToken(CloudBlobContainer container, string sastoken)
        {
            Test.Info("Verify container Create permission");
            CloudStorageAccount sasAccount    = TestBase.GetStorageAccountWithSasToken(container.ServiceClient.Credentials.AccountName, sastoken);
            CloudBlobClient     sasBlobClient = sasAccount.CreateCloudBlobClient();
            CloudBlobContainer  sasContainer  = sasBlobClient.GetContainerReference(container.Name);

            if (!container.Exists())
            {
                sasContainer.Create();
                Test.Assert(sasContainer.Exists(), "The container should  exist after Creating with sas token");
            }
            string        blobName = Utility.GenNameString("saspageblob");
            CloudPageBlob pageblob = sasContainer.GetPageBlobReference(blobName);
            long          blobSize = 1024 * 1024;

            pageblob.Create(blobSize);
            CloudBlob retrievedBlob = StorageExtensions.GetBlobReferenceFromServer(container, blobName);

            Test.Assert(retrievedBlob != null, "Page blob should exist on server");
            TestBase.ExpectEqual(StorageBlobType.PageBlob.ToString(), retrievedBlob.BlobType.ToString(), "blob type");
            TestBase.ExpectEqual(blobSize, retrievedBlob.Properties.Length, "blob size");
        }
        public static void UploadVHDToCloud(string fileName, long fileSize,
                                            CloudBlobClient blobStorage, string containerName, string blobName)
        {
            CloudBlobContainer container = blobStorage.GetContainerReference(containerName);

            CloudPageBlob pageBlob = container.GetPageBlobReference(blobName);

            long blobSize = RoundUpToPageBlobSize(fileSize);

            pageBlob.Create(blobSize);

            FileStream   stream = new FileStream(blobName, FileMode.Open, FileAccess.Read);
            BinaryReader reader = new BinaryReader(stream);

            long totalUploaded    = 0;
            long vhdOffset        = 0;
            int  offsetToTransfer = -1; //xw32: Based on the code below, this var indicates the starting index of a list of consecutive ranges (4Mb each) that are not all zeros.

            while (vhdOffset < fileSize)
            {
                byte[] range = reader.ReadBytes(FourMegabytesAsBytes);

                int offsetInRange = 0;

                // Make sure end is page size aligned
                if ((range.Length % PageBlobPageSize) > 0)
                {
                    int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize));
                    Array.Resize(ref range, range.Length + grow);
                }

                // Upload groups of contiguous non-zero page blob pages.
                while (offsetInRange <= range.Length)
                {
                    if ((offsetInRange == range.Length) ||
                        IsAllZero(range, offsetInRange, PageBlobPageSize))
                    {
                        if (offsetToTransfer != -1)
                        {
                            // Transfer up to this point
                            int          sizeToTransfer = offsetInRange - offsetToTransfer;
                            MemoryStream memoryStream   = new MemoryStream(range,
                                                                           offsetToTransfer, sizeToTransfer, false, false);
                            pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer);
                            Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset)
                                              + " + " + PrintSize(sizeToTransfer));
                            totalUploaded   += sizeToTransfer;
                            offsetToTransfer = -1;
                        }
                    }
                    else
                    {
                        if (offsetToTransfer == -1)
                        {
                            offsetToTransfer = offsetInRange;
                        }
                    }
                    offsetInRange += PageBlobPageSize;
                }
                vhdOffset += range.Length;
            }
            Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize));
        }
 public static AzureEventStoreChunk CreateNewForWriting(CloudPageBlob blob)
 {
     blob.Create(ChunkSize);
     return(new AzureEventStoreChunk(blob, 0, ChunkSize));
 }
        private static void UploadVHDToCloud(Config config)
        {
            StorageCredentialsAccountAndKey creds = new
                                                    StorageCredentialsAccountAndKey(config.Account, config.Key);

            CloudBlobClient    blobStorage = new CloudBlobClient(config.AccountUrl, creds);
            CloudBlobContainer container   = blobStorage.GetContainerReference(config.Container);

            container.CreateIfNotExist();

            CloudPageBlob pageBlob = container.GetPageBlobReference(config.Blob);

            Console.WriteLine("Vhd size:  " + Megabytes(config.Vhd.Length));

            long blobSize = RoundUpToPageBlobSize(config.Vhd.Length);

            pageBlob.Create(blobSize);

            FileStream   stream = new FileStream(config.Vhd.FullName, FileMode.Open, FileAccess.Read);
            BinaryReader reader = new BinaryReader(stream);

            long totalUploaded    = 0;
            long vhdOffset        = 0;
            int  offsetToTransfer = -1;

            while (vhdOffset < config.Vhd.Length)
            {
                byte[] range = reader.ReadBytes(FourMegabytesAsBytes);

                int offsetInRange = 0;

                // Make sure end is page size aligned
                if ((range.Length % PageBlobPageSize) > 0)
                {
                    int grow = (int)(PageBlobPageSize - (range.Length % PageBlobPageSize));
                    Array.Resize(ref range, range.Length + grow);
                }

                // Upload groups of contiguous non-zero page blob pages.
                while (offsetInRange <= range.Length)
                {
                    if ((offsetInRange == range.Length) ||
                        IsAllZero(range, offsetInRange, PageBlobPageSize))
                    {
                        if (offsetToTransfer != -1)
                        {
                            // Transfer up to this point
                            int          sizeToTransfer = offsetInRange - offsetToTransfer;
                            MemoryStream memoryStream   = new MemoryStream(range,
                                                                           offsetToTransfer, sizeToTransfer, false, false);
                            pageBlob.WritePages(memoryStream, vhdOffset + offsetToTransfer);
                            Console.WriteLine("Range ~" + Megabytes(offsetToTransfer + vhdOffset)
                                              + " + " + PrintSize(sizeToTransfer));
                            totalUploaded   += sizeToTransfer;
                            offsetToTransfer = -1;
                        }
                    }
                    else
                    {
                        if (offsetToTransfer == -1)
                        {
                            offsetToTransfer = offsetInRange;
                        }
                    }
                    offsetInRange += PageBlobPageSize;
                }
                vhdOffset += range.Length;
            }
            Console.WriteLine("Uploaded " + Megabytes(totalUploaded) + " of " + Megabytes(blobSize));
        }
示例#17
0
        private static async Task PageBlobProcessAsync()
        {
            CloudStorageAccount storageAccount = null;

            CloudBlobContainer cloudBlobContainer = null;



            // Retrieve the connection string for use with the application. The storage connection string is stored

            // in an environment variable on the machine running the application called storageconnectionstring.

            // If the environment variable is created after the application is launched in a console or with Visual

            // Studio, the shell needs to be closed and reloaded to take the environment variable into account.

            string storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"]; //Environment.GetEnvironmentVariable("StorageConnectionString");


            //storageAccount = CloudStorageAccount.DevelopmentStorageAccount;

            // Check whether the connection string can be parsed.

            if (CloudStorageAccount.TryParse(storageConnectionString, out storageAccount))
            {
                try

                {
                    // Create the CloudBlobClient that represents the Blob storage endpoint for the storage account.

                    CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();

                    //127.0.0.1:10000/devstoreaccount1/testcont

                    // Create a container called 'quickstartblobs' and append a GUID value to it to make the name unique.

                    cloudBlobContainer = cloudBlobClient.GetContainerReference("testcontpageblob");// "quickstartblobs" + Guid.NewGuid().ToString());

                    await cloudBlobContainer.CreateIfNotExistsAsync();

                    Console.WriteLine("Created container '{0}'", cloudBlobContainer.Name);

                    Console.WriteLine();



                    // Set the permissions so the blobs are public.

                    BlobContainerPermissions permissions = new BlobContainerPermissions

                    {
                        PublicAccess = BlobContainerPublicAccessType.Blob
                    };

                    await cloudBlobContainer.SetPermissionsAsync(permissions);



                    byte[] data = new byte[512];
                    Random rnd  = new Random();
                    rnd.NextBytes(data);


                    // Get a reference to the blob address, then upload the file to the blob.

                    // Use the value of localFileName for the blob name.

                    CloudPageBlob cloudPageBlob = cloudBlobContainer.GetPageBlobReference("SamplePage");
                    cloudPageBlob.Create(512);


                    cloudPageBlob.WritePages(new MemoryStream(data), 0);


                    // List the blobs in the container.

                    Console.WriteLine("Listing blobs in container.");

                    BlobContinuationToken blobContinuationToken = null;

                    do

                    {
                        var results = await cloudBlobContainer.ListBlobsSegmentedAsync(null, blobContinuationToken);

                        // Get the value of the continuation token returned by the listing call.

                        blobContinuationToken = results.ContinuationToken;

                        foreach (IListBlobItem item in results.Results)

                        {
                            Console.WriteLine(item.Uri);
                        }
                    } while (blobContinuationToken != null); // Loop while the continuation token is not null.

                    Console.WriteLine("Read the PageRanges");

                    cloudPageBlob.FetchAttributes();
                    Console.WriteLine("Blob length = {0}", cloudPageBlob.Properties.Length);

                    IEnumerable <PageRange> ranges = cloudPageBlob.GetPageRanges();
                    Console.Write("{0}:<", "Writing Data From PageBlob");
                    foreach (PageRange range in ranges)
                    {
                        Console.Write("[{0}-{1}]", range.StartOffset, range.EndOffset);
                    }
                    Console.WriteLine(">");


                    Console.WriteLine("Delete the PageBlob");
                    await cloudPageBlob.DeleteAsync();
                }

                catch (StorageException ex)

                {
                    Console.WriteLine("Error returned from the service: {0}", ex.Message);
                }

                finally

                {
                    // Console.WriteLine("Press any key to delete the sample files and example container.");

                    Console.ReadLine();

                    // Clean up resources. This includes the container and the two temp files.

                    //Console.WriteLine("Deleting the container and any blobs it contains");

                    if (cloudBlobContainer != null)
                    {
                        // await cloudBlobContainer.DeleteIfExistsAsync();
                    }
                }
            }

            else

            {
                Console.WriteLine(

                    "A connection string has not been defined in the system environment variables. " +

                    "Add a environment variable named 'storageconnectionstring' with your storage " +

                    "connection string as a value.");
            }
        }
示例#18
0
        public static AzureBlobDataSet CreateSetWithSmallData(string uri, SerializableDataSetSchema schema, IDictionary <string, Array> dataToPut)
        {
            SerializableDataSetSchema         info         = schema;
            List <SerializableDimension>      dimensions   = schema.Dimensions.ToList();
            List <SerializableVariableSchema> varsUnsorted = schema.Variables.ToList();
            List <SerializableVariableSchema> vars         = new List <SerializableVariableSchema>(varsUnsorted.Count);
            //vars for which data is provided should go first
            int varsWithDataCount = 0;

            foreach (var v in varsUnsorted)
            {
                if (dataToPut.ContainsKey(v.Name))
                {
                    vars.Add(v);
                    ++varsWithDataCount;
                }
            }
            foreach (var v in varsUnsorted)
            {
                if (!dataToPut.ContainsKey(v.Name))
                {
                    vars.Add(v);
                }
            }

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
                //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: dimension added {0}[{1}]", i.Name, i.Length));
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();

            CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);

            blob.DeleteIfExists();
            int schemeSize;

            using (MemoryStream bufferStream = new MemoryStream())
            {
                using (MemoryStream memStream = new MemoryStream())
                {
                    DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                    serializer.WriteObject(memStream, info);

                    schemeSize         = (int)memStream.Length;
                    estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                    for (int i = 0; i < vars.Count; ++i)
                    {
                        varOffsets[i] = estimatedBlobSize;
                        if (vars[i].Dimensions.Length == 1)
                        {
                            //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
                            estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                        }
                        else
                        {
                            int rowSize = 1;
                            for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                            {
                                //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[j], vars[i].Name));
                                rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                            }
                            //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
                            estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
                        }
                    }

                    blob.Create(estimatedBlobSize);

                    //writing scheme size into the 1st page
                    UTF8Encoding utf8 = new UTF8Encoding();
                    using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                    {
                        byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                        sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                        sizeStream.Seek(0, SeekOrigin.Begin);
                        //blob.WritePages(sizeStream, 0);

                        //writing scheme starting with 2nd page
                        int    sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
                        byte[] scheme      = new byte[sizeAligned];
                        sizeStream.Seek(0, SeekOrigin.Begin);
                        sizeStream.Read(scheme, 0, 512);
                        memStream.Seek(0, SeekOrigin.Begin);
                        memStream.Read(scheme, 512, schemeSize);
                        bufferStream.Write(scheme, 0, sizeAligned);
                        //for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                        //    blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
                    }
                }

                for (int i = 0; i < varsWithDataCount; ++i)
                {
                    if (vars[i].Dimensions.Length == 1)
                    {
                        int len  = dimLengthDictionary[vars[i].Dimensions[0]];
                        var data = dataToPut[vars[i].Name];
                        if (vars[i].Type == typeof(DateTime))
                        {
                            var temp = new Int64[data.Length];
                            for (int j = 0; j < temp.Length; ++j)
                            {
                                temp[j] = ((DateTime)data.GetValue(j)).Ticks;
                            }
                            data = temp;
                        }
                        int    bufferSize = 512 * ((len * vars[i].ValueSize + 511) / 512);
                        byte[] buffer     = new byte[bufferSize];
                        Buffer.BlockCopy(data, 0, buffer, 0, len * vars[i].ValueSize);
                        bufferStream.Write(buffer, 0, bufferSize);
                        //for (int j = 0; j < bufferSize; j += maxBlobChunk)
                        //    blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
                    }
                    else
                    {
                        int outerDimLen = dimLengthDictionary[vars[i].Dimensions[0]];
                        int rowLen      = vars[i].ValueSize;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        int rowLenUnaligned = rowLen;
                        rowLen = 512 * ((rowLen + 511) / 512);

                        byte[] buffer = new byte[rowLen];
                        Array  data   = dataToPut[vars[i].Name];
                        if (vars[i].Type == typeof(DateTime))
                        {
                            int[] shapeTemp = new int[data.Rank];
                            for (int k = 0; k < shapeTemp.Length; ++k)
                            {
                                shapeTemp[k] = data.GetUpperBound(k) + 1;
                            }
                            Array temp   = Array.CreateInstance(typeof(Int64), shapeTemp);
                            int[] resPos = new int[shapeTemp.Length];
                            for (int k = 0; k < resPos.Length; ++k)
                            {
                                resPos[k] = 0;
                            }
                            do
                            {
                                temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
                            }while (Move(resPos, shapeTemp));
                            data = temp;
                        }

                        for (int j = 0; j < outerDimLen; ++j)
                        {
                            Buffer.BlockCopy(data, j * rowLenUnaligned, buffer, 0, rowLenUnaligned);
                            bufferStream.Write(buffer, 0, rowLen);
                        }
                    }
                }
                int    bufferStreamSize        = (int)bufferStream.Length;
                int    bufferStreamSizeAligned = ((bufferStreamSize + 511) / 512) * 512;
                byte[] bufferAligned           = new byte[bufferStreamSizeAligned + 512];
                bufferStream.Seek(0, SeekOrigin.Begin);
                bufferStream.Read(bufferAligned, 0, bufferStreamSize);
                for (int i = 0; i < bufferStreamSizeAligned; i += maxBlobChunk)
                {
                    blob.WritePages(new MemoryStream(bufferAligned, i, Math.Min(maxBlobChunk, bufferStreamSizeAligned - i)), i);
                }
            }

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }
示例#19
0
        public static AzureBlobDataSet CreateEmptySet(string uri, SerializableDataSetSchema schema)
        {
            SerializableDataSetSchema         info       = schema;
            List <SerializableDimension>      dimensions = schema.Dimensions.ToList();
            List <SerializableVariableSchema> vars       = schema.Variables.ToList();

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();

            CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);

            blob.DeleteIfExists();

            int schemeSize;

            using (MemoryStream memStream = new MemoryStream())
            {
                DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                serializer.WriteObject(memStream, info);

                schemeSize         = (int)memStream.Length;
                estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                for (int i = 0; i < vars.Count; ++i)
                {
                    varOffsets[i] = estimatedBlobSize;
                    if (vars[i].Dimensions.Length == 1)
                    {
                        estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                    }
                    else
                    {
                        int rowSize = 1;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
                    }
                }

                blob.Create(estimatedBlobSize);

                //writing scheme size into the 1st page
                UTF8Encoding utf8 = new UTF8Encoding();
                using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                {
                    byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                    sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    //blob.WritePages(sizeStream, 0);

                    //writing scheme starting with 2nd page
                    int    sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
                    byte[] scheme      = new byte[sizeAligned];
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    sizeStream.Read(scheme, 0, 512);
                    memStream.Seek(0, SeekOrigin.Begin);
                    memStream.Read(scheme, 512, schemeSize);
                    for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                    {
                        blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
                    }
                }
            }

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }
示例#20
0
        public static AzureBlobDataSet ArrangeData(string uri, DataSet source, SerializableVariableSchema[] emptyVariables)
        {
            List <SerializableDimension> dimensions = new List <SerializableDimension>();

            foreach (var i in source.Dimensions)
            {
                dimensions.Add(new SerializableDimension(i.Name, i.Length));
            }

            List <SerializableVariableSchema> oldVars = source.Variables.Select <Variable, SerializableVariableSchema>(x => x.GetSchema().AsSerializable()).ToList();

            List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(oldVars);

            vars.AddRange(emptyVariables);

            SerializableDataSetSchema info = new SerializableDataSetSchema(dimensions.ToArray(), vars.ToArray(), source.Metadata.AsDictionary());

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();
            container.SetPermissions(new BlobContainerPermissions
            {
                PublicAccess = BlobContainerPublicAccessType.Container
            });

            CloudPageBlob blob;
            int           schemeSize;

            using (MemoryStream memStream = new MemoryStream())
            {
                DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                serializer.WriteObject(memStream, info);

                schemeSize         = (int)memStream.Length;
                estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                for (int i = 0; i < vars.Count; ++i)
                {
                    varOffsets[i] = estimatedBlobSize;
                    if (vars[i].Dimensions.Length == 1)
                    {
                        estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                    }
                    else
                    {
                        int rowSize = 1;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        estimatedBlobSize += dimLengthDictionary[vars[i].Dimensions[0]] * ((rowSize * vars[i].ValueSize + 511) / 512) * 512;
                    }
                }

                blob = container.GetPageBlobReference(azureUri.Blob);
                blob.DeleteIfExists(); // CRITICAL: some may interfere between calls
                blob.Create(estimatedBlobSize);

                //writing scheme size into the 1st page
                UTF8Encoding utf8 = new UTF8Encoding();
                using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                {
                    byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                    sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    blob.WritePages(sizeStream, 0);
                }

                //writing scheme starting with 2nd page
                int    sizeAligned = ((schemeSize + 511) / 512) * 512;
                byte[] scheme      = new byte[sizeAligned];
                memStream.Seek(0, SeekOrigin.Begin);
                memStream.Read(scheme, 0, schemeSize);
                for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                {
                    blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), 512 + i);
                }
            }

            //populating blob with values from source
            for (int i = 0; i < oldVars.Count; ++i)
            {
                if (oldVars[i].Dimensions.Length == 1)
                {
                    int len  = dimLengthDictionary[oldVars[i].Dimensions[0]];
                    var data = source[oldVars[i].Name].GetData();
                    if (oldVars[i].Type == typeof(DateTime))
                    {
                        var temp = new Int64[data.Length];
                        for (int j = 0; j < temp.Length; ++j)
                        {
                            temp[j] = ((DateTime)data.GetValue(j)).Ticks;
                        }
                        data = temp;
                    }
                    int    bufferSize = 512 * ((len * oldVars[i].ValueSize + 511) / 512);
                    byte[] buffer     = new byte[bufferSize];
                    Buffer.BlockCopy(data, 0, buffer, 0, len * oldVars[i].ValueSize);
                    for (int j = 0; j < bufferSize; j += maxBlobChunk)
                    {
                        blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
                    }
                }
                else
                {
                    int outerDimLen = dimLengthDictionary[oldVars[i].Dimensions[0]];
                    int rowLen      = vars[i].ValueSize;
                    for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                    {
                        rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
                    }
                    int rowLenUnaligned = rowLen;
                    rowLen = 512 * ((rowLen + 511) / 512);

                    int[] origin = new int[oldVars[i].Dimensions.Length];
                    for (int j = 0; j < origin.Length; ++j)
                    {
                        origin[j] = 0;
                    }

                    int[] shape = new int[oldVars[i].Dimensions.Length];
                    shape[0] = 1;
                    for (int j = 1; j < origin.Length; ++j)
                    {
                        shape[j] = dimLengthDictionary[oldVars[i].Dimensions[j]];
                    }

                    byte[] buffer = new byte[rowLen];

                    for (int j = 0; j < outerDimLen; ++j)
                    {
                        origin[0] = j;

                        Array data = source[oldVars[i].Name].GetData(origin, shape);

                        if (oldVars[i].Type == typeof(DateTime))
                        {
                            int[] shapeTemp = new int[data.Rank];
                            for (int k = 0; k < shapeTemp.Length; ++k)
                            {
                                shapeTemp[k] = data.GetUpperBound(k) + 1;
                            }
                            Array temp   = Array.CreateInstance(typeof(Int64), shapeTemp);
                            int[] resPos = new int[shapeTemp.Length];
                            for (int k = 0; k < resPos.Length; ++k)
                            {
                                resPos[k] = 0;
                            }
                            do
                            {
                                temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
                            }while (Move(resPos, shapeTemp));
                            data = temp;
                        }

                        Buffer.BlockCopy(data, 0, buffer, 0, rowLenUnaligned);
                        for (int k = 0; k < rowLen; k += maxBlobChunk)
                        {
                            blob.WritePages(new MemoryStream(buffer, k, Math.Min(maxBlobChunk, rowLen - k)), varOffsets[i] + (long)rowLen * (long)j + (long)k);
                        }
                    }
                }
            }
            //blob is prepared: values are where they gotta be, trash is everwhere else!

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }