Esempio n. 1
0
        protected AzureBlobDataSet(string uri, int schemeSize, SerializableDataSetSchema info)
        {
            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            this.uri = azureUri;

            CloudStorageAccount storageAccount;// = CloudStorageAccount.Parse(azureUri.ConnectionString);

            if (CloudStorageAccount.TryParse(azureUri.ConnectionString, out storageAccount))
            {
                CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

                // Retrieve a reference to a container
                CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

                blob = container.GetPageBlobReference(azureUri.Blob);
            }
            else
            {
                blob       = new CloudPageBlob(@"http://" + azureUri.AccountName + @".blob.core.windows.net/" + azureUri.Container + @"/" + azureUri.Blob);
                _anonymous = true;
            }

            bool savedAutoCommitState = IsAutocommitEnabled;

            IsAutocommitEnabled = false;

            Initialize(schemeSize, info);

            Commit();

            if (_anonymous)
            {
                this.SetCompleteReadOnly();
            }

            IsAutocommitEnabled = savedAutoCommitState;

            _IsInitialized = true;
        }
Esempio n. 2
0
        public static AzureBlobDataSet CreateSetWithSmallData(string uri, SerializableDataSetSchema schema, IDictionary <string, Array> dataToPut)
        {
            SerializableDataSetSchema         info         = schema;
            List <SerializableDimension>      dimensions   = schema.Dimensions.ToList();
            List <SerializableVariableSchema> varsUnsorted = schema.Variables.ToList();
            List <SerializableVariableSchema> vars         = new List <SerializableVariableSchema>(varsUnsorted.Count);
            //vars for which data is provided should go first
            int varsWithDataCount = 0;

            foreach (var v in varsUnsorted)
            {
                if (dataToPut.ContainsKey(v.Name))
                {
                    vars.Add(v);
                    ++varsWithDataCount;
                }
            }
            foreach (var v in varsUnsorted)
            {
                if (!dataToPut.ContainsKey(v.Name))
                {
                    vars.Add(v);
                }
            }

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
                //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: dimension added {0}[{1}]", i.Name, i.Length));
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();

            CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);

            blob.DeleteIfExists();
            int schemeSize;

            using (MemoryStream bufferStream = new MemoryStream())
            {
                using (MemoryStream memStream = new MemoryStream())
                {
                    DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                    serializer.WriteObject(memStream, info);

                    schemeSize         = (int)memStream.Length;
                    estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                    for (int i = 0; i < vars.Count; ++i)
                    {
                        varOffsets[i] = estimatedBlobSize;
                        if (vars[i].Dimensions.Length == 1)
                        {
                            //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
                            estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                        }
                        else
                        {
                            int rowSize = 1;
                            for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                            {
                                //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[j], vars[i].Name));
                                rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                            }
                            //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name));
                            estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
                        }
                    }

                    blob.Create(estimatedBlobSize);

                    //writing scheme size into the 1st page
                    UTF8Encoding utf8 = new UTF8Encoding();
                    using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                    {
                        byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                        sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                        sizeStream.Seek(0, SeekOrigin.Begin);
                        //blob.WritePages(sizeStream, 0);

                        //writing scheme starting with 2nd page
                        int    sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
                        byte[] scheme      = new byte[sizeAligned];
                        sizeStream.Seek(0, SeekOrigin.Begin);
                        sizeStream.Read(scheme, 0, 512);
                        memStream.Seek(0, SeekOrigin.Begin);
                        memStream.Read(scheme, 512, schemeSize);
                        bufferStream.Write(scheme, 0, sizeAligned);
                        //for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                        //    blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
                    }
                }

                for (int i = 0; i < varsWithDataCount; ++i)
                {
                    if (vars[i].Dimensions.Length == 1)
                    {
                        int len  = dimLengthDictionary[vars[i].Dimensions[0]];
                        var data = dataToPut[vars[i].Name];
                        if (vars[i].Type == typeof(DateTime))
                        {
                            var temp = new Int64[data.Length];
                            for (int j = 0; j < temp.Length; ++j)
                            {
                                temp[j] = ((DateTime)data.GetValue(j)).Ticks;
                            }
                            data = temp;
                        }
                        int    bufferSize = 512 * ((len * vars[i].ValueSize + 511) / 512);
                        byte[] buffer     = new byte[bufferSize];
                        Buffer.BlockCopy(data, 0, buffer, 0, len * vars[i].ValueSize);
                        bufferStream.Write(buffer, 0, bufferSize);
                        //for (int j = 0; j < bufferSize; j += maxBlobChunk)
                        //    blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
                    }
                    else
                    {
                        int outerDimLen = dimLengthDictionary[vars[i].Dimensions[0]];
                        int rowLen      = vars[i].ValueSize;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        int rowLenUnaligned = rowLen;
                        rowLen = 512 * ((rowLen + 511) / 512);

                        byte[] buffer = new byte[rowLen];
                        Array  data   = dataToPut[vars[i].Name];
                        if (vars[i].Type == typeof(DateTime))
                        {
                            int[] shapeTemp = new int[data.Rank];
                            for (int k = 0; k < shapeTemp.Length; ++k)
                            {
                                shapeTemp[k] = data.GetUpperBound(k) + 1;
                            }
                            Array temp   = Array.CreateInstance(typeof(Int64), shapeTemp);
                            int[] resPos = new int[shapeTemp.Length];
                            for (int k = 0; k < resPos.Length; ++k)
                            {
                                resPos[k] = 0;
                            }
                            do
                            {
                                temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
                            }while (Move(resPos, shapeTemp));
                            data = temp;
                        }

                        for (int j = 0; j < outerDimLen; ++j)
                        {
                            Buffer.BlockCopy(data, j * rowLenUnaligned, buffer, 0, rowLenUnaligned);
                            bufferStream.Write(buffer, 0, rowLen);
                        }
                    }
                }
                int    bufferStreamSize        = (int)bufferStream.Length;
                int    bufferStreamSizeAligned = ((bufferStreamSize + 511) / 512) * 512;
                byte[] bufferAligned           = new byte[bufferStreamSizeAligned + 512];
                bufferStream.Seek(0, SeekOrigin.Begin);
                bufferStream.Read(bufferAligned, 0, bufferStreamSize);
                for (int i = 0; i < bufferStreamSizeAligned; i += maxBlobChunk)
                {
                    blob.WritePages(new MemoryStream(bufferAligned, i, Math.Min(maxBlobChunk, bufferStreamSizeAligned - i)), i);
                }
            }

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }
Esempio n. 3
0
        public static AzureBlobDataSet CreateEmptySet(string uri, SerializableDataSetSchema schema)
        {
            SerializableDataSetSchema         info       = schema;
            List <SerializableDimension>      dimensions = schema.Dimensions.ToList();
            List <SerializableVariableSchema> vars       = schema.Variables.ToList();

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();

            CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob);

            blob.DeleteIfExists();

            int schemeSize;

            using (MemoryStream memStream = new MemoryStream())
            {
                DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                serializer.WriteObject(memStream, info);

                schemeSize         = (int)memStream.Length;
                estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                for (int i = 0; i < vars.Count; ++i)
                {
                    varOffsets[i] = estimatedBlobSize;
                    if (vars[i].Dimensions.Length == 1)
                    {
                        estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                    }
                    else
                    {
                        int rowSize = 1;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512);
                    }
                }

                blob.Create(estimatedBlobSize);

                //writing scheme size into the 1st page
                UTF8Encoding utf8 = new UTF8Encoding();
                using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                {
                    byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                    sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    //blob.WritePages(sizeStream, 0);

                    //writing scheme starting with 2nd page
                    int    sizeAligned = ((schemeSize + 511) / 512) * 512 + 512;
                    byte[] scheme      = new byte[sizeAligned];
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    sizeStream.Read(scheme, 0, 512);
                    memStream.Seek(0, SeekOrigin.Begin);
                    memStream.Read(scheme, 512, schemeSize);
                    for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                    {
                        blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i);
                    }
                }
            }

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }
Esempio n. 4
0
        public static AzureBlobDataSet ArrangeData(string uri, DataSet source, SerializableVariableSchema[] emptyVariables)
        {
            List <SerializableDimension> dimensions = new List <SerializableDimension>();

            foreach (var i in source.Dimensions)
            {
                dimensions.Add(new SerializableDimension(i.Name, i.Length));
            }

            List <SerializableVariableSchema> oldVars = source.Variables.Select <Variable, SerializableVariableSchema>(x => x.GetSchema().AsSerializable()).ToList();

            List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(oldVars);

            vars.AddRange(emptyVariables);

            SerializableDataSetSchema info = new SerializableDataSetSchema(dimensions.ToArray(), vars.ToArray(), source.Metadata.AsDictionary());

            Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count);

            foreach (var i in dimensions)
            {
                dimLengthDictionary.Add(i.Name, i.Length);
            }

            long estimatedBlobSize = 512;//only scheme size on 1st page

            long[] varOffsets = new long[vars.Count];

            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString);

            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Retrieve a reference to a container
            CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

            container.CreateIfNotExist();
            container.SetPermissions(new BlobContainerPermissions
            {
                PublicAccess = BlobContainerPublicAccessType.Container
            });

            CloudPageBlob blob;
            int           schemeSize;

            using (MemoryStream memStream = new MemoryStream())
            {
                DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                serializer.WriteObject(memStream, info);

                schemeSize         = (int)memStream.Length;
                estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data

                for (int i = 0; i < vars.Count; ++i)
                {
                    varOffsets[i] = estimatedBlobSize;
                    if (vars[i].Dimensions.Length == 1)
                    {
                        estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512;
                    }
                    else
                    {
                        int rowSize = 1;
                        for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                        {
                            rowSize *= dimLengthDictionary[vars[i].Dimensions[j]];
                        }
                        estimatedBlobSize += dimLengthDictionary[vars[i].Dimensions[0]] * ((rowSize * vars[i].ValueSize + 511) / 512) * 512;
                    }
                }

                blob = container.GetPageBlobReference(azureUri.Blob);
                blob.DeleteIfExists(); // CRITICAL: some may interfere between calls
                blob.Create(estimatedBlobSize);

                //writing scheme size into the 1st page
                UTF8Encoding utf8 = new UTF8Encoding();
                using (MemoryStream sizeStream = new MemoryStream(new byte[512], true))
                {
                    byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString());
                    sizeStream.Write(sizeBuf, 0, sizeBuf.Length);
                    sizeStream.Seek(0, SeekOrigin.Begin);
                    blob.WritePages(sizeStream, 0);
                }

                //writing scheme starting with 2nd page
                int    sizeAligned = ((schemeSize + 511) / 512) * 512;
                byte[] scheme      = new byte[sizeAligned];
                memStream.Seek(0, SeekOrigin.Begin);
                memStream.Read(scheme, 0, schemeSize);
                for (int i = 0; i < sizeAligned; i += maxBlobChunk)
                {
                    blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), 512 + i);
                }
            }

            //populating blob with values from source
            for (int i = 0; i < oldVars.Count; ++i)
            {
                if (oldVars[i].Dimensions.Length == 1)
                {
                    int len  = dimLengthDictionary[oldVars[i].Dimensions[0]];
                    var data = source[oldVars[i].Name].GetData();
                    if (oldVars[i].Type == typeof(DateTime))
                    {
                        var temp = new Int64[data.Length];
                        for (int j = 0; j < temp.Length; ++j)
                        {
                            temp[j] = ((DateTime)data.GetValue(j)).Ticks;
                        }
                        data = temp;
                    }
                    int    bufferSize = 512 * ((len * oldVars[i].ValueSize + 511) / 512);
                    byte[] buffer     = new byte[bufferSize];
                    Buffer.BlockCopy(data, 0, buffer, 0, len * oldVars[i].ValueSize);
                    for (int j = 0; j < bufferSize; j += maxBlobChunk)
                    {
                        blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j);
                    }
                }
                else
                {
                    int outerDimLen = dimLengthDictionary[oldVars[i].Dimensions[0]];
                    int rowLen      = vars[i].ValueSize;
                    for (int j = 1; j < vars[i].Dimensions.Length; ++j)
                    {
                        rowLen *= dimLengthDictionary[vars[i].Dimensions[j]];
                    }
                    int rowLenUnaligned = rowLen;
                    rowLen = 512 * ((rowLen + 511) / 512);

                    int[] origin = new int[oldVars[i].Dimensions.Length];
                    for (int j = 0; j < origin.Length; ++j)
                    {
                        origin[j] = 0;
                    }

                    int[] shape = new int[oldVars[i].Dimensions.Length];
                    shape[0] = 1;
                    for (int j = 1; j < origin.Length; ++j)
                    {
                        shape[j] = dimLengthDictionary[oldVars[i].Dimensions[j]];
                    }

                    byte[] buffer = new byte[rowLen];

                    for (int j = 0; j < outerDimLen; ++j)
                    {
                        origin[0] = j;

                        Array data = source[oldVars[i].Name].GetData(origin, shape);

                        if (oldVars[i].Type == typeof(DateTime))
                        {
                            int[] shapeTemp = new int[data.Rank];
                            for (int k = 0; k < shapeTemp.Length; ++k)
                            {
                                shapeTemp[k] = data.GetUpperBound(k) + 1;
                            }
                            Array temp   = Array.CreateInstance(typeof(Int64), shapeTemp);
                            int[] resPos = new int[shapeTemp.Length];
                            for (int k = 0; k < resPos.Length; ++k)
                            {
                                resPos[k] = 0;
                            }
                            do
                            {
                                temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos);
                            }while (Move(resPos, shapeTemp));
                            data = temp;
                        }

                        Buffer.BlockCopy(data, 0, buffer, 0, rowLenUnaligned);
                        for (int k = 0; k < rowLen; k += maxBlobChunk)
                        {
                            blob.WritePages(new MemoryStream(buffer, k, Math.Min(maxBlobChunk, rowLen - k)), varOffsets[i] + (long)rowLen * (long)j + (long)k);
                        }
                    }
                }
            }
            //blob is prepared: values are where they gotta be, trash is everwhere else!

            return(new AzureBlobDataSet(uri, schemeSize, info));
        }
Esempio n. 5
0
        public AzureBlobDataSet(string uri)
        {
            AzureBlobDataSetUri azureUri = null;

            if (DataSetUri.IsDataSetUri(uri))
            {
                azureUri = new AzureBlobDataSetUri(uri);
            }
            else
            {
                azureUri = AzureBlobDataSetUri.ToUri(uri);
            }

            this.uri = azureUri;

            CloudStorageAccount storageAccount;// = CloudStorageAccount.Parse(azureUri.ConnectionString);

            if (CloudStorageAccount.TryParse(azureUri.ConnectionString, out storageAccount))
            {
                CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

                // Retrieve a reference to a container
                CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container);

                blob = container.GetPageBlobReference(azureUri.Blob);
            }
            else
            {
                blob       = new CloudPageBlob(@"http://" + azureUri.AccountName + @".blob.core.windows.net/" + azureUri.Container + @"/" + azureUri.Blob);
                _anonymous = true;
            }
            SerializableDataSetSchema info;
            Int32 schemeSize;

            using (BinaryReader br = new BinaryReader(blob.OpenRead()))
            {
                int curOffset = 0;
                int curCount  = 512;
                int temp      = 0;

                UTF8Encoding utf8   = new UTF8Encoding();
                byte[]       buffer = new byte[512];
                do
                {
                    temp       = br.BaseStream.Read(buffer, curOffset, curCount);
                    curOffset += temp;
                    curCount  -= temp;
                } while (curOffset < 512);
                string sizeStr = utf8.GetString(buffer);
                schemeSize = Int32.Parse(sizeStr);
                br.BaseStream.Seek(512, SeekOrigin.Begin);
                DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema));
                byte[] scheme = new byte[schemeSize];
                curOffset = 0;
                curCount  = schemeSize;
                do
                {
                    temp       = br.BaseStream.Read(scheme, curOffset, curCount);
                    curOffset += temp;
                    curCount  -= temp;
                } while (curOffset < schemeSize);
                info = (SerializableDataSetSchema)serializer.ReadObject(new MemoryStream(scheme));
            }

            bool savedAutoCommitState = IsAutocommitEnabled;

            IsAutocommitEnabled = false;

            Initialize(schemeSize, info);

            Commit();

            if (_anonymous)
            {
                this.SetCompleteReadOnly();
            }

            IsAutocommitEnabled = savedAutoCommitState;

            _IsInitialized = true;
        }