protected AzureBlobDataSet(string uri, int schemeSize, SerializableDataSetSchema info) { AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } this.uri = azureUri; CloudStorageAccount storageAccount;// = CloudStorageAccount.Parse(azureUri.ConnectionString); if (CloudStorageAccount.TryParse(azureUri.ConnectionString, out storageAccount)) { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); blob = container.GetPageBlobReference(azureUri.Blob); } else { blob = new CloudPageBlob(@"http://" + azureUri.AccountName + @".blob.core.windows.net/" + azureUri.Container + @"/" + azureUri.Blob); _anonymous = true; } bool savedAutoCommitState = IsAutocommitEnabled; IsAutocommitEnabled = false; Initialize(schemeSize, info); Commit(); if (_anonymous) { this.SetCompleteReadOnly(); } IsAutocommitEnabled = savedAutoCommitState; _IsInitialized = true; }
public static AzureBlobDataSet CreateSetWithSmallData(string uri, SerializableDataSetSchema schema, IDictionary <string, Array> dataToPut) { SerializableDataSetSchema info = schema; List <SerializableDimension> dimensions = schema.Dimensions.ToList(); List <SerializableVariableSchema> varsUnsorted = schema.Variables.ToList(); List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(varsUnsorted.Count); //vars for which data is provided should go first int varsWithDataCount = 0; foreach (var v in varsUnsorted) { if (dataToPut.ContainsKey(v.Name)) { vars.Add(v); ++varsWithDataCount; } } foreach (var v in varsUnsorted) { if (!dataToPut.ContainsKey(v.Name)) { vars.Add(v); } } Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: dimension added {0}[{1}]", i.Name, i.Length)); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); int schemeSize; using (MemoryStream bufferStream = new MemoryStream()) { using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name)); estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[j], vars[i].Name)); rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } //System.Diagnostics.Trace.WriteLine(string.Format("ABDS: looking for dim \"{0}\" for var \"{1}\"", vars[i].Dimensions[0], vars[i].Name)); estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512); } } blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); //blob.WritePages(sizeStream, 0); //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512; byte[] scheme = new byte[sizeAligned]; sizeStream.Seek(0, SeekOrigin.Begin); sizeStream.Read(scheme, 0, 512); memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 512, schemeSize); bufferStream.Write(scheme, 0, sizeAligned); //for (int i = 0; i < sizeAligned; i += maxBlobChunk) // blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i); } } for (int i = 0; i < varsWithDataCount; ++i) { if (vars[i].Dimensions.Length == 1) { int len = dimLengthDictionary[vars[i].Dimensions[0]]; var data = dataToPut[vars[i].Name]; if (vars[i].Type == typeof(DateTime)) { var temp = new Int64[data.Length]; for (int j = 0; j < temp.Length; ++j) { temp[j] = ((DateTime)data.GetValue(j)).Ticks; } data = temp; } int bufferSize = 512 * ((len * vars[i].ValueSize + 511) / 512); byte[] buffer = new byte[bufferSize]; Buffer.BlockCopy(data, 0, buffer, 0, len * vars[i].ValueSize); bufferStream.Write(buffer, 0, bufferSize); //for (int j = 0; j < bufferSize; j += maxBlobChunk) // blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j); } else { int outerDimLen = dimLengthDictionary[vars[i].Dimensions[0]]; int rowLen = vars[i].ValueSize; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowLen *= dimLengthDictionary[vars[i].Dimensions[j]]; } int rowLenUnaligned = rowLen; rowLen = 512 * ((rowLen + 511) / 512); byte[] buffer = new byte[rowLen]; Array data = dataToPut[vars[i].Name]; if (vars[i].Type == typeof(DateTime)) { int[] shapeTemp = new int[data.Rank]; for (int k = 0; k < shapeTemp.Length; ++k) { shapeTemp[k] = data.GetUpperBound(k) + 1; } Array temp = Array.CreateInstance(typeof(Int64), shapeTemp); int[] resPos = new int[shapeTemp.Length]; for (int k = 0; k < resPos.Length; ++k) { resPos[k] = 0; } do { temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos); }while (Move(resPos, shapeTemp)); data = temp; } for (int j = 0; j < outerDimLen; ++j) { Buffer.BlockCopy(data, j * rowLenUnaligned, buffer, 0, rowLenUnaligned); bufferStream.Write(buffer, 0, rowLen); } } } int bufferStreamSize = (int)bufferStream.Length; int bufferStreamSizeAligned = ((bufferStreamSize + 511) / 512) * 512; byte[] bufferAligned = new byte[bufferStreamSizeAligned + 512]; bufferStream.Seek(0, SeekOrigin.Begin); bufferStream.Read(bufferAligned, 0, bufferStreamSize); for (int i = 0; i < bufferStreamSizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(bufferAligned, i, Math.Min(maxBlobChunk, bufferStreamSizeAligned - i)), i); } } return(new AzureBlobDataSet(uri, schemeSize, info)); }
public static AzureBlobDataSet CreateEmptySet(string uri, SerializableDataSetSchema schema) { SerializableDataSetSchema info = schema; List <SerializableDimension> dimensions = schema.Dimensions.ToList(); List <SerializableVariableSchema> vars = schema.Variables.ToList(); Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); CloudPageBlob blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); int schemeSize; using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } estimatedBlobSize += (long)dimLengthDictionary[vars[i].Dimensions[0]] * (long)(((rowSize * vars[i].ValueSize + 511) / 512) * 512); } } blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); //blob.WritePages(sizeStream, 0); //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512 + 512; byte[] scheme = new byte[sizeAligned]; sizeStream.Seek(0, SeekOrigin.Begin); sizeStream.Read(scheme, 0, 512); memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 512, schemeSize); for (int i = 0; i < sizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), i); } } } return(new AzureBlobDataSet(uri, schemeSize, info)); }
public static AzureBlobDataSet ArrangeData(string uri, DataSet source, SerializableVariableSchema[] emptyVariables) { List <SerializableDimension> dimensions = new List <SerializableDimension>(); foreach (var i in source.Dimensions) { dimensions.Add(new SerializableDimension(i.Name, i.Length)); } List <SerializableVariableSchema> oldVars = source.Variables.Select <Variable, SerializableVariableSchema>(x => x.GetSchema().AsSerializable()).ToList(); List <SerializableVariableSchema> vars = new List <SerializableVariableSchema>(oldVars); vars.AddRange(emptyVariables); SerializableDataSetSchema info = new SerializableDataSetSchema(dimensions.ToArray(), vars.ToArray(), source.Metadata.AsDictionary()); Dictionary <string, int> dimLengthDictionary = new Dictionary <string, int>(dimensions.Count); foreach (var i in dimensions) { dimLengthDictionary.Add(i.Name, i.Length); } long estimatedBlobSize = 512;//only scheme size on 1st page long[] varOffsets = new long[vars.Count]; AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } CloudStorageAccount storageAccount = CloudStorageAccount.Parse(azureUri.ConnectionString); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); container.CreateIfNotExist(); container.SetPermissions(new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Container }); CloudPageBlob blob; int schemeSize; using (MemoryStream memStream = new MemoryStream()) { DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); serializer.WriteObject(memStream, info); schemeSize = (int)memStream.Length; estimatedBlobSize += 512 * ((schemeSize + 511) / 512);//remembering the need to align data for (int i = 0; i < vars.Count; ++i) { varOffsets[i] = estimatedBlobSize; if (vars[i].Dimensions.Length == 1) { estimatedBlobSize += ((dimLengthDictionary[vars[i].Dimensions[0]] * vars[i].ValueSize + 511) / 512) * 512; } else { int rowSize = 1; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowSize *= dimLengthDictionary[vars[i].Dimensions[j]]; } estimatedBlobSize += dimLengthDictionary[vars[i].Dimensions[0]] * ((rowSize * vars[i].ValueSize + 511) / 512) * 512; } } blob = container.GetPageBlobReference(azureUri.Blob); blob.DeleteIfExists(); // CRITICAL: some may interfere between calls blob.Create(estimatedBlobSize); //writing scheme size into the 1st page UTF8Encoding utf8 = new UTF8Encoding(); using (MemoryStream sizeStream = new MemoryStream(new byte[512], true)) { byte[] sizeBuf = utf8.GetBytes(schemeSize.ToString()); sizeStream.Write(sizeBuf, 0, sizeBuf.Length); sizeStream.Seek(0, SeekOrigin.Begin); blob.WritePages(sizeStream, 0); } //writing scheme starting with 2nd page int sizeAligned = ((schemeSize + 511) / 512) * 512; byte[] scheme = new byte[sizeAligned]; memStream.Seek(0, SeekOrigin.Begin); memStream.Read(scheme, 0, schemeSize); for (int i = 0; i < sizeAligned; i += maxBlobChunk) { blob.WritePages(new MemoryStream(scheme, i, Math.Min(maxBlobChunk, sizeAligned - i)), 512 + i); } } //populating blob with values from source for (int i = 0; i < oldVars.Count; ++i) { if (oldVars[i].Dimensions.Length == 1) { int len = dimLengthDictionary[oldVars[i].Dimensions[0]]; var data = source[oldVars[i].Name].GetData(); if (oldVars[i].Type == typeof(DateTime)) { var temp = new Int64[data.Length]; for (int j = 0; j < temp.Length; ++j) { temp[j] = ((DateTime)data.GetValue(j)).Ticks; } data = temp; } int bufferSize = 512 * ((len * oldVars[i].ValueSize + 511) / 512); byte[] buffer = new byte[bufferSize]; Buffer.BlockCopy(data, 0, buffer, 0, len * oldVars[i].ValueSize); for (int j = 0; j < bufferSize; j += maxBlobChunk) { blob.WritePages(new MemoryStream(buffer, j, Math.Min(maxBlobChunk, bufferSize - j)), varOffsets[i] + j); } } else { int outerDimLen = dimLengthDictionary[oldVars[i].Dimensions[0]]; int rowLen = vars[i].ValueSize; for (int j = 1; j < vars[i].Dimensions.Length; ++j) { rowLen *= dimLengthDictionary[vars[i].Dimensions[j]]; } int rowLenUnaligned = rowLen; rowLen = 512 * ((rowLen + 511) / 512); int[] origin = new int[oldVars[i].Dimensions.Length]; for (int j = 0; j < origin.Length; ++j) { origin[j] = 0; } int[] shape = new int[oldVars[i].Dimensions.Length]; shape[0] = 1; for (int j = 1; j < origin.Length; ++j) { shape[j] = dimLengthDictionary[oldVars[i].Dimensions[j]]; } byte[] buffer = new byte[rowLen]; for (int j = 0; j < outerDimLen; ++j) { origin[0] = j; Array data = source[oldVars[i].Name].GetData(origin, shape); if (oldVars[i].Type == typeof(DateTime)) { int[] shapeTemp = new int[data.Rank]; for (int k = 0; k < shapeTemp.Length; ++k) { shapeTemp[k] = data.GetUpperBound(k) + 1; } Array temp = Array.CreateInstance(typeof(Int64), shapeTemp); int[] resPos = new int[shapeTemp.Length]; for (int k = 0; k < resPos.Length; ++k) { resPos[k] = 0; } do { temp.SetValue(((DateTime)data.GetValue(resPos)).Ticks, resPos); }while (Move(resPos, shapeTemp)); data = temp; } Buffer.BlockCopy(data, 0, buffer, 0, rowLenUnaligned); for (int k = 0; k < rowLen; k += maxBlobChunk) { blob.WritePages(new MemoryStream(buffer, k, Math.Min(maxBlobChunk, rowLen - k)), varOffsets[i] + (long)rowLen * (long)j + (long)k); } } } } //blob is prepared: values are where they gotta be, trash is everwhere else! return(new AzureBlobDataSet(uri, schemeSize, info)); }
public AzureBlobDataSet(string uri) { AzureBlobDataSetUri azureUri = null; if (DataSetUri.IsDataSetUri(uri)) { azureUri = new AzureBlobDataSetUri(uri); } else { azureUri = AzureBlobDataSetUri.ToUri(uri); } this.uri = azureUri; CloudStorageAccount storageAccount;// = CloudStorageAccount.Parse(azureUri.ConnectionString); if (CloudStorageAccount.TryParse(azureUri.ConnectionString, out storageAccount)) { CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve a reference to a container CloudBlobContainer container = blobClient.GetContainerReference(azureUri.Container); blob = container.GetPageBlobReference(azureUri.Blob); } else { blob = new CloudPageBlob(@"http://" + azureUri.AccountName + @".blob.core.windows.net/" + azureUri.Container + @"/" + azureUri.Blob); _anonymous = true; } SerializableDataSetSchema info; Int32 schemeSize; using (BinaryReader br = new BinaryReader(blob.OpenRead())) { int curOffset = 0; int curCount = 512; int temp = 0; UTF8Encoding utf8 = new UTF8Encoding(); byte[] buffer = new byte[512]; do { temp = br.BaseStream.Read(buffer, curOffset, curCount); curOffset += temp; curCount -= temp; } while (curOffset < 512); string sizeStr = utf8.GetString(buffer); schemeSize = Int32.Parse(sizeStr); br.BaseStream.Seek(512, SeekOrigin.Begin); DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(SerializableDataSetSchema)); byte[] scheme = new byte[schemeSize]; curOffset = 0; curCount = schemeSize; do { temp = br.BaseStream.Read(scheme, curOffset, curCount); curOffset += temp; curCount -= temp; } while (curOffset < schemeSize); info = (SerializableDataSetSchema)serializer.ReadObject(new MemoryStream(scheme)); } bool savedAutoCommitState = IsAutocommitEnabled; IsAutocommitEnabled = false; Initialize(schemeSize, info); Commit(); if (_anonymous) { this.SetCompleteReadOnly(); } IsAutocommitEnabled = savedAutoCommitState; _IsInitialized = true; }