public void CloudBlobContainerGetBlobReferenceFromServer() { CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); CloudBlockBlob blockBlob = container.GetBlockBlobReference("bb"); blockBlob.PutBlockList(new string[] {}); CloudPageBlob pageBlob = container.GetPageBlobReference("pb"); pageBlob.Create(0); ICloudBlob blob1 = container.GetBlobReferenceFromServer("bb"); Assert.IsInstanceOfType(blob1, typeof(CloudBlockBlob)); ICloudBlob blob2 = container.GetBlobReferenceFromServer("pb"); Assert.IsInstanceOfType(blob2, typeof(CloudPageBlob)); ICloudBlob blob3 = container.ServiceClient.GetBlobReferenceFromServer(blockBlob.Uri); Assert.IsInstanceOfType(blob3, typeof(CloudBlockBlob)); ICloudBlob blob4 = container.ServiceClient.GetBlobReferenceFromServer(pageBlob.Uri); Assert.IsInstanceOfType(blob4, typeof(CloudPageBlob)); } finally { container.DeleteIfExists(); } }
public void SetBlobContentWithProperties(StorageBlob.BlobType blobType) { string filePath = FileUtil.GenerateOneTempTestFile(); StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(); Hashtable properties = new Hashtable(); properties.Add("CacheControl", Utility.GenNameString(string.Empty)); properties.Add("ContentEncoding", Utility.GenNameString(string.Empty)); properties.Add("ContentLanguage", Utility.GenNameString(string.Empty)); properties.Add("ContentMD5", Utility.GenNameString(string.Empty)); properties.Add("ContentType", Utility.GenNameString(string.Empty)); try { Test.Assert(agent.SetAzureStorageBlobContent(filePath, container.Name, blobType, string.Empty, true, -1, properties), "set blob content with property should succeed"); StorageBlob.ICloudBlob blob = container.GetBlobReferenceFromServer(Path.GetFileName(filePath)); blob.FetchAttributes(); ExpectEqual(properties["CacheControl"].ToString(), blob.Properties.CacheControl, "Cache control"); ExpectEqual(properties["ContentEncoding"].ToString(), blob.Properties.ContentEncoding, "Content Encoding"); ExpectEqual(properties["ContentLanguage"].ToString(), blob.Properties.ContentLanguage, "Content Language"); ExpectEqual(properties["ContentMD5"].ToString(), blob.Properties.ContentMD5, "Content MD5"); ExpectEqual(properties["ContentType"].ToString(), blob.Properties.ContentType, "Content Type"); } finally { blobUtil.RemoveContainer(container.Name); FileUtil.RemoveFile(filePath); } }
public void deleteAll(CloudBlobContainer container, string pathInSyncFolder) { // update path on server pathInSyncFolder = pathInSyncFolder.Replace("\\", "/"); // check if it is dir bool isDir = false; CloudBlobDirectory dir = container.GetDirectoryReference(pathInSyncFolder); List<IListBlobItem> blobs = dir.ListBlobs().ToList(); if (blobs.Count != 0) { isDir = true; } try { if (!isDir) { // this can only get the blob type, not the dir type var item = container.GetBlobReferenceFromServer(pathInSyncFolder); if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)item; blob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots); //blob.Delete(); //System.Windows.Forms.MessageBox.Show(string.Format("File: {0} Deleted!", pathInSyncFolder), "DBLike Server"); } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob blob = (CloudPageBlob)item; //blob.Delete(); blob.DeleteIfExists(DeleteSnapshotsOption.IncludeSnapshots); //System.Windows.Forms.MessageBox.Show(string.Format("File: {0} Deleted!", pathInSyncFolder), "DBLike Server"); Program.ServerForm.addtoConsole(string.Format("File: {0} Deleted!", pathInSyncFolder)); } } else { // get the directory reference CloudBlobDirectory dira = container.GetDirectoryReference(pathInSyncFolder); deleteFolder(container, dira); //System.Windows.Forms.MessageBox.Show(string.Format("Deleted!\n Folder: {0}", pathInSyncFolder), "DBLike Server"); Program.ServerForm.addtoConsole(string.Format("Deleted!\n Folder: {0}", pathInSyncFolder)); } } catch { System.Windows.Forms.MessageBox.Show(string.Format("File: {0} \n doesn't exist!", pathInSyncFolder), "DBLike Server"); Program.ServerForm.addtoConsole(string.Format("File: {0} \n doesn't exist!", pathInSyncFolder)); } }
public void SetBlobContentWithMetadata(StorageBlob.BlobType blobType) { string filePath = FileUtil.GenerateOneTempTestFile(); StorageBlob.CloudBlobContainer container = blobUtil.CreateContainer(); Hashtable metadata = new Hashtable(); int metaCount = GetRandomTestCount(); for (int i = 0; i < metaCount; i++) { string key = Utility.GenRandomAlphabetString(); string value = Utility.GenNameString(string.Empty); if (!metadata.ContainsKey(key)) { Test.Info(string.Format("Add meta key: {0} value : {1}", key, value)); metadata.Add(key, value); } } try { Test.Assert(agent.SetAzureStorageBlobContent(filePath, container.Name, blobType, string.Empty, true, -1, null, metadata), "set blob content with meta should succeed"); StorageBlob.ICloudBlob blob = container.GetBlobReferenceFromServer(Path.GetFileName(filePath)); blob.FetchAttributes(); ExpectEqual(metadata.Count, blob.Metadata.Count, "meta data count"); foreach (string key in metadata.Keys) { ExpectEqual(metadata[key].ToString(), blob.Metadata[key], "Meta data key " + key); } } finally { blobUtil.RemoveContainer(container.Name); FileUtil.RemoveFile(filePath); } }
public void CloudBlobContainerGetBlobReferenceFromServer() { CloudBlobContainer container = GetRandomContainerReference(); try { container.Create(); SharedAccessBlobPolicy policy = new SharedAccessBlobPolicy() { Permissions = SharedAccessBlobPermissions.Read, SharedAccessStartTime = DateTimeOffset.UtcNow.AddMinutes(-5), SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddMinutes(30), }; CloudBlockBlob blockBlob = container.GetBlockBlobReference("bb"); blockBlob.PutBlockList(new string[] {}); CloudPageBlob pageBlob = container.GetPageBlobReference("pb"); pageBlob.Create(0); ICloudBlob blob1 = container.GetBlobReferenceFromServer("bb"); Assert.IsInstanceOfType(blob1, typeof(CloudBlockBlob)); CloudBlockBlob blob1Snapshot = ((CloudBlockBlob)blob1).CreateSnapshot(); blob1.SetProperties(); Uri blob1SnapshotUri = new Uri(blob1Snapshot.Uri.AbsoluteUri + "?snapshot=" + blob1Snapshot.SnapshotTime.Value.UtcDateTime.ToString("o")); ICloudBlob blob1SnapshotReference = container.ServiceClient.GetBlobReferenceFromServer(blob1SnapshotUri); AssertAreEqual(blob1Snapshot.Properties, blob1SnapshotReference.Properties); ICloudBlob blob2 = container.GetBlobReferenceFromServer("pb"); Assert.IsInstanceOfType(blob2, typeof(CloudPageBlob)); CloudPageBlob blob2Snapshot = ((CloudPageBlob)blob2).CreateSnapshot(); blob2.SetProperties(); Uri blob2SnapshotUri = new Uri(blob2Snapshot.Uri.AbsoluteUri + "?snapshot=" + blob2Snapshot.SnapshotTime.Value.UtcDateTime.ToString("o")); ICloudBlob blob2SnapshotReference = container.ServiceClient.GetBlobReferenceFromServer(blob2SnapshotUri); AssertAreEqual(blob2Snapshot.Properties, blob2SnapshotReference.Properties); ICloudBlob blob3 = container.ServiceClient.GetBlobReferenceFromServer(blockBlob.Uri); Assert.IsInstanceOfType(blob3, typeof(CloudBlockBlob)); ICloudBlob blob4 = container.ServiceClient.GetBlobReferenceFromServer(pageBlob.Uri); Assert.IsInstanceOfType(blob4, typeof(CloudPageBlob)); string blockBlobToken = blockBlob.GetSharedAccessSignature(policy); StorageCredentials blockBlobSAS = new StorageCredentials(blockBlobToken); Uri blockBlobSASUri = blockBlobSAS.TransformUri(blockBlob.Uri); string pageBlobToken = pageBlob.GetSharedAccessSignature(policy); StorageCredentials pageBlobSAS = new StorageCredentials(pageBlobToken); Uri pageBlobSASUri = pageBlobSAS.TransformUri(pageBlob.Uri); ICloudBlob blob5 = container.ServiceClient.GetBlobReferenceFromServer(blockBlobSASUri); Assert.IsInstanceOfType(blob5, typeof(CloudBlockBlob)); ICloudBlob blob6 = container.ServiceClient.GetBlobReferenceFromServer(pageBlobSASUri); Assert.IsInstanceOfType(blob6, typeof(CloudPageBlob)); CloudBlobClient client7 = new CloudBlobClient(container.ServiceClient.BaseUri, blockBlobSAS); ICloudBlob blob7 = client7.GetBlobReferenceFromServer(blockBlobSASUri); Assert.IsInstanceOfType(blob7, typeof(CloudBlockBlob)); CloudBlobClient client8 = new CloudBlobClient(container.ServiceClient.BaseUri, pageBlobSAS); ICloudBlob blob8 = client8.GetBlobReferenceFromServer(pageBlobSASUri); Assert.IsInstanceOfType(blob8, typeof(CloudPageBlob)); } finally { container.DeleteIfExists(); } }
/// <summary> /// Get blob reference with properties and meta data from server /// </summary> /// <param name="container">A cloudblobcontainer object</param> /// <param name="blobName">Blob name</param> /// <param name="accessCondition">Access condition</param> /// <param name="options">Blob request options</param> /// <param name="operationContext">Operation context</param> /// <returns>Return an ICloudBlob if the specific blob exists on azure, otherwise return null</returns> public ICloudBlob GetBlobReferenceFromServer(CloudBlobContainer container, string blobName, AccessCondition accessCondition, BlobRequestOptions options, OperationContext operationContext) { try { ICloudBlob blob = container.GetBlobReferenceFromServer(blobName, accessCondition, options, operationContext); return blob; } catch(StorageException e) { if (e.IsNotFoundException()) { return null; } else { throw; } } }
/// <summary> /// Gets a reference to a blob by making a request to the service. /// Note that the GetBlobReferenceFromServer method is called synchronously, for the purposes of the sample. However, in a real-world /// application using the async/await pattern, best practices recommend using asynchronous methods consistently. /// </summary> /// <param name="container">The container.</param> /// <param name="blobName">The blob name.</param> /// <returns>A Task object.</returns> private static void GetExistingBlobReference(CloudBlobContainer container, string blobName) { try { // Get a reference to a blob with a request to the server. // If the blob does not exist, this call will fail with a 404 (Not Found). ICloudBlob blob = container.GetBlobReferenceFromServer(blobName); // The previous call gets the blob's properties, so it's not necessary to call FetchAttributes // to read a property. Console.WriteLine("Blob {0} was last modified at {1} local time.", blobName, blob.Properties.LastModified.Value.LocalDateTime); } catch (StorageException e) { if (e.RequestInformation.HttpStatusCode == 404) { Console.WriteLine("Blob {0} does not exist.", blobName); Console.WriteLine("Additional error information: " + e.Message); } else { Console.WriteLine(e.Message); Console.ReadLine(); throw; } } Console.WriteLine(); }
private void DownloadFiles(CloudBlobContainer logsContainer, List<DownloadListItem> downloadList) { if (downloadList.Count == 0) { return; } long bytes = downloadList.Sum(x => x.Length); Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "Downloading {0} files, {1} MB..", downloadList.Count, bytes / (1024 * 1024))); foreach (DownloadListItem item in downloadList) { ICloudBlob blobRef = null; try { blobRef = logsContainer.GetBlobReferenceFromServer(item.RemoteBlobPath); } catch (StorageException se) { Stdout.WriteLine("Error accessing blob on server. skipping. Exception=" + se.Message); } Utils.CreateLocalFolderIfNotExists(Path.GetDirectoryName(item.LocalFullPath)); Stdout.Write(string.Format(CultureInfo.CurrentCulture, item.RemoteBlobPath + " ({0} MB)..", item.Length / (1024 * 1024))); try { blobRef.DownloadToFile(item.LocalFullPath, FileMode.Create); Stdout.WriteLine("done."); } catch (StorageException se) { Stdout.WriteLine("Error: " + se.Message); } } }
private List<DownloadListItem> CalculateDownloadList( CloudBlobContainer logsContainer, string rootfolder, string cachefolder, List<string> blobList) { var downloadList = new List<DownloadListItem>(); long countFilesMatchingTimeRange = 0; Utils.CreateLocalFolderIfNotExists(rootfolder); foreach (string blobName in blobList) { int idx = 0; int year = int.Parse(blobName.Substring(idx, 4), CultureInfo.InvariantCulture); int month = int.Parse(blobName.Substring(idx + 5, 2), CultureInfo.InvariantCulture); int day = int.Parse(blobName.Substring(idx + 8, 2), CultureInfo.InvariantCulture); int hour = int.Parse(blobName.Substring(idx + 11, 2), CultureInfo.InvariantCulture); string fileName = blobName.Substring(idx + 16); var logfileDateMin = new DateTime(year, month, day, hour, 0, 0, DateTimeKind.Utc); // TODO: review UTC/Local conversions. DateTime logfileDateMax = logfileDateMin.AddHours(1); string pathComponents = string.Format( CultureInfo.InvariantCulture, "{0:00}\\{1:00}\\{2:00}\\{3:00}00\\{4}", year, month, day, hour, fileName); string localPath = Path.Combine(cachefolder, pathComponents); string uriComponents = string.Format( CultureInfo.InvariantCulture, "{0:00}/{1:00}/{2:00}/{3:00}00/{4}", year, month, day, hour, fileName); string blobPath = "blob/" + uriComponents; Stdout.VerboseWrite(blobPath + " .. "); ICloudBlob blobRef = null; try { blobRef = logsContainer.GetBlobReferenceFromServer(blobPath); } catch (StorageException se) { Stdout.WriteLine("Error accessing blob on server. skipping. Exception=" + se.Message); continue; } bool needsDownload = true; if (this.startTime < logfileDateMax && this.endTime > logfileDateMin) { countFilesMatchingTimeRange++; if (File.Exists(localPath)) { var info = new FileInfo(localPath); if (blobRef.Properties.Length == info.Length) { Stdout.VerboseWrite("cached OK. "); needsDownload = false; } else { Stdout.VerboseWrite("cached but doesn't match. "); } } if (needsDownload) { Stdout.VerboseWrite("requires download.."); downloadList.Add(new DownloadListItem(blobPath, localPath, blobRef.Properties.Length)); } } else { Stdout.VerboseWrite("not required"); } Stdout.VerboseWriteLine(); } Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "{0} files in specified time range", countFilesMatchingTimeRange)); Stdout.WriteLine(string.Format(CultureInfo.CurrentCulture, "{0} files require download", downloadList.Count)); return downloadList; }
private void RenameBlob(CloudBlobContainer container, string oldName, string newName) { var source = container.GetBlobReferenceFromServer(oldName); var target = container.GetBlockBlobReference(newName); target.StartCopy(source.Uri); while (target.CopyState.Status == CopyStatus.Pending) Task.Delay(100).Wait(); if (target.CopyState.Status != CopyStatus.Success) throw new ApplicationException("Rename failed: " + target.CopyState.Status); source.Delete(); }
/// <summary> /// Gets the content from azure as a stream. /// </summary> /// <param name="blobName"> /// Name of the blob. /// </param> /// <param name="outputStream"> /// The content is exposed as output stream. /// </param> /// <param name="container"> /// COntainer where we could find the blob. /// </param> /// <returns> /// The blob properties. /// </returns> private static BlobProperties GetContent(string blobName, Stream outputStream, CloudBlobContainer container) { var blob = container.GetBlobReferenceFromServer(blobName.ToUpperInvariant()); blob.DownloadToStream(outputStream); return blob.Properties; }
/// <summary> /// Gets the content from azure as a stream. /// </summary> /// <param name="blobName"> /// Name of the blob. /// </param> /// <param name="outputStream"> /// The content is exposed as output stream. /// </param> /// <param name="container"> /// Container where we could find the blob. /// </param> /// <returns> /// The blob properties. /// </returns> private static BlobProperties GetContent(string blobName, Stream outputStream, CloudBlobContainer container) { new { blobName, outputStream, container }.CheckNotNull(); var blob = container.GetBlobReferenceFromServer(blobName.ToUpperInvariant()); blob.DownloadToStream(outputStream, null, new BlobRequestOptions() { ServerTimeout = TimeSpan.FromMinutes(30) }); return blob.Properties; }
/// <summary> /// Execute method is the only method of IDotNetActivity interface you must implement. /// In this sample, the method invokes the Calculate method to perform the core logic. /// </summary> public IDictionary<string, string> Execute( IEnumerable<LinkedService> linkedServices, IEnumerable<Dataset> datasets, Activity activity, IActivityLogger logger) { ///////////////// // Log input parameters // to get extended properties (for example: SliceStart) foreach (LinkedService ls in linkedServices) { logger.Write("linkedServices: {0}, {1}, {2}, {3}", ls.Name, ls.Properties.Type, ls.Properties.Description, ls.Properties.ErrorMessage); } var sliceYear = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Year"]; var sliceMonth = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Month"]; var sliceDay = ((DotNetActivity)activity.TypeProperties).ExtendedProperties["Day"]; logger.Write("dataSlice: {0}-{1}-{2}", sliceYear, sliceMonth, sliceDay); ///////////////// // Open up input Blob var inputDataset = datasets.Single(dataset => dataset.Name == activity.Inputs.Single().Name); var inputLinkedService = linkedServices.Single( linkedService => linkedService.Name == inputDataset.Properties.LinkedServiceName); var inputLocation = new BlobLocation(inputLinkedService, inputDataset, sliceYear, sliceMonth, sliceDay); var inputContainer = new CloudBlobContainer(inputLocation.ConnectionSasUri); var sourceBlob = inputContainer.GetBlobReferenceFromServer(inputLocation.BlobFullPath); //////////////// // Get output location var outputDataset = datasets.Single(dataset => dataset.Name == activity.Outputs.Single().Name); var outputLinkedService = linkedServices.Single( linkedService => linkedService.Name == outputDataset.Properties.LinkedServiceName); var outputLocation = new BlobLocation(outputLinkedService, outputDataset, sliceYear, sliceMonth, sliceDay); CloudStorageAccount outputStorageAccount = CloudStorageAccount.Parse(outputLocation.ConnectionString); CloudBlobClient outputClient = outputStorageAccount.CreateCloudBlobClient(); var outContainer = outputClient.GetContainerReference(outputLocation.ContainerName); outContainer.CreateIfNotExists(); //format output path string var outputFilenameFormatString = outputLocation.BlobFullPath; using (var sourceBlobStream = sourceBlob.OpenRead()) using (var unZipStream = new System.IO.Compression.GZipStream(sourceBlobStream, System.IO.Compression.CompressionMode.Decompress)) using (var tarStream = new TarStream(unZipStream)) { logger.Write("BlobRead: {0}/{1}", inputLocation.ContainerName, inputLocation.BlobFullPath); while (tarStream.NextFile()) { var tableName = Path.GetFileNameWithoutExtension(tarStream.CurrentFilename); var taredFileExtention = Path.GetExtension(tarStream.CurrentFilename); if (taredFileExtention == ".bson") { var outputBlob = outContainer.GetBlockBlobReference(outputFilenameFormatString.Replace("{EventName}", tableName)); using (var outBlobStream = outputBlob.OpenWrite()) using (var gzipOut = new GZipStream(outBlobStream, System.IO.Compression.CompressionLevel.Optimal)) using (var outText = new StreamWriter(gzipOut, Encoding.UTF8)) using (var reader = new BsonReader(tarStream)) { logger.Write("BlobWrite: {0}/{1}", outputLocation.ContainerName, outputBlob.Name); reader.CloseInput = false; var jsonSerializer = new JsonSerializer(); reader.ReadRootValueAsArray = false; reader.SupportMultipleContent = true; while (reader.Read()) { var row = (JObject)jsonSerializer.Deserialize(reader); var outString = row.ToString(Formatting.None); outText.WriteLine(outString); } } } } ; } // return a new Dictionary object return new Dictionary<string, string>(); }
/// <summary> /// Print blob copy progress /// </summary> /// <param name="blobContainer">destination container name</param> /// <param name="blobName">blob name</param> private void WaitForBlobCopy(CloudBlobContainer blobContainer, string blobName) { string methodName = System.Reflection.MethodBase.GetCurrentMethod().Name; Logger.Info(methodName, ProgressResources.ExecutionStarted, ResourceType.Blob.ToString(), blobName); double bytesCopied = 0; // Initial status CloudPageBlob blob = (CloudPageBlob)blobContainer.GetBlobReferenceFromServer(blobName); CopyStatus copyStatus = blob.CopyState.Status; // Loop until status becomes success while (copyStatus == CopyStatus.Pending) { try { Task.Delay(Constants.DelayTimeInMilliseconds).Wait(); blob = (CloudPageBlob)blobContainer.GetBlobReferenceFromServer(blobName); copyStatus = blob.CopyState.Status; if (blob.CopyState.BytesCopied.HasValue) bytesCopied = blob.CopyState.BytesCopied.Value; var totalBytes = blob.CopyState.TotalBytes; if (totalBytes.HasValue) { // Print status dcMigration.ReportProgress(string.Format(ProgressResources.CopyBlobProgressInPercentage, blob.Name, (bytesCopied / totalBytes.Value) * 100)); } } catch (Exception ex) { Logger.Error(methodName, ex, ResourceType.Blob.ToString(), blobName); } } if (copyStatus == CopyStatus.Aborted || copyStatus == CopyStatus.Failed || copyStatus == CopyStatus.Invalid) { blob = (CloudPageBlob)blobContainer.GetBlobReferenceFromServer(blobName); BlobRequestOptions requestOptions = Retry.GetBlobRequestOptions(importParameters.DeltaBackOff, importParameters.RetryCount); blob.Delete(DeleteSnapshotsOption.IncludeSnapshots, null, requestOptions, null); Logger.Info(methodName, string.Format(ProgressResources.DeleteNonSuccessBlob, copyStatus), ResourceType.Blob.ToString(), blobName); throw new Exception(string.Format(ProgressResources.DeleteNonSuccessBlob, copyStatus)); } Logger.Info(methodName, ProgressResources.ExecutionCompleted, ResourceType.Blob.ToString(), blobName); }
// delete one item public void deleteItem(CloudBlobContainer container, string pathInSyncFolder) { // get blob var item = container.GetBlobReferenceFromServer(pathInSyncFolder); try { if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)item; blob.Delete(DeleteSnapshotsOption.IncludeSnapshots); //System.Windows.Forms.MessageBox.Show(string.Format("File: {0} Deleted!", pathInSyncFolder), "DBLike Server"); } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob blob = (CloudPageBlob)item; blob.Delete(DeleteSnapshotsOption.IncludeSnapshots); //System.Windows.Forms.MessageBox.Show(string.Format("File: {0} Deleted!", pathInSyncFolder), "DBLike Server"); } } catch { throw; } }