/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { IEnumerable <IListBlobItem> azureBlobList; CloudBlobContainer container; if (string.IsNullOrEmpty(containerName)) { container = client.GetRootContainerReference(); // add container. // Assuming no blobs at root level. // incorrect assumption. FIXME! var containerList = client.ListContainers(); foreach (var cont in containerList) { var b = new BasicBlobContainer(); b.Name = cont.Name; b.DisplayName = AzureHelper.GetDisplayName(cont.Name); b.Container = ""; b.Url = cont.Uri.AbsoluteUri; b.BlobType = BlobEntryType.Container; yield return(b); } } else { container = client.GetContainerReference(containerName); // if we were only passed the container name, then list contents of container. if (string.IsNullOrEmpty(blobPrefix)) { // add blobs azureBlobList = container.ListBlobs(useFlatBlobListing: true); } else { throw new NotImplementedException(); // if passed virtual directory information, then filter based off that. //var vd = container.GetDirectoryReference(virtualDirectoryName); //azureBlobList = vd.ListBlobs(); } foreach (var blob in azureBlobList) { var b = new BasicBlobContainer(); var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri); b.Name = bn; var sp = bn.Split('/'); var displayName = sp[sp.Length - 1]; b.DisplayName = displayName; b.Container = blob.Container.Name; b.Url = blob.Uri.AbsoluteUri; b.BlobType = BlobEntryType.Blob; yield return(b); } } }
/// <summary> /// Lists all blobs in a container. /// Recurse directories to get all contents. /// Either do it here, or when reading blob (and it turns out to be a directory). /// Always recursing might not always be wanted, but think its a good default (for now). /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { //var metadata = client.GetMetaData(containerName, null, false, false); var metadata = client.GetMetaData(path: containerName); if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } // generate list of dirs and files. foreach (var entry in metadata.Contents) { // basic blob info. var blob = new BasicBlobContainer(); blob.Container = containerName; blob.DisplayName = entry.Name; blob.Url = "https://dropbox.com" + entry.Path; blob.BlobPrefix = blobPrefix; if (entry.Is_Dir) { blob.BlobType = BlobEntryType.Container; var newContainerName = string.Empty; if (containerName.EndsWith("/")) { newContainerName = containerName + entry.Name + "/"; } else { newContainerName = containerName + "/" + entry.Name + "/"; } var recursiveBlobList = ListBlobsInContainer(newContainerName, blobPrefix, debug); foreach (var d in recursiveBlobList) { yield return(d); } } else { //var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name; var name = entry.Name; blob.Name = name; blob.BlobType = BlobEntryType.Blob; yield return(blob); } } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var files = Directory.EnumerateFiles(baseUrl); foreach (var file in files) { var f = new BasicBlobContainer(); var name = Path.GetFileName(file); f.BlobType = BlobEntryType.Blob; f.DisplayName = name; f.Url = file; f.Name = name; yield return(f); } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var skydriveListing = SkyDriveHelper.ListSkyDriveDirectoryContent(containerName); foreach (var skyDriveEntry in skydriveListing) { var blob = new BasicBlobContainer(); blob.Name = skyDriveEntry.Name; var resolvedOneDriveEntry = SkyDriveHelper.ListSkyDriveFileWithUrl(skyDriveEntry.Id); // keep display name same as name until determine otherwise. blob.DisplayName = blob.Name; blob.Container = containerName; blob.Url = string.Format("{0}&blobName={1}", resolvedOneDriveEntry.Source, blob.Name); // modify link so we can determine blob name purely from link. yield return(blob); } }
/// <summary> /// Makes a usable URL for a blob. This will need to handle security on the source blob. /// Each cloud provider is different. /// Cloud providers developed: /// Azure /// S3xx /// /// Cloud providers soon: /// Dropbox /// Onedrive /// </summary> /// <param name="origBlob"></param> /// <returns></returns> private static string GeneratedAccessibleUrl(BasicBlobContainer origBlob) { var sourceUrl = origBlob.Url; // +"/" + origBlob.Name; string url = ""; // if S3, then generate signed url. if (S3Helper.MatchHandler(sourceUrl)) { var bucket = S3Helper.GetBucketFromUrl(sourceUrl); var key = origBlob.Name; url = S3Helper.GeneratePreSignedUrl(bucket, key); } else if (AzureHelper.MatchHandler(sourceUrl)) { // generate Azure signed url. var client = AzureHelper.GetSourceCloudBlobClient(sourceUrl); var policy = new SharedAccessBlobPolicy(); policy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-1); policy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ConfigHelper.SharedAccessSignatureDurationInSeconds / 60); policy.Permissions = SharedAccessBlobPermissions.Read; var blob = client.GetBlobReferenceFromServer(new Uri(sourceUrl)); url = sourceUrl + blob.GetSharedAccessSignature(policy); } else if (DropboxHelper.MatchHandler(sourceUrl)) { // need shorter url. (no base url); var uri = new Uri(sourceUrl); var shortUrl = uri.PathAndQuery; var client = DropboxHelper.GetClient(); var media = client.GetMedia(shortUrl); return(media.Url); } else if (SkyDriveHelper.MatchHandler(sourceUrl)) { throw new NotImplementedException("Blobcopy against onedrive is not implemented yet"); } Console.WriteLine("shared url is {0}", url); return(url); }
/// <summary> /// List containers/directories off the root. For storage schemes that allow real directories maybe /// the root will be /// </summary> /// <returns></returns> public List <BasicBlobContainer> ListContainers(string root) { var dirListing = new List <BasicBlobContainer>(); var containerName = ""; var blobPrefix = ""; //var metadata = client.GetMetaData(containerName, null, false, false); var metadata = client.GetMetaData(); // generate list of dirs and files. foreach (var entry in metadata.Contents) { // basic blob info. var blob = new BasicBlobContainer(); blob.Container = containerName; blob.DisplayName = entry.Name; blob.Url = entry.Path; blob.BlobPrefix = blobPrefix; if (entry.Is_Dir) { blob.BlobType = BlobEntryType.Container; blob.Name = entry.Name; dirListing.Add(blob); } else { //var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name; var name = entry.Name; blob.Name = containerName + name; blob.BlobType = BlobEntryType.Blob; dirListing.Add(blob); } } return(dirListing); }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { IEnumerable<IListBlobItem> azureBlobList; CloudBlobContainer container; if (string.IsNullOrEmpty(containerName)) { container = client.GetRootContainerReference(); // add container. // Assuming no blobs at root level. // incorrect assumption. FIXME! var containerList = client.ListContainers(); foreach (var cont in containerList) { var b = new BasicBlobContainer(); b.Name = cont.Name; b.DisplayName = AzureHelper.GetDisplayName(cont.Name); b.Container = ""; b.Url = cont.Uri.AbsoluteUri; b.BlobType = BlobEntryType.Container; yield return b; } } else { container = client.GetContainerReference(containerName); // if we were only passed the container name, then list contents of container. if (string.IsNullOrEmpty(blobPrefix)) { // add blobs azureBlobList = container.ListBlobs(useFlatBlobListing:true); } else { throw new NotImplementedException(); // if passed virtual directory information, then filter based off that. //var vd = container.GetDirectoryReference(virtualDirectoryName); //azureBlobList = vd.ListBlobs(); } foreach (var blob in azureBlobList) { var b = new BasicBlobContainer(); var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri); b.Name = bn; var sp = bn.Split('/'); var displayName = sp[ sp.Length -1]; b.DisplayName = displayName; b.Container = blob.Container.Name; b.Url = blob.Uri.AbsoluteUri; b.BlobType = BlobEntryType.Blob; yield return b; } } }
/// <summary> /// Makes a usable URL for a blob. This will need to handle security on the source blob. /// Each cloud provider is different. /// Cloud providers developed: /// Azure /// S3xx /// /// Cloud providers soon: /// Dropbox /// Onedrive /// </summary> /// <param name="origBlob"></param> /// <returns></returns> private static string GeneratedAccessibleUrl( BasicBlobContainer origBlob) { var sourceUrl = origBlob.Url; // +"/" + origBlob.Name; string url = ""; // if S3, then generate signed url. if (S3Helper.MatchHandler(sourceUrl)) { var bucket = S3Helper.GetBucketFromUrl(sourceUrl); var key = origBlob.Name; url = S3Helper.GeneratePreSignedUrl(bucket, key); } else if (AzureHelper.MatchHandler( sourceUrl)) { // generate Azure signed url. var client = AzureHelper.GetSourceCloudBlobClient(sourceUrl); var policy = new SharedAccessBlobPolicy(); policy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-1); policy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes( ConfigHelper.SharedAccessSignatureDurationInSeconds / 60 ); policy.Permissions = SharedAccessBlobPermissions.Read; var blob = client.GetBlobReferenceFromServer( new Uri(sourceUrl)); url = sourceUrl+ blob.GetSharedAccessSignature(policy); } else if (DropboxHelper.MatchHandler(sourceUrl)) { // need shorter url. (no base url); var uri = new Uri(sourceUrl); var shortUrl = uri.PathAndQuery; var client = DropboxHelper.GetClient(); var media = client.GetMedia(shortUrl); return media.Url; } else if (SkyDriveHelper.MatchHandler( sourceUrl)) { throw new NotImplementedException("Blobcopy against onedrive is not implemented yet"); } return url; }
/// <summary> /// Get all possible blobs for the handler (which had the original url passed into the constructor). /// </summary> /// <param name="inputHandler"></param> /// <param name="url"></param> /// <returns></returns> private static IEnumerable<BasicBlobContainer> GetSourceBlobList(IBlobHandler inputHandler) { var containerName = inputHandler.GetContainerNameFromUrl(inputHandler.GetBaseUrl()); if (CommonHelper.IsABlob(inputHandler.GetBaseUrl())) { var blobName = inputHandler.GetBlobNameFromUrl( inputHandler.GetBaseUrl()); var blob = new BasicBlobContainer{ Name = blobName, DisplayName = blobName, BlobType = BlobEntryType.Blob, Url = inputHandler.GetBaseUrl(), Container = containerName }; yield return blob; //blobList.Add(blob); } else { var res = inputHandler.ListBlobsInContainer(containerName); foreach( var i in res) { yield return i; } } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var bucket = containerName; using (IAmazonS3 client = S3Helper.GenerateS3Client(ConfigHelper.SrcAWSAccessKeyID, ConfigHelper.SrcAWSSecretAccessKeyID, bucket)) { var request = new ListObjectsRequest(); request.BucketName = bucket; if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } if (!string.IsNullOrEmpty(blobPrefix)) { request.Prefix = blobPrefix; } // FIXME... check virtual directories workhere. do { ListObjectsResponse response = client.ListObjects(request); foreach (var obj in response.S3Objects) { string fullPath; if (baseUrlProvided) { //var fullPath = GenerateUrl(baseUrl, bucket, obj.Key); fullPath = baseUrl + obj.Key; } else { fullPath = string.Format("https://{0}/{1}/{2}", baseUrl, containerName, obj.Key); } if (!fullPath.EndsWith("/")) { //var fullPath = Path.Combine(baseUrl, obj.Key); var blob = new BasicBlobContainer(); blob.Name = obj.Key; blob.Url = fullPath; blob.Container = bucket; blob.BlobType = BlobEntryType.Blob; blob.DisplayName = S3Helper.GetDisplayName(blob.Name); blob.BlobPrefix = blobPrefix; yield return(blob); //blobList.Add(blob); } } if (response.IsTruncated) { request.Marker = response.NextMarker; } else { request = null; } } while (request != null); } }
// have destination location. // have original blob name and prefix // new name should be destination name + (blob.name - blob.prefix) public static BlobCopyData StartCopy(BasicBlobContainer origBlob, string DestinationUrl, DestinationBlobType destBlobType, bool skipIfExists) { var client = AzureHelper.GetTargetCloudBlobClient(DestinationUrl); var opt = client.GetServiceProperties(); var containerName = AzureHelper.GetContainerFromUrl(DestinationUrl); var destBlobPrefix = AzureHelper.GetBlobFromUrl(DestinationUrl); var container = client.GetContainerReference(containerName); container.CreateIfNotExists(); ICloudBlob blob = null; var url = GeneratedAccessibleUrl(origBlob); string blobName; string blobWithoutPrefix = string.Empty; if (!string.IsNullOrWhiteSpace(origBlob.BlobPrefix) && origBlob.Name.Length > origBlob.BlobPrefix.Length) { blobWithoutPrefix = origBlob.Name.Substring(origBlob.BlobPrefix.Length); } if (!string.IsNullOrWhiteSpace(blobWithoutPrefix)) { blobName = string.Format("{0}/{1}", destBlobPrefix, blobWithoutPrefix); } else { // need to get just filename. ie last element of / var actualBlobName = origBlob.Name.Split('/').Last(); if (string.IsNullOrWhiteSpace(destBlobPrefix)) { blobName = actualBlobName; } else { blobName = string.Format("{0}/{1}", destBlobPrefix, actualBlobName); } } // include unknown for now. Unsure. if (destBlobType == DestinationBlobType.Block || destBlobType == DestinationBlobType.Unknown) { blob = container.GetBlockBlobReference(blobName); } else if (destBlobType == DestinationBlobType.Page) { blob = container.GetPageBlobReference(blobName); } if (skipIfExists) { try { blob.Exists(); } catch (Exception) { Console.WriteLine("Skipping {0}", url); return(null); } } if (blob != null) { try { // crazy large values, want to try and debug an issue. var brOptions = new BlobRequestOptions(); brOptions.MaximumExecutionTime = new TimeSpan(0, maxExecutionTimeInMins, 0); brOptions.ServerTimeout = new TimeSpan(0, maxServerTimeoutInMins, 0); // return copyID incase user wants to kill the process later. var copyID = blob.StartCopyFromBlob(new Uri(url), options: brOptions); var bcd = new BlobCopyData { CopyID = copyID, Blob = blob }; return(bcd); } catch (Exception ex) { Console.WriteLine("StartCopyFromBlob error msg " + ex.Message); Console.WriteLine("StartCopyFromBlob error stack " + ex.StackTrace); throw; } } else { throw new NotImplementedException("Cannot copy blobs that are not block or page"); } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var skydriveListing = SkyDriveHelper.ListSkyDriveDirectoryContent(containerName); foreach (var skyDriveEntry in skydriveListing) { var blob = new BasicBlobContainer(); blob.Name = skyDriveEntry.Name; var resolvedOneDriveEntry = SkyDriveHelper.ListSkyDriveFileWithUrl(skyDriveEntry.Id); // keep display name same as name until determine otherwise. blob.DisplayName = blob.Name; blob.Container = containerName; blob.Url = string.Format("{0}&blobName={1}", resolvedOneDriveEntry.Source, blob.Name); // modify link so we can determine blob name purely from link. yield return blob; } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="container"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { IEnumerable<IListBlobItem> azureBlobList; CloudBlobContainer container; if (string.IsNullOrWhiteSpace(containerName)) { containerName = defaultContainerName; } if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } if (string.IsNullOrEmpty(containerName)) { container = client.GetRootContainerReference(); // add container. // Assuming no blobs at root level. // incorrect assumption. FIXME! var containerList = client.ListContainers(); foreach (var cont in containerList) { var b = new BasicBlobContainer(); b.Name = Uri.UnescapeDataString(cont.Name); b.DisplayName = AzureHelper.GetDisplayName(cont.Name); b.Container = ""; b.Url = Uri.UnescapeDataString(cont.Uri.AbsoluteUri); b.BlobType = BlobEntryType.Container; yield return b; } } else { container = client.GetContainerReference(containerName); // if we were only passed the container name, then list contents of container. if (string.IsNullOrEmpty(blobPrefix)) { // add blobs azureBlobList = container.ListBlobs(useFlatBlobListing: true); } else { var vd = container.GetDirectoryReference(blobPrefix); azureBlobList = ListVirtualDirectoryBlobs(vd); } foreach (var blob in azureBlobList) { var b = new BasicBlobContainer(); var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri); b.BlobPrefix = blobPrefix; b.Name = Uri.UnescapeDataString(bn); b.DisplayName = AzureHelper.GetDisplayName(bn); b.Container = blob.Container.Name; b.Url = Uri.UnescapeDataString(blob.Uri.AbsoluteUri); b.BlobType = BlobEntryType.Blob; yield return b; } } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var bucket = containerName; using (IAmazonS3 client = S3Helper.GenerateS3Client(ConfigHelper.SrcAWSAccessKeyID, ConfigHelper.SrcAWSSecretAccessKeyID, bucket)) { var request = new ListObjectsRequest(); request.BucketName = bucket; if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } if (!string.IsNullOrEmpty(blobPrefix)) { request.Prefix = blobPrefix; } // FIXME... check virtual directories workhere. do { ListObjectsResponse response = client.ListObjects(request); foreach (var obj in response.S3Objects) { string fullPath; if (baseUrlProvided) { //var fullPath = GenerateUrl(baseUrl, bucket, obj.Key); fullPath = baseUrl + obj.Key; } else { fullPath = string.Format("https://{0}/{1}/{2}", baseUrl, containerName, obj.Key); } if (!fullPath.EndsWith("/")) { //var fullPath = Path.Combine(baseUrl, obj.Key); var blob = new BasicBlobContainer(); blob.Name = obj.Key; blob.Url = fullPath; blob.Container = bucket; blob.BlobType = BlobEntryType.Blob; blob.DisplayName = S3Helper.GetDisplayName(blob.Name); blob.BlobPrefix = blobPrefix; yield return blob; //blobList.Add(blob); } } if (response.IsTruncated) { request.Marker = response.NextMarker; } else { request = null; } } while (request != null); } }
/// <summary> /// Lists all blobs in a container. /// Recurse directories to get all contents. /// Either do it here, or when reading blob (and it turns out to be a directory). /// Always recursing might not always be wanted, but think its a good default (for now). /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { //var metadata = client.GetMetaData(containerName, null, false, false); var metadata = client.GetMetaData(path:containerName); if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } // generate list of dirs and files. foreach (var entry in metadata.Contents) { // basic blob info. var blob = new BasicBlobContainer(); blob.Container = containerName; blob.DisplayName = entry.Name; blob.Url = "https://dropbox.com"+entry.Path; blob.BlobPrefix = blobPrefix; if (entry.Is_Dir) { blob.BlobType = BlobEntryType.Container; var newContainerName = string.Empty; if (containerName.EndsWith("/")) { newContainerName = containerName + entry.Name + "/"; } else { newContainerName = containerName + "/" + entry.Name + "/"; } var recursiveBlobList = ListBlobsInContainer(newContainerName, blobPrefix, debug); foreach( var d in recursiveBlobList) { yield return d; } } else { //var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name; var name = entry.Name; blob.Name = name; blob.BlobType = BlobEntryType.Blob; yield return blob; } } }
// have destination location. // have original blob name and prefix // new name should be destination name + (blob.name - blob.prefix) public static BlobCopyData StartCopy(BasicBlobContainer origBlob, string DestinationUrl, DestinationBlobType destBlobType) { var client = AzureHelper.GetTargetCloudBlobClient(DestinationUrl); var opt = client.GetServiceProperties(); var containerName = AzureHelper.GetContainerFromUrl( DestinationUrl); var destBlobPrefix = AzureHelper.GetBlobFromUrl( DestinationUrl ); var container = client.GetContainerReference( containerName ); container.CreateIfNotExists(); ICloudBlob blob = null; var url = GeneratedAccessibleUrl(origBlob); string blobName; string blobWithoutPrefix = string.Empty; if (!string.IsNullOrWhiteSpace(origBlob.BlobPrefix) && origBlob.Name.Length > origBlob.BlobPrefix.Length) { blobWithoutPrefix = origBlob.Name.Substring(origBlob.BlobPrefix.Length); } if (!string.IsNullOrWhiteSpace(blobWithoutPrefix)) { blobName = string.Format("{0}/{1}", destBlobPrefix, blobWithoutPrefix); } else { // need to get just filename. ie last element of / var actualBlobName = origBlob.Name.Split('/').Last(); if (string.IsNullOrWhiteSpace(destBlobPrefix)) { blobName = actualBlobName; } else { blobName = string.Format("{0}/{1}", destBlobPrefix, actualBlobName); } } // include unknown for now. Unsure. if (destBlobType == DestinationBlobType.Block || destBlobType == DestinationBlobType.Unknown) { blob = container.GetBlockBlobReference(blobName); } else if (destBlobType == DestinationBlobType.Page) { blob = container.GetPageBlobReference(blobName); } if (blob != null) { try { // crazy large values, want to try and debug an issue. var brOptions = new BlobRequestOptions(); brOptions.MaximumExecutionTime = new TimeSpan(0, maxExecutionTimeInMins, 0); brOptions.ServerTimeout = new TimeSpan(0, maxServerTimeoutInMins, 0); // return copyID incase user wants to kill the process later. var copyID = blob.StartCopyFromBlob(new Uri(url), options: brOptions); var bcd = new BlobCopyData { CopyID = copyID, Blob = blob }; return bcd; } catch(Exception ex) { Console.WriteLine("StartCopyFromBlob error msg " + ex.Message); Console.WriteLine("StartCopyFromBlob error stack " + ex.StackTrace); throw; } } else { throw new NotImplementedException("Cannot copy blobs that are not block or page"); } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="containerName"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { var files = Directory.EnumerateFiles(baseUrl); foreach( var file in files) { var f = new BasicBlobContainer(); var name = Path.GetFileName(file); f.BlobType = BlobEntryType.Blob; f.DisplayName = name; f.Url = file; f.Name = name; yield return f; } }
/// <summary> /// Lists all blobs in a container. /// Can be supplied a blobPrefix which basically acts as virtual directory options. /// eg, if we have blobs called: "virt1/virt2/myblob" and /// "virt1/virt2/myblob2" /// Although the blob names are the complete strings mentioned above, we might like to think that the blobs /// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of /// as a directory, but again, its just really a prefix behind the scenes. /// /// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate /// </summary> /// <param name="container"></param> /// <param name="blobPrefix"></param> /// <returns></returns> public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false) { IEnumerable <IListBlobItem> azureBlobList; CloudBlobContainer container; if (string.IsNullOrWhiteSpace(containerName)) { containerName = defaultContainerName; } if (string.IsNullOrWhiteSpace(blobPrefix)) { blobPrefix = defaultBlobPrefix; } if (string.IsNullOrEmpty(containerName)) { container = client.GetRootContainerReference(); // add container. // Assuming no blobs at root level. // incorrect assumption. FIXME! var containerList = client.ListContainers(); foreach (var cont in containerList) { var b = new BasicBlobContainer(); b.Name = Uri.UnescapeDataString(cont.Name); b.DisplayName = AzureHelper.GetDisplayName(cont.Name); b.Container = ""; b.Url = Uri.UnescapeDataString(cont.Uri.AbsoluteUri); b.BlobType = BlobEntryType.Container; yield return(b); } } else { container = client.GetContainerReference(containerName); // if we were only passed the container name, then list contents of container. if (string.IsNullOrEmpty(blobPrefix)) { // add blobs azureBlobList = container.ListBlobs(useFlatBlobListing: true); } else { var vd = container.GetDirectoryReference(blobPrefix); azureBlobList = ListVirtualDirectoryBlobs(vd); } foreach (var blob in azureBlobList) { var b = new BasicBlobContainer(); var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri); b.BlobPrefix = blobPrefix; b.Name = Uri.UnescapeDataString(bn); b.DisplayName = AzureHelper.GetDisplayName(bn); b.Container = blob.Container.Name; b.Url = Uri.UnescapeDataString(blob.Uri.AbsoluteUri); b.BlobType = BlobEntryType.Blob; yield return(b); } } }
/// <summary> /// List containers/directories off the root. For storage schemes that allow real directories maybe /// the root will be /// </summary> /// <returns></returns> public List<BasicBlobContainer> ListContainers(string root) { var dirListing = new List<BasicBlobContainer>(); var containerName = ""; var blobPrefix = ""; //var metadata = client.GetMetaData(containerName, null, false, false); var metadata = client.GetMetaData(); // generate list of dirs and files. foreach (var entry in metadata.Contents) { // basic blob info. var blob = new BasicBlobContainer(); blob.Container = containerName; blob.DisplayName = entry.Name; blob.Url = entry.Path; blob.BlobPrefix = blobPrefix; if (entry.Is_Dir) { blob.BlobType = BlobEntryType.Container; blob.Name = entry.Name; dirListing.Add(blob); } else { //var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name; var name = entry.Name; blob.Name = containerName + name; blob.BlobType = BlobEntryType.Blob; dirListing.Add(blob); } } return dirListing; }