/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
IEnumerable <IListBlobItem> azureBlobList;
CloudBlobContainer container;
if (string.IsNullOrEmpty(containerName))
{
container = client.GetRootContainerReference();
// add container.
// Assuming no blobs at root level.
// incorrect assumption. FIXME!
var containerList = client.ListContainers();
foreach (var cont in containerList)
{
var b = new BasicBlobContainer();
b.Name = cont.Name;
b.DisplayName = AzureHelper.GetDisplayName(cont.Name);
b.Container = "";
b.Url = cont.Uri.AbsoluteUri;
b.BlobType = BlobEntryType.Container;
yield return(b);
}
}
else
{
container = client.GetContainerReference(containerName);
// if we were only passed the container name, then list contents of container.
if (string.IsNullOrEmpty(blobPrefix))
{
// add blobs
azureBlobList = container.ListBlobs(useFlatBlobListing: true);
}
else
{
throw new NotImplementedException();
// if passed virtual directory information, then filter based off that.
//var vd = container.GetDirectoryReference(virtualDirectoryName);
//azureBlobList = vd.ListBlobs();
}
foreach (var blob in azureBlobList)
{
var b = new BasicBlobContainer();
var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri);
b.Name = bn;
var sp = bn.Split('/');
var displayName = sp[sp.Length - 1];
b.DisplayName = displayName;
b.Container = blob.Container.Name;
b.Url = blob.Uri.AbsoluteUri;
b.BlobType = BlobEntryType.Blob;
yield return(b);
}
}
}
/// <summary>
/// Lists all blobs in a container.
/// Recurse directories to get all contents.
/// Either do it here, or when reading blob (and it turns out to be a directory).
/// Always recursing might not always be wanted, but think its a good default (for now).
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
//var metadata = client.GetMetaData(containerName, null, false, false);
var metadata = client.GetMetaData(path: containerName);
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
// generate list of dirs and files.
foreach (var entry in metadata.Contents)
{
// basic blob info.
var blob = new BasicBlobContainer();
blob.Container = containerName;
blob.DisplayName = entry.Name;
blob.Url = "https://dropbox.com" + entry.Path;
blob.BlobPrefix = blobPrefix;
if (entry.Is_Dir)
{
blob.BlobType = BlobEntryType.Container;
var newContainerName = string.Empty;
if (containerName.EndsWith("/"))
{
newContainerName = containerName + entry.Name + "/";
}
else
{
newContainerName = containerName + "/" + entry.Name + "/";
}
var recursiveBlobList = ListBlobsInContainer(newContainerName, blobPrefix, debug);
foreach (var d in recursiveBlobList)
{
yield return(d);
}
}
else
{
//var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name;
var name = entry.Name;
blob.Name = name;
blob.BlobType = BlobEntryType.Blob;
yield return(blob);
}
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var files = Directory.EnumerateFiles(baseUrl);
foreach (var file in files)
{
var f = new BasicBlobContainer();
var name = Path.GetFileName(file);
f.BlobType = BlobEntryType.Blob;
f.DisplayName = name;
f.Url = file;
f.Name = name;
yield return(f);
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var skydriveListing = SkyDriveHelper.ListSkyDriveDirectoryContent(containerName);
foreach (var skyDriveEntry in skydriveListing)
{
var blob = new BasicBlobContainer();
blob.Name = skyDriveEntry.Name;
var resolvedOneDriveEntry = SkyDriveHelper.ListSkyDriveFileWithUrl(skyDriveEntry.Id);
// keep display name same as name until determine otherwise.
blob.DisplayName = blob.Name;
blob.Container = containerName;
blob.Url = string.Format("{0}&blobName={1}", resolvedOneDriveEntry.Source, blob.Name); // modify link so we can determine blob name purely from link.
yield return(blob);
}
}
/// <summary>
/// Makes a usable URL for a blob. This will need to handle security on the source blob.
/// Each cloud provider is different.
/// Cloud providers developed:
/// Azure
/// S3xx
///
/// Cloud providers soon:
/// Dropbox
/// Onedrive
/// </summary>
/// <param name="origBlob"></param>
/// <returns></returns>
private static string GeneratedAccessibleUrl(BasicBlobContainer origBlob)
{
var sourceUrl = origBlob.Url; // +"/" + origBlob.Name;
string url = "";
// if S3, then generate signed url.
if (S3Helper.MatchHandler(sourceUrl))
{
var bucket = S3Helper.GetBucketFromUrl(sourceUrl);
var key = origBlob.Name;
url = S3Helper.GeneratePreSignedUrl(bucket, key);
}
else if (AzureHelper.MatchHandler(sourceUrl))
{
// generate Azure signed url.
var client = AzureHelper.GetSourceCloudBlobClient(sourceUrl);
var policy = new SharedAccessBlobPolicy();
policy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-1);
policy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(ConfigHelper.SharedAccessSignatureDurationInSeconds / 60);
policy.Permissions = SharedAccessBlobPermissions.Read;
var blob = client.GetBlobReferenceFromServer(new Uri(sourceUrl));
url = sourceUrl + blob.GetSharedAccessSignature(policy);
}
else if (DropboxHelper.MatchHandler(sourceUrl))
{
// need shorter url. (no base url);
var uri = new Uri(sourceUrl);
var shortUrl = uri.PathAndQuery;
var client = DropboxHelper.GetClient();
var media = client.GetMedia(shortUrl);
return(media.Url);
}
else if (SkyDriveHelper.MatchHandler(sourceUrl))
{
throw new NotImplementedException("Blobcopy against onedrive is not implemented yet");
}
Console.WriteLine("shared url is {0}", url);
return(url);
}
/// <summary>
/// List containers/directories off the root. For storage schemes that allow real directories maybe
/// the root will be
/// </summary>
/// <returns></returns>
public List <BasicBlobContainer> ListContainers(string root)
{
var dirListing = new List <BasicBlobContainer>();
var containerName = "";
var blobPrefix = "";
//var metadata = client.GetMetaData(containerName, null, false, false);
var metadata = client.GetMetaData();
// generate list of dirs and files.
foreach (var entry in metadata.Contents)
{
// basic blob info.
var blob = new BasicBlobContainer();
blob.Container = containerName;
blob.DisplayName = entry.Name;
blob.Url = entry.Path;
blob.BlobPrefix = blobPrefix;
if (entry.Is_Dir)
{
blob.BlobType = BlobEntryType.Container;
blob.Name = entry.Name;
dirListing.Add(blob);
}
else
{
//var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name;
var name = entry.Name;
blob.Name = containerName + name;
blob.BlobType = BlobEntryType.Blob;
dirListing.Add(blob);
}
}
return(dirListing);
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
IEnumerable<IListBlobItem> azureBlobList;
CloudBlobContainer container;
if (string.IsNullOrEmpty(containerName))
{
container = client.GetRootContainerReference();
// add container.
// Assuming no blobs at root level.
// incorrect assumption. FIXME!
var containerList = client.ListContainers();
foreach (var cont in containerList)
{
var b = new BasicBlobContainer();
b.Name = cont.Name;
b.DisplayName = AzureHelper.GetDisplayName(cont.Name);
b.Container = "";
b.Url = cont.Uri.AbsoluteUri;
b.BlobType = BlobEntryType.Container;
yield return b;
}
}
else
{
container = client.GetContainerReference(containerName);
// if we were only passed the container name, then list contents of container.
if (string.IsNullOrEmpty(blobPrefix))
{
// add blobs
azureBlobList = container.ListBlobs(useFlatBlobListing:true);
}
else
{
throw new NotImplementedException();
// if passed virtual directory information, then filter based off that.
//var vd = container.GetDirectoryReference(virtualDirectoryName);
//azureBlobList = vd.ListBlobs();
}
foreach (var blob in azureBlobList)
{
var b = new BasicBlobContainer();
var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri);
b.Name = bn;
var sp = bn.Split('/');
var displayName = sp[ sp.Length -1];
b.DisplayName = displayName;
b.Container = blob.Container.Name;
b.Url = blob.Uri.AbsoluteUri;
b.BlobType = BlobEntryType.Blob;
yield return b;
}
}
}
/// <summary>
/// Makes a usable URL for a blob. This will need to handle security on the source blob.
/// Each cloud provider is different.
/// Cloud providers developed:
/// Azure
/// S3xx
///
/// Cloud providers soon:
/// Dropbox
/// Onedrive
/// </summary>
/// <param name="origBlob"></param>
/// <returns></returns>
private static string GeneratedAccessibleUrl( BasicBlobContainer origBlob)
{
var sourceUrl = origBlob.Url; // +"/" + origBlob.Name;
string url = "";
// if S3, then generate signed url.
if (S3Helper.MatchHandler(sourceUrl))
{
var bucket = S3Helper.GetBucketFromUrl(sourceUrl);
var key = origBlob.Name;
url = S3Helper.GeneratePreSignedUrl(bucket, key);
} else if (AzureHelper.MatchHandler( sourceUrl))
{
// generate Azure signed url.
var client = AzureHelper.GetSourceCloudBlobClient(sourceUrl);
var policy = new SharedAccessBlobPolicy();
policy.SharedAccessStartTime = DateTime.UtcNow.AddMinutes(-1);
policy.SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes( ConfigHelper.SharedAccessSignatureDurationInSeconds / 60 );
policy.Permissions = SharedAccessBlobPermissions.Read;
var blob = client.GetBlobReferenceFromServer( new Uri(sourceUrl));
url = sourceUrl+ blob.GetSharedAccessSignature(policy);
} else if (DropboxHelper.MatchHandler(sourceUrl))
{
// need shorter url. (no base url);
var uri = new Uri(sourceUrl);
var shortUrl = uri.PathAndQuery;
var client = DropboxHelper.GetClient();
var media = client.GetMedia(shortUrl);
return media.Url;
} else if (SkyDriveHelper.MatchHandler( sourceUrl))
{
throw new NotImplementedException("Blobcopy against onedrive is not implemented yet");
}
return url;
}
/// <summary>
/// Get all possible blobs for the handler (which had the original url passed into the constructor).
/// </summary>
/// <param name="inputHandler"></param>
/// <param name="url"></param>
/// <returns></returns>
private static IEnumerable<BasicBlobContainer> GetSourceBlobList(IBlobHandler inputHandler)
{
var containerName = inputHandler.GetContainerNameFromUrl(inputHandler.GetBaseUrl());
if (CommonHelper.IsABlob(inputHandler.GetBaseUrl()))
{
var blobName = inputHandler.GetBlobNameFromUrl( inputHandler.GetBaseUrl());
var blob = new BasicBlobContainer{
Name = blobName,
DisplayName = blobName,
BlobType = BlobEntryType.Blob,
Url = inputHandler.GetBaseUrl(),
Container = containerName
};
yield return blob;
//blobList.Add(blob);
}
else
{
var res = inputHandler.ListBlobsInContainer(containerName);
foreach( var i in res)
{
yield return i;
}
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var bucket = containerName;
using (IAmazonS3 client = S3Helper.GenerateS3Client(ConfigHelper.SrcAWSAccessKeyID, ConfigHelper.SrcAWSSecretAccessKeyID, bucket))
{
var request = new ListObjectsRequest();
request.BucketName = bucket;
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
if (!string.IsNullOrEmpty(blobPrefix))
{
request.Prefix = blobPrefix;
}
// FIXME... check virtual directories workhere.
do
{
ListObjectsResponse response = client.ListObjects(request);
foreach (var obj in response.S3Objects)
{
string fullPath;
if (baseUrlProvided)
{
//var fullPath = GenerateUrl(baseUrl, bucket, obj.Key);
fullPath = baseUrl + obj.Key;
}
else
{
fullPath = string.Format("https://{0}/{1}/{2}", baseUrl, containerName, obj.Key);
}
if (!fullPath.EndsWith("/"))
{
//var fullPath = Path.Combine(baseUrl, obj.Key);
var blob = new BasicBlobContainer();
blob.Name = obj.Key;
blob.Url = fullPath;
blob.Container = bucket;
blob.BlobType = BlobEntryType.Blob;
blob.DisplayName = S3Helper.GetDisplayName(blob.Name);
blob.BlobPrefix = blobPrefix;
yield return(blob);
//blobList.Add(blob);
}
}
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
}
}
// have destination location.
// have original blob name and prefix
// new name should be destination name + (blob.name - blob.prefix)
public static BlobCopyData StartCopy(BasicBlobContainer origBlob, string DestinationUrl, DestinationBlobType destBlobType, bool skipIfExists)
{
var client = AzureHelper.GetTargetCloudBlobClient(DestinationUrl);
var opt = client.GetServiceProperties();
var containerName = AzureHelper.GetContainerFromUrl(DestinationUrl);
var destBlobPrefix = AzureHelper.GetBlobFromUrl(DestinationUrl);
var container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
ICloudBlob blob = null;
var url = GeneratedAccessibleUrl(origBlob);
string blobName;
string blobWithoutPrefix = string.Empty;
if (!string.IsNullOrWhiteSpace(origBlob.BlobPrefix) && origBlob.Name.Length > origBlob.BlobPrefix.Length)
{
blobWithoutPrefix = origBlob.Name.Substring(origBlob.BlobPrefix.Length);
}
if (!string.IsNullOrWhiteSpace(blobWithoutPrefix))
{
blobName = string.Format("{0}/{1}", destBlobPrefix, blobWithoutPrefix);
}
else
{
// need to get just filename. ie last element of /
var actualBlobName = origBlob.Name.Split('/').Last();
if (string.IsNullOrWhiteSpace(destBlobPrefix))
{
blobName = actualBlobName;
}
else
{
blobName = string.Format("{0}/{1}", destBlobPrefix, actualBlobName);
}
}
// include unknown for now. Unsure.
if (destBlobType == DestinationBlobType.Block || destBlobType == DestinationBlobType.Unknown)
{
blob = container.GetBlockBlobReference(blobName);
}
else if (destBlobType == DestinationBlobType.Page)
{
blob = container.GetPageBlobReference(blobName);
}
if (skipIfExists)
{
try
{
blob.Exists();
}
catch (Exception)
{
Console.WriteLine("Skipping {0}", url);
return(null);
}
}
if (blob != null)
{
try
{
// crazy large values, want to try and debug an issue.
var brOptions = new BlobRequestOptions();
brOptions.MaximumExecutionTime = new TimeSpan(0, maxExecutionTimeInMins, 0);
brOptions.ServerTimeout = new TimeSpan(0, maxServerTimeoutInMins, 0);
// return copyID incase user wants to kill the process later.
var copyID = blob.StartCopyFromBlob(new Uri(url), options: brOptions);
var bcd = new BlobCopyData {
CopyID = copyID, Blob = blob
};
return(bcd);
}
catch (Exception ex)
{
Console.WriteLine("StartCopyFromBlob error msg " + ex.Message);
Console.WriteLine("StartCopyFromBlob error stack " + ex.StackTrace);
throw;
}
}
else
{
throw new NotImplementedException("Cannot copy blobs that are not block or page");
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var skydriveListing = SkyDriveHelper.ListSkyDriveDirectoryContent(containerName);
foreach (var skyDriveEntry in skydriveListing)
{
var blob = new BasicBlobContainer();
blob.Name = skyDriveEntry.Name;
var resolvedOneDriveEntry = SkyDriveHelper.ListSkyDriveFileWithUrl(skyDriveEntry.Id);
// keep display name same as name until determine otherwise.
blob.DisplayName = blob.Name;
blob.Container = containerName;
blob.Url = string.Format("{0}&blobName={1}", resolvedOneDriveEntry.Source, blob.Name); // modify link so we can determine blob name purely from link.
yield return blob;
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="container"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
IEnumerable<IListBlobItem> azureBlobList;
CloudBlobContainer container;
if (string.IsNullOrWhiteSpace(containerName))
{
containerName = defaultContainerName;
}
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
if (string.IsNullOrEmpty(containerName))
{
container = client.GetRootContainerReference();
// add container.
// Assuming no blobs at root level.
// incorrect assumption. FIXME!
var containerList = client.ListContainers();
foreach (var cont in containerList)
{
var b = new BasicBlobContainer();
b.Name = Uri.UnescapeDataString(cont.Name);
b.DisplayName = AzureHelper.GetDisplayName(cont.Name);
b.Container = "";
b.Url = Uri.UnescapeDataString(cont.Uri.AbsoluteUri);
b.BlobType = BlobEntryType.Container;
yield return b;
}
}
else
{
container = client.GetContainerReference(containerName);
// if we were only passed the container name, then list contents of container.
if (string.IsNullOrEmpty(blobPrefix))
{
// add blobs
azureBlobList = container.ListBlobs(useFlatBlobListing: true);
}
else
{
var vd = container.GetDirectoryReference(blobPrefix);
azureBlobList = ListVirtualDirectoryBlobs(vd);
}
foreach (var blob in azureBlobList)
{
var b = new BasicBlobContainer();
var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri);
b.BlobPrefix = blobPrefix;
b.Name = Uri.UnescapeDataString(bn);
b.DisplayName = AzureHelper.GetDisplayName(bn);
b.Container = blob.Container.Name;
b.Url = Uri.UnescapeDataString(blob.Uri.AbsoluteUri);
b.BlobType = BlobEntryType.Blob;
yield return b;
}
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var bucket = containerName;
using (IAmazonS3 client = S3Helper.GenerateS3Client(ConfigHelper.SrcAWSAccessKeyID, ConfigHelper.SrcAWSSecretAccessKeyID, bucket))
{
var request = new ListObjectsRequest();
request.BucketName = bucket;
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
if (!string.IsNullOrEmpty(blobPrefix))
{
request.Prefix = blobPrefix;
}
// FIXME... check virtual directories workhere.
do
{
ListObjectsResponse response = client.ListObjects(request);
foreach (var obj in response.S3Objects)
{
string fullPath;
if (baseUrlProvided)
{
//var fullPath = GenerateUrl(baseUrl, bucket, obj.Key);
fullPath = baseUrl + obj.Key;
}
else
{
fullPath = string.Format("https://{0}/{1}/{2}", baseUrl, containerName, obj.Key);
}
if (!fullPath.EndsWith("/"))
{
//var fullPath = Path.Combine(baseUrl, obj.Key);
var blob = new BasicBlobContainer();
blob.Name = obj.Key;
blob.Url = fullPath;
blob.Container = bucket;
blob.BlobType = BlobEntryType.Blob;
blob.DisplayName = S3Helper.GetDisplayName(blob.Name);
blob.BlobPrefix = blobPrefix;
yield return blob;
//blobList.Add(blob);
}
}
if (response.IsTruncated)
{
request.Marker = response.NextMarker;
}
else
{
request = null;
}
} while (request != null);
}
}
/// <summary>
/// Lists all blobs in a container.
/// Recurse directories to get all contents.
/// Either do it here, or when reading blob (and it turns out to be a directory).
/// Always recursing might not always be wanted, but think its a good default (for now).
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
//var metadata = client.GetMetaData(containerName, null, false, false);
var metadata = client.GetMetaData(path:containerName);
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
// generate list of dirs and files.
foreach (var entry in metadata.Contents)
{
// basic blob info.
var blob = new BasicBlobContainer();
blob.Container = containerName;
blob.DisplayName = entry.Name;
blob.Url = "https://dropbox.com"+entry.Path;
blob.BlobPrefix = blobPrefix;
if (entry.Is_Dir)
{
blob.BlobType = BlobEntryType.Container;
var newContainerName = string.Empty;
if (containerName.EndsWith("/"))
{
newContainerName = containerName + entry.Name + "/";
}
else
{
newContainerName = containerName + "/" + entry.Name + "/";
}
var recursiveBlobList = ListBlobsInContainer(newContainerName, blobPrefix, debug);
foreach( var d in recursiveBlobList)
{
yield return d;
}
}
else
{
//var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name;
var name = entry.Name;
blob.Name = name;
blob.BlobType = BlobEntryType.Blob;
yield return blob;
}
}
}
// have destination location.
// have original blob name and prefix
// new name should be destination name + (blob.name - blob.prefix)
public static BlobCopyData StartCopy(BasicBlobContainer origBlob, string DestinationUrl, DestinationBlobType destBlobType)
{
var client = AzureHelper.GetTargetCloudBlobClient(DestinationUrl);
var opt = client.GetServiceProperties();
var containerName = AzureHelper.GetContainerFromUrl( DestinationUrl);
var destBlobPrefix = AzureHelper.GetBlobFromUrl( DestinationUrl );
var container = client.GetContainerReference( containerName );
container.CreateIfNotExists();
ICloudBlob blob = null;
var url = GeneratedAccessibleUrl(origBlob);
string blobName;
string blobWithoutPrefix = string.Empty;
if (!string.IsNullOrWhiteSpace(origBlob.BlobPrefix) && origBlob.Name.Length > origBlob.BlobPrefix.Length)
{
blobWithoutPrefix = origBlob.Name.Substring(origBlob.BlobPrefix.Length);
}
if (!string.IsNullOrWhiteSpace(blobWithoutPrefix))
{
blobName = string.Format("{0}/{1}", destBlobPrefix, blobWithoutPrefix);
}
else
{
// need to get just filename. ie last element of /
var actualBlobName = origBlob.Name.Split('/').Last();
if (string.IsNullOrWhiteSpace(destBlobPrefix))
{
blobName = actualBlobName;
}
else
{
blobName = string.Format("{0}/{1}", destBlobPrefix, actualBlobName);
}
}
// include unknown for now. Unsure.
if (destBlobType == DestinationBlobType.Block || destBlobType == DestinationBlobType.Unknown)
{
blob = container.GetBlockBlobReference(blobName);
} else if (destBlobType == DestinationBlobType.Page)
{
blob = container.GetPageBlobReference(blobName);
}
if (blob != null)
{
try
{
// crazy large values, want to try and debug an issue.
var brOptions = new BlobRequestOptions();
brOptions.MaximumExecutionTime = new TimeSpan(0, maxExecutionTimeInMins, 0);
brOptions.ServerTimeout = new TimeSpan(0, maxServerTimeoutInMins, 0);
// return copyID incase user wants to kill the process later.
var copyID = blob.StartCopyFromBlob(new Uri(url), options: brOptions);
var bcd = new BlobCopyData { CopyID = copyID, Blob = blob };
return bcd;
}
catch(Exception ex)
{
Console.WriteLine("StartCopyFromBlob error msg " + ex.Message);
Console.WriteLine("StartCopyFromBlob error stack " + ex.StackTrace);
throw;
}
}
else
{
throw new NotImplementedException("Cannot copy blobs that are not block or page");
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="containerName"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable<BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
var files = Directory.EnumerateFiles(baseUrl);
foreach( var file in files)
{
var f = new BasicBlobContainer();
var name = Path.GetFileName(file);
f.BlobType = BlobEntryType.Blob;
f.DisplayName = name;
f.Url = file;
f.Name = name;
yield return f;
}
}
/// <summary>
/// Lists all blobs in a container.
/// Can be supplied a blobPrefix which basically acts as virtual directory options.
/// eg, if we have blobs called: "virt1/virt2/myblob" and
/// "virt1/virt2/myblob2"
/// Although the blob names are the complete strings mentioned above, we might like to think that the blobs
/// are just called myblob and myblob2. We can supply a blobPrefix of "virt1/virt2/" which we can *think* of
/// as a directory, but again, its just really a prefix behind the scenes.
///
/// For other sytems (not Azure) the blobPrefix might be real directories.... will need to investigate
/// </summary>
/// <param name="container"></param>
/// <param name="blobPrefix"></param>
/// <returns></returns>
public IEnumerable <BasicBlobContainer> ListBlobsInContainer(string containerName = null, string blobPrefix = null, bool debug = false)
{
IEnumerable <IListBlobItem> azureBlobList;
CloudBlobContainer container;
if (string.IsNullOrWhiteSpace(containerName))
{
containerName = defaultContainerName;
}
if (string.IsNullOrWhiteSpace(blobPrefix))
{
blobPrefix = defaultBlobPrefix;
}
if (string.IsNullOrEmpty(containerName))
{
container = client.GetRootContainerReference();
// add container.
// Assuming no blobs at root level.
// incorrect assumption. FIXME!
var containerList = client.ListContainers();
foreach (var cont in containerList)
{
var b = new BasicBlobContainer();
b.Name = Uri.UnescapeDataString(cont.Name);
b.DisplayName = AzureHelper.GetDisplayName(cont.Name);
b.Container = "";
b.Url = Uri.UnescapeDataString(cont.Uri.AbsoluteUri);
b.BlobType = BlobEntryType.Container;
yield return(b);
}
}
else
{
container = client.GetContainerReference(containerName);
// if we were only passed the container name, then list contents of container.
if (string.IsNullOrEmpty(blobPrefix))
{
// add blobs
azureBlobList = container.ListBlobs(useFlatBlobListing: true);
}
else
{
var vd = container.GetDirectoryReference(blobPrefix);
azureBlobList = ListVirtualDirectoryBlobs(vd);
}
foreach (var blob in azureBlobList)
{
var b = new BasicBlobContainer();
var bn = AzureHelper.GetBlobFromUrl(blob.Uri.AbsoluteUri);
b.BlobPrefix = blobPrefix;
b.Name = Uri.UnescapeDataString(bn);
b.DisplayName = AzureHelper.GetDisplayName(bn);
b.Container = blob.Container.Name;
b.Url = Uri.UnescapeDataString(blob.Uri.AbsoluteUri);
b.BlobType = BlobEntryType.Blob;
yield return(b);
}
}
}
/// <summary>
/// List containers/directories off the root. For storage schemes that allow real directories maybe
/// the root will be
/// </summary>
/// <returns></returns>
public List<BasicBlobContainer> ListContainers(string root)
{
var dirListing = new List<BasicBlobContainer>();
var containerName = "";
var blobPrefix = "";
//var metadata = client.GetMetaData(containerName, null, false, false);
var metadata = client.GetMetaData();
// generate list of dirs and files.
foreach (var entry in metadata.Contents)
{
// basic blob info.
var blob = new BasicBlobContainer();
blob.Container = containerName;
blob.DisplayName = entry.Name;
blob.Url = entry.Path;
blob.BlobPrefix = blobPrefix;
if (entry.Is_Dir)
{
blob.BlobType = BlobEntryType.Container;
blob.Name = entry.Name;
dirListing.Add(blob);
}
else
{
//var name = entry.Name.StartsWith("/") ? entry.Name : "/" + entry.Name;
var name = entry.Name;
blob.Name = containerName + name;
blob.BlobType = BlobEntryType.Blob;
dirListing.Add(blob);
}
}
return dirListing;
}
