private async Task <BlobMetadata> AzureGetMetadata(string key)
{
try
{
CloudBlobContainer container = _AzureBlobClient.GetContainerReference(_AzureSettings.Container);
CloudBlockBlob blockBlob = container.GetBlockBlobReference(key);
await blockBlob.FetchAttributesAsync();
BlobMetadata md = new BlobMetadata();
md.Key = key;
md.ContentLength = blockBlob.Properties.Length;
md.ContentType = blockBlob.Properties.ContentType;
md.ETag = blockBlob.Properties.ETag;
md.Created = blockBlob.Properties.Created.Value.UtcDateTime;
if (!String.IsNullOrEmpty(md.ETag))
{
while (md.ETag.Contains("\""))
{
md.ETag = md.ETag.Replace("\"", "");
}
}
return(md);
}
catch (Exception)
{
throw new IOException("Unable to read object.");
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
private async Task <EnumerationResult> DiskEnumerate(string prefix, string continuationToken)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
int startIndex = 0;
int count = 1000;
if (!String.IsNullOrEmpty(continuationToken))
{
if (!DiskParseContinuationToken(continuationToken, out startIndex, out count))
{
throw new ArgumentException("Unable to parse continuation token.");
}
}
long maxIndex = startIndex + count;
long currCount = 0;
IEnumerable <string> files = null;
if (!String.IsNullOrEmpty(prefix))
{
files = Directory.EnumerateDirectories(_DiskSettings.Directory, prefix + "*", SearchOption.TopDirectoryOnly);
}
else
{
files = Directory.EnumerateFiles(_DiskSettings.Directory, "*", SearchOption.TopDirectoryOnly);
}
files = files.Skip(startIndex).Take(count);
EnumerationResult ret = new EnumerationResult();
if (files.Count() < 1)
{
return(ret);
}
ret.NextContinuationToken = DiskBuildContinuationToken(startIndex + count, count);
foreach (string file in files)
{
string key = Path.GetFileName(file);
FileInfo fi = new FileInfo(file);
BlobMetadata md = new BlobMetadata();
md.Key = key;
md.ContentLength = fi.Length;
md.Created = fi.CreationTimeUtc;
ret.Blobs.Add(md);
currCount++;
continue;
}
return(ret);
}
private async Task <EnumerationResult> KvpbaseEnumerate(string prefix, string continuationToken)
{
int startIndex = 0;
int count = 1000;
if (!String.IsNullOrEmpty(continuationToken))
{
if (!KvpbaseParseContinuationToken(continuationToken, out startIndex, out count))
{
throw new ArgumentException("Unable to parse continuation token.");
}
}
ContainerMetadata cmd = null;
EnumerationResult ret = new EnumerationResult();
try
{
if (String.IsNullOrEmpty(prefix))
{
cmd = await _Kvpbase.EnumerateContainer(_KvpbaseSettings.Container, startIndex, count);
}
else
{
EnumerationFilter filter = new EnumerationFilter();
filter.Prefix = prefix;
cmd = await _Kvpbase.EnumerateContainer(filter, _KvpbaseSettings.Container, startIndex, count);
}
}
catch (Exception)
{
throw new IOException("Unable to enumerate objects.");
}
ret.NextContinuationToken = KvpbaseBuildContinuationToken(startIndex + count, count);
if (cmd.Objects != null && cmd.Objects.Count > 0)
{
foreach (ObjectMetadata curr in cmd.Objects)
{
BlobMetadata md = new BlobMetadata();
md.Key = curr.ObjectKey;
md.ETag = curr.Md5;
md.ContentLength = Convert.ToInt64(curr.ContentLength);
md.ContentType = curr.ContentType;
md.Created = curr.CreatedUtc.Value;
ret.Blobs.Add(md);
}
}
return(ret);
}
private async Task <EnumerationResult> S3Enumerate(string prefix, string continuationToken)
{
ListObjectsRequest req = new ListObjectsRequest();
req.BucketName = _AwsSettings.Bucket;
if (!String.IsNullOrEmpty(prefix))
{
req.Prefix = prefix;
}
if (!String.IsNullOrEmpty(continuationToken))
{
req.Marker = continuationToken;
}
ListObjectsResponse resp = await _S3Client.ListObjectsAsync(req);
EnumerationResult ret = new EnumerationResult();
if (resp.S3Objects != null && resp.S3Objects.Count > 0)
{
foreach (S3Object curr in resp.S3Objects)
{
BlobMetadata md = new BlobMetadata();
md.Key = curr.Key;
md.ContentLength = curr.Size;
md.ETag = curr.ETag;
md.Created = curr.LastModified;
if (!String.IsNullOrEmpty(md.ETag))
{
while (md.ETag.Contains("\""))
{
md.ETag = md.ETag.Replace("\"", "");
}
}
ret.Blobs.Add(md);
}
}
if (!String.IsNullOrEmpty(resp.NextMarker))
{
ret.NextContinuationToken = resp.NextMarker;
}
return(ret);
}
private async Task <BlobMetadata> KvpbaseGetMetadata(string key)
{
try
{
ObjectMetadata objMd = await _Kvpbase.ReadObjectMetadata(_KvpbaseSettings.Container, key);
BlobMetadata md = new BlobMetadata();
md.Key = objMd.ObjectKey;
md.ContentLength = Convert.ToInt64(objMd.ContentLength);
md.ContentType = objMd.ContentType;
md.ETag = objMd.Md5;
md.Created = objMd.CreatedUtc.Value;
return(md);
}
catch (Exception)
{
throw new IOException("Unable to read object.");
}
}
#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously
private async Task <BlobMetadata> DiskGetMetadata(string key)
#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously
{
try
{
string url = DiskGenerateUrl(key);
FileInfo fi = new FileInfo(url);
BlobMetadata md = new BlobMetadata();
md.Key = key;
md.ContentLength = fi.Length;
md.Created = fi.CreationTimeUtc;
return(md);
}
catch (Exception)
{
throw new IOException("Unable to read object.");
}
}
private async Task <BlobMetadata> S3GetMetadata(string key)
{
try
{
GetObjectMetadataRequest request = new GetObjectMetadataRequest();
request.BucketName = _AwsSettings.Bucket;
request.Key = key;
GetObjectMetadataResponse response = await _S3Client.GetObjectMetadataAsync(request);
if (response.ContentLength > 0)
{
BlobMetadata md = new BlobMetadata();
md.Key = key;
md.ContentLength = response.ContentLength;
md.ContentType = response.Headers.ContentType;
md.ETag = response.ETag;
md.Created = response.LastModified;
if (!String.IsNullOrEmpty(md.ETag))
{
while (md.ETag.Contains("\""))
{
md.ETag = md.ETag.Replace("\"", "");
}
}
return(md);
}
else
{
throw new IOException("Unable to read object.");
}
}
catch (Exception)
{
throw new IOException("Unable to read object.");
}
}
private async Task <EnumerationResult> AzureEnumerate(string prefix, string continuationToken)
{
BlobContinuationToken bct = null;
if (!String.IsNullOrEmpty(continuationToken))
{
if (!AzureGetContinuationToken(continuationToken, out bct))
{
throw new IOException("Unable to find continuation token.");
}
}
BlobResultSegment segment = null;
EnumerationResult ret = new EnumerationResult();
if (!String.IsNullOrEmpty(prefix))
{
segment = await _AzureContainer.ListBlobsSegmentedAsync(prefix, bct);
}
else
{
segment = await _AzureContainer.ListBlobsSegmentedAsync(bct);
}
if (segment == null || segment.Results == null || segment.Results.Count() < 1)
{
return(ret);
}
foreach (IListBlobItem item in segment.Results)
{
if (item.GetType() == typeof(CloudBlockBlob))
{
CloudBlockBlob blob = (CloudBlockBlob)item;
BlobMetadata md = new BlobMetadata();
md.Key = blob.Name;
md.ETag = blob.Properties.ETag;
md.ContentType = blob.Properties.ContentType;
md.ContentLength = blob.Properties.Length;
md.Created = blob.Properties.Created.Value.DateTime;
if (!String.IsNullOrEmpty(md.ETag))
{
while (md.ETag.Contains("\""))
{
md.ETag = md.ETag.Replace("\"", "");
}
}
ret.Blobs.Add(md);
}
}
if (segment.ContinuationToken != null)
{
ret.NextContinuationToken = Guid.NewGuid().ToString();
AzureStoreContinuationToken(ret.NextContinuationToken, segment.ContinuationToken);
}
if (!String.IsNullOrEmpty(continuationToken))
{
AzureRemoveContinuationToken(continuationToken);
}
return(ret);
}
