public async Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken = default)
{
FtpClient client = await GetClientAsync();
var results = new List <BlobMeta>();
foreach (string path in ids)
{
string cpath = StoragePath.Normalize(path, true);
string parentPath = StoragePath.GetParent(cpath);
FtpListItem[] all = await client.GetListingAsync(parentPath);
FtpListItem foundItem = all.FirstOrDefault(i => i.FullName == cpath);
if (foundItem == null)
{
results.Add(null);
continue;
}
var meta = new BlobMeta(foundItem.Size, null, foundItem.Modified);
results.Add(meta);
}
return(results);
}
private BlobMeta BlobMetaFromId(string id)
{
string path = GetFilePath(StoragePath.Normalize(id, false));
if (!File.Exists(path))
{
return(null);
}
var fi = new FileInfo(path);
string md5;
using (Stream fs = File.OpenRead(fi.FullName))
{
md5 = fs.GetHash(HashType.Md5);
}
var meta = new BlobMeta(
fi.Length,
md5,
fi.CreationTimeUtc);
return(meta);
}
public async Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
GenericValidation.CheckBlobId(ids);
var result = new List <BlobMeta>();
using (ServiceFabricTransaction tx = GetTransaction())
{
IReliableDictionary <string, byte[]> coll = await OpenCollectionAsync();
foreach (string id in ids)
{
ConditionalValue <byte[]> value = await coll.TryGetValueAsync(tx.Tx, ToId(id));
if (!value.HasValue)
{
result.Add(null);
}
else
{
var meta = new BlobMeta(value.Value.Length, null);
result.Add(meta);
}
}
}
return(result);
}
public async Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
var result = new List <BlobMeta>();
foreach (string id in ids)
{
GenericValidation.CheckBlobId(id);
}
foreach (string id in ids)
{
CloudBlob blob = _blobContainer.GetBlobReference(StoragePath.Normalize(id, false));
if (!(await blob.ExistsAsync()))
{
return(null);
}
await blob.FetchAttributesAsync();
//ContentMD5 is base64-encoded hash, whereas we work with HEX encoded ones
string md5 = blob.Properties.ContentMD5.Base64DecodeAsBytes().ToHexString();
var meta = new BlobMeta(
blob.Properties.Length,
md5);
result.Add(meta);
}
return(result);
}
public static async Task <IList <BlobMeta> > GetFilesListAsync(ContainerType containerType, string prefix)
{
var container = GetContainer(containerType);
var allBlobsList = new List <BlobMeta>();
BlobContinuationToken token = null;
do
{
var result = await container.ListBlobsSegmentedAsync(prefix, token);
if (result.Results.Count() > 0)
{
var blobs = result.Results.Cast <CloudBlockBlob>().OrderByDescending(m => m.Properties.LastModified).ToList();
foreach (var blob in blobs)
{
//blob.FetchAttributesAsync
BlobMeta meta = new BlobMeta();
meta.Name = blob.Name;
meta.Type = blob.Properties.ContentType;
meta.LastModified = blob.Properties.LastModified ?? DateTimeOffset.MinValue;
allBlobsList.Add(meta);
}
//allBlobsList.AddRange(blobs);
}
token = result.ContinuationToken;
} while (token != null);
return(allBlobsList);
}
private async Task <BlobMeta> GetMetaAsync(string id, DataLakeStoreFileSystemManagementClient client)
{
FileStatusResult fsr = await client.FileSystem.GetFileStatusAsync(_accountName, id);
var meta = new BlobMeta(fsr.FileStatus.Length.Value, null);
return(meta);
}
public async Task GetMeta_doesnt_exist_returns_null()
{
string id = RandomBlobId();
BlobMeta meta = (await _storage.GetMetaAsync(new[] { id })).First();
Assert.Null(meta);
}
internal static BlobMeta GetblobMeta(CloudBlob blob)
{
//ContentMD5 is base64-encoded hash, whereas we work with HEX encoded ones
string md5 = blob.Properties.ContentMD5.Base64DecodeAsBytes().ToHexString();
var meta = new BlobMeta(
blob.Properties.Length,
md5,
blob.Properties.LastModified);
return(meta);
}
public async Task GetMeta_for_one_file_succeeds()
{
string content = RandomGenerator.GetRandomString(1000, false);
string id = RandomGenerator.RandomString;
await _bs.WriteTextAsync(id, content);
BlobMeta meta = (await _provider.GetMetaAsync(new[] { id })).First();
long size = Encoding.UTF8.GetBytes(content).Length;
string md5 = content.GetHash(HashType.Md5);
Assert.Equal(size, meta.Size);
Assert.True(meta.MD5 == null || meta.MD5 == md5);
}
private async Task <BlobMeta> GetMetaAsync(string id, DataLakeStoreFileSystemManagementClient client)
{
FileStatusResult fsr;
try
{
fsr = await client.FileSystem.GetFileStatusAsync(_accountName, id);
}
catch (AdlsErrorException ex) when(ex.Response.StatusCode == HttpStatusCode.NotFound)
{
return(null);
}
var meta = new BlobMeta(fsr.FileStatus.Length.Value, null);
return(meta);
}
internal static BlobMeta GetblobMeta(CloudBlob blob)
{
//ContentMD5 is base64-encoded hash, whereas we work with HEX encoded ones
string md5 = blob.Properties.ContentMD5.Base64DecodeAsBytes().ToHexString();
var meta = new BlobMeta(
blob.Properties.Length,
md5,
blob.Properties.LastModified);
/*meta.Properties["BlobType"] = blob.BlobType.ToString();
* meta.Properties["IsDeleted"] = blob.IsDeleted;
* meta.Properties["IsSnapshot"] = blob.IsSnapshot;
* meta.Properties["ContentDisposition"] = blob.Properties.ContentDisposition;
* meta.Properties["ContentEncoding"] = blob.Properties.ContentEncoding;
* meta.Properties["ContentLanguage"] = blob.Properties.ContentLanguage;
* meta.Properties["ContentType"] = blob.Properties.ContentType;*/
return(meta);
}
private async Task <BlobMeta> GetMetaAsync(string id, CancellationToken cancellationToken)
{
CloudBlob blob = _blobContainer.GetBlobReference(StoragePath.Normalize(id, false));
if (!(await blob.ExistsAsync()))
{
return(null);
}
await blob.FetchAttributesAsync();
//ContentMD5 is base64-encoded hash, whereas we work with HEX encoded ones
string md5 = blob.Properties.ContentMD5.Base64DecodeAsBytes().ToHexString();
var meta = new BlobMeta(
blob.Properties.Length,
md5);
return(meta);
}
private static BlobId ToBlobId(string path, DirectoryEntry entry, bool includeMeta)
{
BlobMeta meta = includeMeta ? new BlobMeta(entry.Length, null, entry.LastModifiedTime) : null;
if (entry.Type == DirectoryEntryType.FILE)
{
return new BlobId(path, entry.Name, BlobItemKind.File)
{
Meta = meta
}
}
;
else
{
return new BlobId(path, entry.Name, BlobItemKind.Folder)
{
Meta = meta
}
};
}
}
/// <summary>
/// Gets file metadata
/// </summary>
public Task <IEnumerable <BlobMeta> > GetMetaAsync(IEnumerable <string> ids, CancellationToken cancellationToken)
{
if (ids == null)
{
return(null);
}
GenericValidation.CheckBlobId(ids);
var result = new List <BlobMeta>();
foreach (string id in ids)
{
string path = GetFilePath(StoragePath.Normalize(id, false));
if (!File.Exists(path))
{
result.Add(null);
}
else
{
var fi = new FileInfo(path);
string md5;
using (Stream fs = File.OpenRead(fi.FullName))
{
md5 = fs.GetHash(HashType.Md5);
}
var meta = new BlobMeta(
fi.Length,
md5);
result.Add(meta);
}
}
return(Task.FromResult((IEnumerable <BlobMeta>)result));
}
public async Task GetMeta_for_one_file_succeeds()
{
string content = RandomGenerator.GetRandomString(1000, false);
string id = RandomBlobId();
await _storage.WriteTextAsync(id, content);
BlobMeta meta = await _storage.GetMetaAsync(id);
long size = Encoding.UTF8.GetBytes(content).Length;
string md5 = content.GetHash(HashType.Md5);
Assert.Equal(size, meta.Size);
if (meta.MD5 != null)
{
Assert.Equal(md5, meta.MD5);
}
if (meta.LastModificationTime != null)
{
Assert.Equal(DateTime.UtcNow.RoundToDay(), meta.LastModificationTime.Value.DateTime.RoundToDay());
}
}
