private IList<FileProperties> ListObjectImpl(string localKeyPrefix, CancellationToken ct)
{
var result = new List<FileProperties>();
localKeyPrefix = SanitizeKey(localKeyPrefix ?? "");
if (Directory.Exists(_rootPath))
{
foreach (string path in Directory.EnumerateFiles(_rootPath, "*", SearchOption.AllDirectories))
{
if (ct.IsCancellationRequested)
break;
var info = new FileInfo(path);
var prop = new FileProperties
{
LocalKey = path,
Size = info.Length,
Key = ResolveKey(path),
Timestamp = ResolveTimestamp(path)
};
if (prop.Key.StartsWith(localKeyPrefix))
{
result.Add(prop);
}
}
}
return result.OrderBy(x => x.Key).ToList();
}
public override async Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct)
{
string path = ResolvePath(properties.Key);
using (FileStream file = File.OpenRead(path))
{
await file.CopyToAsync(stream);
}
}
public override Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct)
{
const double mb = 1024*1024;
string key = ResolveLocalKey(properties.Key);
if (MaxParallelStreams == 1)
return ReadObject(stream, ct, key);
if (properties.Size > 1*mb)
return ReadObjectParallel(properties, stream, ct, key, 1*(int) mb);
return ReadObject(stream, ct, key);
}
private async Task ReadObjectParallel(FileProperties properties, Stream stream, CancellationToken ct, string key,
int partSize)
{
var requests = new List<GetObjectRequest>();
long partCount = properties.Size/partSize;
if (properties.Size%partSize > 0)
partCount++;
for (int i = 0; i < partCount; i++)
{
long startIncl = i*partSize;
long endExcl = Math.Min(startIncl + partSize, properties.Size);
requests.Add(new GetObjectRequest
{
BucketName = _bucket,
Key = key,
ByteRangeLong = new Amazon.S3.Model.Tuple<long, long>(startIncl, endExcl - 1)
});
}
var fileLock = new CloudSync.AsyncLock();
await CloudSync.ForEachPooled(requests, MaxParallelStreams, ct, async (r, t) =>
{
using (var buffer = new MemoryStream())
using (GetObjectResponse response = await _s3.GetObjectAsync(r).ConfigureAwait(false))
{
await response.ResponseStream.CopyToAsync(buffer).ConfigureAwait(false);
await buffer.FlushAsync().ConfigureAwait(false);
// truncate buffer to be safe
buffer.Position = 0;
buffer.SetLength(r.ByteRangeLong.Second - r.ByteRangeLong.First + 1);
using (await fileLock.LockAsync().ConfigureAwait(false))
{
stream.Position = r.ByteRangeLong.First;
buffer.CopyToAsync(stream, 64*1024, ct).Wait();
stream.FlushAsync().Wait();
}
}
}).ConfigureAwait(false);
}
public override async Task AddObject(FileProperties properties, Stream stream, bool overwrite,
CancellationToken ct)
{
string path = ResolvePath(properties.Key);
Directory.CreateDirectory(Path.GetDirectoryName(path));
FileMode mode = overwrite ? FileMode.Create : FileMode.CreateNew;
using (FileStream dst = File.Open(path, mode, FileAccess.Write, FileShare.None))
{
dst.SetLength(properties.Size);
dst.Position = 0;
await stream.CopyToAsync(dst).ConfigureAwait(false);
}
File.SetLastWriteTimeUtc(path, properties.Timestamp.UtcDateTime);
}
private async Task<IList<FileProperties>> ListObjectImpl(string localKeyPrefix, CancellationToken ct)
{
var result = new List<FileProperties>();
localKeyPrefix = SanitizeKey(localKeyPrefix ?? "");
var request = new ListObjectsRequest
{
BucketName = _bucket,
Prefix = SanitizeKey(_rootKey + "/")
};
IList<S3Object> objects = await _s3.ListAllObjectsAsync(request, ct).ConfigureAwait(false);
foreach (S3Object obj in objects)
{
if (ct.IsCancellationRequested)
break;
if (obj.Size == 0 && obj.Key.EndsWith("/"))
{
// one type of virtual folder
continue;
}
DateTimeOffset timestamp = DateTimeOffset.Parse(obj.LastModified);
var prop = new FileProperties
{
LocalKey = obj.Key,
Size = obj.Size,
Key = ResolveKey(obj.Key),
Timestamp = timestamp
};
if (prop.Key.StartsWith(localKeyPrefix))
{
result.Add(prop);
}
}
return result.OrderBy(x => x.Key).ToList();
}
public override Task DeleteObject(FileProperties properties, CancellationToken ct)
{
string key = ResolveLocalKey(properties.Key);
var request = new DeleteObjectsRequest
{
BucketName = _bucket,
Keys = {new KeyVersion(key)}
};
return _s3.DeleteObjectsAsync(request);
}
public override async Task AddObject(FileProperties properties, Stream stream, bool overwrite,
CancellationToken ct)
{
string key = ResolveLocalKey(properties.Key);
var request = new TransferUtilityUploadRequest
{
BucketName = _bucket,
InputStream = stream,
Key = key
};
request.AddHeader("x-amz-meta-yas4-ts", properties.Timestamp.ToString("o"));
using (var util = new TransferUtility(_s3))
{
await util.UploadAsync(request).ConfigureAwait(false);
}
}
public abstract Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct);
public abstract Task DeleteObject(FileProperties properties, CancellationToken ct);
public abstract Task AddObject(FileProperties properties, Stream stream, bool overwrite, CancellationToken ct);
public override Task DeleteObject(FileProperties properties, CancellationToken ct)
{
string path = ResolvePath(properties.Key);
File.Delete(path);
return Task.FromResult(true);
}
public StorageAction(FileProperties properties, StorageOperation action)
: this()
{
Properties = properties;
Operation = action;
}
