private IList<FileProperties> ListObjectImpl(string localKeyPrefix, CancellationToken ct) { var result = new List<FileProperties>(); localKeyPrefix = SanitizeKey(localKeyPrefix ?? ""); if (Directory.Exists(_rootPath)) { foreach (string path in Directory.EnumerateFiles(_rootPath, "*", SearchOption.AllDirectories)) { if (ct.IsCancellationRequested) break; var info = new FileInfo(path); var prop = new FileProperties { LocalKey = path, Size = info.Length, Key = ResolveKey(path), Timestamp = ResolveTimestamp(path) }; if (prop.Key.StartsWith(localKeyPrefix)) { result.Add(prop); } } } return result.OrderBy(x => x.Key).ToList(); }
public override async Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct) { string path = ResolvePath(properties.Key); using (FileStream file = File.OpenRead(path)) { await file.CopyToAsync(stream); } }
public override Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct) { const double mb = 1024*1024; string key = ResolveLocalKey(properties.Key); if (MaxParallelStreams == 1) return ReadObject(stream, ct, key); if (properties.Size > 1*mb) return ReadObjectParallel(properties, stream, ct, key, 1*(int) mb); return ReadObject(stream, ct, key); }
private async Task ReadObjectParallel(FileProperties properties, Stream stream, CancellationToken ct, string key, int partSize) { var requests = new List<GetObjectRequest>(); long partCount = properties.Size/partSize; if (properties.Size%partSize > 0) partCount++; for (int i = 0; i < partCount; i++) { long startIncl = i*partSize; long endExcl = Math.Min(startIncl + partSize, properties.Size); requests.Add(new GetObjectRequest { BucketName = _bucket, Key = key, ByteRangeLong = new Amazon.S3.Model.Tuple<long, long>(startIncl, endExcl - 1) }); } var fileLock = new CloudSync.AsyncLock(); await CloudSync.ForEachPooled(requests, MaxParallelStreams, ct, async (r, t) => { using (var buffer = new MemoryStream()) using (GetObjectResponse response = await _s3.GetObjectAsync(r).ConfigureAwait(false)) { await response.ResponseStream.CopyToAsync(buffer).ConfigureAwait(false); await buffer.FlushAsync().ConfigureAwait(false); // truncate buffer to be safe buffer.Position = 0; buffer.SetLength(r.ByteRangeLong.Second - r.ByteRangeLong.First + 1); using (await fileLock.LockAsync().ConfigureAwait(false)) { stream.Position = r.ByteRangeLong.First; buffer.CopyToAsync(stream, 64*1024, ct).Wait(); stream.FlushAsync().Wait(); } } }).ConfigureAwait(false); }
public override async Task AddObject(FileProperties properties, Stream stream, bool overwrite, CancellationToken ct) { string path = ResolvePath(properties.Key); Directory.CreateDirectory(Path.GetDirectoryName(path)); FileMode mode = overwrite ? FileMode.Create : FileMode.CreateNew; using (FileStream dst = File.Open(path, mode, FileAccess.Write, FileShare.None)) { dst.SetLength(properties.Size); dst.Position = 0; await stream.CopyToAsync(dst).ConfigureAwait(false); } File.SetLastWriteTimeUtc(path, properties.Timestamp.UtcDateTime); }
private async Task<IList<FileProperties>> ListObjectImpl(string localKeyPrefix, CancellationToken ct) { var result = new List<FileProperties>(); localKeyPrefix = SanitizeKey(localKeyPrefix ?? ""); var request = new ListObjectsRequest { BucketName = _bucket, Prefix = SanitizeKey(_rootKey + "/") }; IList<S3Object> objects = await _s3.ListAllObjectsAsync(request, ct).ConfigureAwait(false); foreach (S3Object obj in objects) { if (ct.IsCancellationRequested) break; if (obj.Size == 0 && obj.Key.EndsWith("/")) { // one type of virtual folder continue; } DateTimeOffset timestamp = DateTimeOffset.Parse(obj.LastModified); var prop = new FileProperties { LocalKey = obj.Key, Size = obj.Size, Key = ResolveKey(obj.Key), Timestamp = timestamp }; if (prop.Key.StartsWith(localKeyPrefix)) { result.Add(prop); } } return result.OrderBy(x => x.Key).ToList(); }
public override Task DeleteObject(FileProperties properties, CancellationToken ct) { string key = ResolveLocalKey(properties.Key); var request = new DeleteObjectsRequest { BucketName = _bucket, Keys = {new KeyVersion(key)} }; return _s3.DeleteObjectsAsync(request); }
public override async Task AddObject(FileProperties properties, Stream stream, bool overwrite, CancellationToken ct) { string key = ResolveLocalKey(properties.Key); var request = new TransferUtilityUploadRequest { BucketName = _bucket, InputStream = stream, Key = key }; request.AddHeader("x-amz-meta-yas4-ts", properties.Timestamp.ToString("o")); using (var util = new TransferUtility(_s3)) { await util.UploadAsync(request).ConfigureAwait(false); } }
public abstract Task ReadObject(FileProperties properties, Stream stream, CancellationToken ct);
public abstract Task DeleteObject(FileProperties properties, CancellationToken ct);
public abstract Task AddObject(FileProperties properties, Stream stream, bool overwrite, CancellationToken ct);
public override Task DeleteObject(FileProperties properties, CancellationToken ct) { string path = ResolvePath(properties.Key); File.Delete(path); return Task.FromResult(true); }
public StorageAction(FileProperties properties, StorageOperation action) : this() { Properties = properties; Operation = action; }