public async Task <SyncInfo> Upload() { var si = new SyncInfo(st); var bkp = st.destination.ToBucketKeyPair(); var bucket = bkp.bucket; var key = bkp.key; si.start = DateTimeEx.UnixTimestampNow(); if (bucket.IsNullOrEmpty()) { throw new Exception($"Destination '{st.destination ?? "undefined"}' does not contain bucket name."); } var path = st.destination; var sourceInfo = st.GetSourceInfo(); if (sourceInfo.rootDirectory == null) { return(si); } var directory = st.source.ToDirectoryInfo(); var prefix = directory.FullName; var counter = 0; var status = await S3HashStoreStatus.GetStatusFile(s3h, st, S3HashStoreStatus.UploadStatusFilePrefix); var elspased = DateTimeEx.UnixTimestampNow() - status.timestamp; if (status.finalized) { var remaining = st.retention - elspased; Console.WriteLine($"Upload sync file '{st.status}' was already finalized {elspased}s ago. Next sync in {st.retention - elspased}s."); await Task.Delay(millisecondsDelay : 1000); si.success = true; return(si); } si.total = sourceInfo.files.Sum(x => x?.Length ?? 0); var cleanup = st.cleanup ? Cleanup(status) : null; var isStatusFileUpdated = false; var files = new List <SilyFileInfo>(); var uploadedBytes = new List <long>(); double compressionSum = 0; await ParallelEx.ForEachAsync(sourceInfo.files, async file => { double compression = 1; try { var sw = Stopwatch.StartNew(); var uploadedFile = status.files?.FirstOrDefault(x => x.FullNameEqual(file)); string localMD5; string destination; if (uploadedFile != null) //file was already uploaded to AWS { if (uploadedFile.LastWriteTime == file.LastWriteTime.ToUnixTimestamp()) { if (st.verbose) { Console.WriteLine($"Skipping upload of '{file.FullName}', file did not changed since last upload."); } await ss.LockAsync(() => { files.Add(uploadedFile); ++counter; }); return; //do not uplad, file did not changed } localMD5 = file.MD5().ToHexString(); destination = $"{key}/{localMD5}"; if (localMD5 == uploadedFile.MD5) { if (st.verbose) { Console.WriteLine($"Skipping upload of '{file.FullName}', file alredy exists in the '{bucket}/{destination}'."); } await ss.LockAsync(() => { ++counter; files.Add(uploadedFile); }); return; } } else //file was not uploaded to AWS yet { localMD5 = file.MD5().ToHexString(); destination = $"{key}/{localMD5}"; var metadata = await s3h.ObjectMetadataAsync( bucketName: bucket, key: $"{key}/{localMD5}", throwIfNotFound: false) .Timeout(msTimeout: st.timeout) .TryCatchRetryAsync(maxRepeats: st.retry); if (metadata != null) //file exists { await ss.LockAsync(() => { ++counter; var sfi = file.ToSilyFileInfo(md5: localMD5); if (sfi.Length >= (metadata.ContentLength + 128)) { sfi.TrySetProperty("compress", "zip"); } files.Add(sfi); }); if (st.verbose) { Console.WriteLine($"Skipping upload of '{file.FullName}', file was found in the '{bucket}/{destination}'."); } return; } } await ss.LockAsync(async() => { if (!isStatusFileUpdated) //update status file { status.timestamp = si.start; status.version = status.version + 1; status.finalized = false; var statusUploadResult = await s3h.UploadJsonAsync(status.bucket, status.key, status) .Timeout(msTimeout: st.timeout) .TryCatchRetryAsync(maxRepeats: st.retry); isStatusFileUpdated = true; } ++counter; }); async Task <string> UploadFile() { file?.Refresh(); if (file == null || !file.Exists) { return(null); } var shareMode = EnumEx.ToEnum <FileShare>(st.filesShare); FileInfo compressedFile = null; await ss.LockAsync(() => { if (st.compress) { compressedFile = PathEx.RuntimeCombine(st.sync, localMD5).ToFileInfo(); file.Zip(compressedFile); compressedFile.Refresh(); if ((compressedFile.Length + 128) < file.Length) { compression = (double)compressedFile.Length / Math.Max(file.Length, 1); } else { compression = 1; } } }); FileStream fs = null; await ss.LockAsync(() => { fs = File.Open( //upload new file to AWS compression < 1 ? compressedFile.FullName : file.FullName, FileMode.Open, FileAccess.Read, shareMode); }); var hash = await s3h.UploadStreamAsync(bucketName: bucket, key: destination, inputStream: fs, throwIfAlreadyExists: false, msTimeout: st.timeout).TryCatchRetryAsync(maxRepeats: st.retry); fs.Close(); if (!compressedFile.TryDelete()) { throw new Exception($"Failed to remove temporary file '{compressedFile?.FullName ?? "undefined"}' after deletion."); } return(hash.IsNullOrEmpty() ? null : hash); } if (st.verbose) { Console.WriteLine($"Uploading [{counter}/{sourceInfo.files.Length}][{file.Length.ToPrettyBytes()}] '{file.FullName}' => '{bucket}/{destination}' ..."); } var md5 = await UploadFile().TryCatchRetryAsync(maxRepeats: st.retry).Timeout(msTimeout: st.timeout); if (md5.IsNullOrEmpty()) { throw new Exception($"FAILED, Upload '{file.FullName}' => '{bucket}/{destination}'"); } var silyFile = file.ToSilyFileInfo(localMD5); if (compression < 1) { if (st.verbose) { Console.WriteLine($"File size reduced by [{compression * 100:0.00} %], file: '{file.FullName}' ({md5})"); } silyFile.TrySetProperty("compress", "zip"); compressionSum += compression; } else { if (md5 != localMD5 && st.verbose) { Console.WriteLine($"Warning! file hash changed during upload '{file.FullName}' {localMD5} => {md5}."); } compressionSum += 1; } await ss.LockAsync(() => { files.Add(silyFile); si.transferred += (long)(file.Length *compressionSum); }); } finally { await ss.LockAsync(() => { si.processed += file.Length; si.progress = ((double)si.processed / si.total) * 100; st.WriteInfoFile(si); }); } }, maxDegreeOfParallelism : st.parallelism); var directories = sourceInfo.directories.Select(x => x.ToSilyDirectoryInfo()).ToArray(); si.speed = (double)si.transferred / Math.Max(si.stop - si.start, 1); si.success = true; si.stop = DateTimeEx.UnixTimestampNow(); si.compression = (double)si.transferred / si.total; if (cleanup != null) { await cleanup; } if (isStatusFileUpdated || //if modifications were made to files !status.directories.JsonEquals(directories)) // or directories { status.files = files.ToArray(); status.finalized = true; status.directories = directories; status.source = st.source; status.destination = st.destination; var uploadResult = await s3h.UploadJsonAsync(status.bucket, status.key, status) .Timeout(msTimeout: st.timeout) .TryCatchRetryAsync(maxRepeats: st.retry); if (st.verbose) { Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated."); Console.WriteLine($"Uploaded {si.transferred.ToPrettyBytes()}, Speed: {si.speed.ToPrettyBytes()}/s, Compressed: {si.compression*100:0.00}%"); } } return(si); }
public async Task <SyncInfo> Download() { var bkp = st.source.ToBucketKeyPair(); var bucket = bkp.bucket; si = new SyncInfo(st); si.start = DateTimeEx.UnixTimestampNow(); if (bucket.IsNullOrEmpty()) { throw new Exception($"Source '{st.source ?? "undefined"}' does not contain bucket name."); } var destination = st.destination?.ToDirectoryInfo(); if (destination?.TryCreate() != true) { throw new Exception($"Destination '{st.destination ?? "undefined"}' does not exist and coudn't be created."); } if (st.verbose) { Console.WriteLine($"Processing Download Target: '{st?.id ?? "undefined"}'"); } var status = await S3HashStoreStatus.GetStatusFile(s3h, st, st.minTimestamp, st.maxTimestamp); var downloadStatus = st.ReadSyncFile(); if (status == null) { throw new Exception($"Could not download latest data from the source '{st.source}', status file was not found in '{st?.status ?? "undefined"}' within time range of <{st.minTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()},{st.maxTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()}>"); } status.files = status?.files?.Where(x => x != null)?.ToArray() ?? new SilyFileInfo[0]; si.total = status.files.Sum(x => x?.Length ?? 0); if (downloadStatus.finalized) { var elspased = DateTimeEx.UnixTimestampNow() - si.start; if (st.verbose) { Console.WriteLine($"Download sync file '{st.status}' was already finalized {elspased}s ago."); } await Task.Delay(millisecondsDelay : 1000); si.success = true; return(si); } if (st.verbose) { Console.WriteLine($"Download Target: '{st?.id ?? "undefined"}' status indicates that targt is not finalized"); } int counter = 0; var directories = new List <DirectoryInfo>(); directories.Add(st.destination.ToDirectoryInfo()); foreach (var dir in status.directories) { if (dir == null) { continue; } var relativeDir = dir.FullName.ToRuntimePath().TrimStart(status.source.ToRuntimePath()); var downloadDir = PathEx.RuntimeCombine(st.destination, relativeDir).ToDirectoryInfo(); if (!downloadDir.Exists && st.verbose) { Console.WriteLine($"Creating Directory [{++counter}/{status.directories.Length}] '{downloadDir.FullName}' ..."); } if (downloadDir?.TryCreate() != true) { throw new Exception($"Could not find or create directory '{downloadDir?.FullName ?? "undefined"}'."); } directories.Add(downloadDir); } if (st.wipe) { counter = 0; var currentDirectories = st.destination.ToDirectoryInfo().GetDirectories(recursive: st.recursive); foreach (var dir in currentDirectories) { if (!directories.Any(x => x.FullName == dir.FullName)) { Console.WriteLine($"Removing Directory [{++counter}/{currentDirectories.Length - directories.Count}] '{dir.FullName}' ..."); dir.Delete(recursive: st.recursive); } } } if (st.verbose) { Console.WriteLine($"Found {status.files} files and {status.directories} directories for target '{st?.id ?? "undefined"}'."); } counter = 1; var files = new List <FileInfo>(); await ParallelEx.ForEachAsync(status.files, async file => { try { var relativePath = file.FullName.ToRuntimePath().TrimStart(status.source.ToRuntimePath()); var downloadPath = PathEx.RuntimeCombine(st.destination, relativePath).ToFileInfo(); files.Add(downloadPath); if (downloadPath.Exists && downloadPath.MD5().ToHexString() == file.MD5) { if (st.verbose) { Console.WriteLine($"Found [{counter}/{status.files.Length}][{file.Length.ToPrettyBytes()}], file '{downloadPath.FullName}' ({file.MD5}) already exists."); } return; //file already exists } var key = $"{st.source.TrimEnd('/')}/{file.MD5}".ToBucketKeyPair().key; if (st.verbose) { Console.WriteLine($"Downloading [{counter}/{status.files.Length}][{file.Length.ToPrettyBytes()}] '{bucket}/{key}' => '{downloadPath.FullName}' ..."); } var sw = Stopwatch.StartNew(); async Task DownloadFile() { downloadPath.Refresh(); if (downloadPath.Exists && downloadPath.TryDelete() != true) { throw new Exception($"Obsolete file was found in '{downloadPath?.FullName ?? "undefined"}' but couldn't be deleted."); } using (var stream = await s3h.DownloadObjectAsync(bucketName: bucket, key: key, throwIfNotFound: true)) { var compressed = file.TryGetProperty("compress") == "zip"; if (!downloadPath.Directory.TryCreate()) { throw new Exception($"Failed to create directory '{downloadPath?.Directory.FullName ?? "undefined"}'."); } if (compressed) { if (st.verbose) { Console.WriteLine($"UnZipping '{downloadPath.FullName}' ..."); } downloadPath.UnZipStream(stream); } else { using (var fs = File.Create(downloadPath.FullName)) stream.CopyTo(fs); } } downloadPath.Refresh(); if (!downloadPath.Exists) { throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}'."); } if (st.verify) { var md5 = downloadPath.MD5().ToHexString(); if (md5 != file.MD5) { throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}', expected MD5 to be '{md5 ?? "undefined"}' but was '{file.MD5 ?? "undefined"}'."); } } await ss.LockAsync(() => { si.transferred += file.Length; }); } await DownloadFile().TryCatchRetryAsync(maxRepeats: st.retry).Timeout(msTimeout: st.timeout); } finally { await ss.LockAsync(() => { ++counter; si.processed += file.Length; si.progress = ((double)si.processed / si.total) * 100; st.WriteInfoFile(si); }); } }, maxDegreeOfParallelism : st.parallelism); if (st.wipe) { counter = 0; var currentFiles = st.destination.ToDirectoryInfo().GetFiles("*", recursive: st.recursive); foreach (var file in currentFiles) { if (!files.Any(x => x.FullName == file.FullName)) { if (st.verbose) { Console.WriteLine($"Removing File [{++counter}/{currentFiles.Length - files.Count}] '{file.FullName}' ..."); } file.Delete(); } } } downloadStatus.finalized = true; si.stop = DateTimeEx.UnixTimestampNow(); si.speed = (double)si.transferred / Math.Max(si.stop - si.start, 1); si.success = true; st.WriteSyncFile(downloadStatus); if (st.verbose) { Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated."); Console.WriteLine($"Average Download Speed: {si.speed.ToPrettyBytes()}/s"); } return(si); }