public async Task <SyncInfo> Upload()
{
var si = new SyncInfo(st);
var bkp = st.destination.ToBucketKeyPair();
var bucket = bkp.bucket;
var key = bkp.key;
si.start = DateTimeEx.UnixTimestampNow();
if (bucket.IsNullOrEmpty())
{
throw new Exception($"Destination '{st.destination ?? "undefined"}' does not contain bucket name.");
}
var path = st.destination;
var sourceInfo = st.GetSourceInfo();
if (sourceInfo.rootDirectory == null)
{
return(si);
}
var directory = st.source.ToDirectoryInfo();
var prefix = directory.FullName;
var counter = 0;
var status = await S3HashStoreStatus.GetStatusFile(s3h, st, S3HashStoreStatus.UploadStatusFilePrefix);
var elspased = DateTimeEx.UnixTimestampNow() - status.timestamp;
if (status.finalized)
{
var remaining = st.retention - elspased;
Console.WriteLine($"Upload sync file '{st.status}' was already finalized {elspased}s ago. Next sync in {st.retention - elspased}s.");
await Task.Delay(millisecondsDelay : 1000);
si.success = true;
return(si);
}
si.total = sourceInfo.files.Sum(x => x?.Length ?? 0);
var cleanup = st.cleanup ? Cleanup(status) : null;
var isStatusFileUpdated = false;
var files = new List <SilyFileInfo>();
var uploadedBytes = new List <long>();
double compressionSum = 0;
await ParallelEx.ForEachAsync(sourceInfo.files, async file =>
{
double compression = 1;
try
{
var sw = Stopwatch.StartNew();
var uploadedFile = status.files?.FirstOrDefault(x => x.FullNameEqual(file));
string localMD5;
string destination;
if (uploadedFile != null) //file was already uploaded to AWS
{
if (uploadedFile.LastWriteTime == file.LastWriteTime.ToUnixTimestamp())
{
if (st.verbose)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file did not changed since last upload.");
}
await ss.LockAsync(() =>
{
files.Add(uploadedFile);
++counter;
});
return; //do not uplad, file did not changed
}
localMD5 = file.MD5().ToHexString();
destination = $"{key}/{localMD5}";
if (localMD5 == uploadedFile.MD5)
{
if (st.verbose)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file alredy exists in the '{bucket}/{destination}'.");
}
await ss.LockAsync(() =>
{
++counter;
files.Add(uploadedFile);
});
return;
}
}
else //file was not uploaded to AWS yet
{
localMD5 = file.MD5().ToHexString();
destination = $"{key}/{localMD5}";
var metadata = await s3h.ObjectMetadataAsync(
bucketName: bucket,
key: $"{key}/{localMD5}",
throwIfNotFound: false)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry);
if (metadata != null) //file exists
{
await ss.LockAsync(() =>
{
++counter;
var sfi = file.ToSilyFileInfo(md5: localMD5);
if (sfi.Length >= (metadata.ContentLength + 128))
{
sfi.TrySetProperty("compress", "zip");
}
files.Add(sfi);
});
if (st.verbose)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file was found in the '{bucket}/{destination}'.");
}
return;
}
}
await ss.LockAsync(async() =>
{
if (!isStatusFileUpdated) //update status file
{
status.timestamp = si.start;
status.version = status.version + 1;
status.finalized = false;
var statusUploadResult = await s3h.UploadJsonAsync(status.bucket, status.key, status)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry);
isStatusFileUpdated = true;
}
++counter;
});
async Task <string> UploadFile()
{
file?.Refresh();
if (file == null || !file.Exists)
{
return(null);
}
var shareMode = EnumEx.ToEnum <FileShare>(st.filesShare);
FileInfo compressedFile = null;
await ss.LockAsync(() =>
{
if (st.compress)
{
compressedFile = PathEx.RuntimeCombine(st.sync, localMD5).ToFileInfo();
file.Zip(compressedFile);
compressedFile.Refresh();
if ((compressedFile.Length + 128) < file.Length)
{
compression = (double)compressedFile.Length / Math.Max(file.Length, 1);
}
else
{
compression = 1;
}
}
});
FileStream fs = null;
await ss.LockAsync(() =>
{
fs = File.Open( //upload new file to AWS
compression < 1 ? compressedFile.FullName : file.FullName,
FileMode.Open,
FileAccess.Read,
shareMode);
});
var hash = await s3h.UploadStreamAsync(bucketName: bucket,
key: destination,
inputStream: fs,
throwIfAlreadyExists: false, msTimeout: st.timeout).TryCatchRetryAsync(maxRepeats: st.retry);
fs.Close();
if (!compressedFile.TryDelete())
{
throw new Exception($"Failed to remove temporary file '{compressedFile?.FullName ?? "undefined"}' after deletion.");
}
return(hash.IsNullOrEmpty() ? null : hash);
}
if (st.verbose)
{
Console.WriteLine($"Uploading [{counter}/{sourceInfo.files.Length}][{file.Length.ToPrettyBytes()}] '{file.FullName}' => '{bucket}/{destination}' ...");
}
var md5 = await UploadFile().TryCatchRetryAsync(maxRepeats: st.retry).Timeout(msTimeout: st.timeout);
if (md5.IsNullOrEmpty())
{
throw new Exception($"FAILED, Upload '{file.FullName}' => '{bucket}/{destination}'");
}
var silyFile = file.ToSilyFileInfo(localMD5);
if (compression < 1)
{
if (st.verbose)
{
Console.WriteLine($"File size reduced by [{compression * 100:0.00} %], file: '{file.FullName}' ({md5})");
}
silyFile.TrySetProperty("compress", "zip");
compressionSum += compression;
}
else
{
if (md5 != localMD5 && st.verbose)
{
Console.WriteLine($"Warning! file hash changed during upload '{file.FullName}' {localMD5} => {md5}.");
}
compressionSum += 1;
}
await ss.LockAsync(() =>
{
files.Add(silyFile);
si.transferred += (long)(file.Length *compressionSum);
});
}
finally
{
await ss.LockAsync(() =>
{
si.processed += file.Length;
si.progress = ((double)si.processed / si.total) * 100;
st.WriteInfoFile(si);
});
}
}, maxDegreeOfParallelism : st.parallelism);
var directories = sourceInfo.directories.Select(x => x.ToSilyDirectoryInfo()).ToArray();
si.speed = (double)si.transferred / Math.Max(si.stop - si.start, 1);
si.success = true;
si.stop = DateTimeEx.UnixTimestampNow();
si.compression = (double)si.transferred / si.total;
if (cleanup != null)
{
await cleanup;
}
if (isStatusFileUpdated || //if modifications were made to files
!status.directories.JsonEquals(directories)) // or directories
{
status.files = files.ToArray();
status.finalized = true;
status.directories = directories;
status.source = st.source;
status.destination = st.destination;
var uploadResult = await s3h.UploadJsonAsync(status.bucket, status.key, status)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry);
if (st.verbose)
{
Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated.");
Console.WriteLine($"Uploaded {si.transferred.ToPrettyBytes()}, Speed: {si.speed.ToPrettyBytes()}/s, Compressed: {si.compression*100:0.00}%");
}
}
return(si);
}
public async Task <SyncResult> DownloadAWS(SyncTarget st)
{
var bkp = st.source.ToBucketKeyPair();
var bucket = bkp.bucket;
var timestamp = DateTimeEx.UnixTimestampNow();
if (bucket.IsNullOrEmpty())
{
throw new Exception($"Source '{st.source ?? "undefined"}' does not contain bucket name.");
}
var destination = st.destination?.ToDirectoryInfo();
if (destination?.TryCreate() != true)
{
throw new Exception($"Destination '{st.destination ?? "undefined"}' does not exist and coudn't be created.");
}
var status = await GetStatusFile(st, st.minTimestamp, st.maxTimestamp);
var downloadStatus = await GetStatusFile(st, DownloadStatusFilePrefix);
if (status == null)
{
throw new Exception($"Could not download latest data from the source '{st.source}', status file was not found in '{st?.status ?? "undefined"}' within time range of <{st.minTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()},{st.maxTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()}>");
}
if (downloadStatus.finalized)
{
var elspased = DateTimeEx.UnixTimestampNow() - downloadStatus.timestamp;
Console.WriteLine($"Download sync file '{st.status}' was already finalized {elspased}s ago.");
await Task.Delay(millisecondsDelay : 1000);
return(new SyncResult(success: true));
}
_syncInfo[st.id] = new SyncInfo(st);
_syncInfo[st.id].total = status.files.Sum(x => x?.Length ?? 0);
_syncInfo[st.id].timestamp = timestamp;
int counter = 0;
var directories = new List <DirectoryInfo>();
directories.Add(st.destination.ToDirectoryInfo());
foreach (var dir in status.directories)
{
if (dir == null)
{
continue;
}
var relativeDir = dir.FullName.TrimStart(status.source);
var downloadDir = PathEx.RuntimeCombine(st.destination, relativeDir).ToDirectoryInfo();
if (!downloadDir.Exists && st.verbose >= 1)
{
Console.WriteLine($"Creating Directory [{++counter}/{status.directories.Length}] '{downloadDir.FullName}' ...");
}
if (downloadDir?.TryCreate() != true)
{
throw new Exception($"Could not find or create directory '{downloadDir?.FullName ?? "undefined"}'.");
}
directories.Add(downloadDir);
}
if (st.wipe)
{
counter = 0;
var currentDirectories = st.destination.ToDirectoryInfo().GetDirectories(recursive: st.recursive);
foreach (var dir in currentDirectories)
{
if (!directories.Any(x => x.FullName == dir.FullName))
{
Console.WriteLine($"Removing Directory [{++counter}/{currentDirectories.Length - directories.Count}] '{dir.FullName}' ...");
dir.Delete(recursive: st.recursive);
}
}
}
counter = 0;
var files = new List <FileInfo>();
var speedList = new List <double>();
await ParallelEx.ForEachAsync(status.files, async file =>
{
if (file == null)
{
return;
}
try
{
var relativePath = file.FullName.TrimStart(status.source);
var downloadPath = PathEx.RuntimeCombine(st.destination, relativePath).ToFileInfo();
files.Add(downloadPath);
if (downloadPath.Exists && downloadPath.MD5().ToHexString() == file.MD5)
{
return; //file already exists
}
if (downloadPath.Exists && downloadPath.TryDelete() != true)
{
throw new Exception($"Obsolete file was found in '{downloadPath?.FullName ?? "undefined"}' but couldn't be deleted.");
}
var key = $"{st.source.TrimEnd('/')}/{file.MD5}".ToBucketKeyPair().key;
++counter;
if (st.verbose >= 1)
{
Console.WriteLine($"Downloading [{counter}/{status.files.Length}][{file.Length}B] '{bucket}/{key}' => '{downloadPath.FullName}' ...");
}
var sw = Stopwatch.StartNew();
var stream = await _S3Helper.DownloadObjectAsync(bucketName: bucket, key: key, throwIfNotFound: true)
.Timeout(msTimeout: st.timeout);
if (!downloadPath.Directory.TryCreate())
{
throw new Exception($"Failed to create directory '{downloadPath?.Directory.FullName ?? "undefined"}'.");
}
using (var fs = File.Create(downloadPath.FullName))
stream.CopyTo(fs);
downloadPath.Refresh();
if (!downloadPath.Exists)
{
throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}'.");
}
if (st.verify)
{
var md5 = downloadPath.MD5().ToHexString();
if (md5 != file.MD5)
{
throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}', expected MD5 to be '{md5 ?? "undefined"}' but was '{file.MD5 ?? "undefined"}'.");
}
else
{
lock (_locker)
{
var megabytes = (double)(file.Length + (md5.Length + bucket.Length + key.Length) * sizeof(char)) / (1024 * 1024);
var seconds = (double)(sw.ElapsedMilliseconds + 1) / 1000;
var speed = megabytes / seconds;
speedList.Add(speed);
}
}
}
}
finally
{
_syncInfo[st.id].processed += file.Length;
_syncInfo[st.id].progress = ((double)_syncInfo[st.id].processed / _syncInfo[st.id].total) * 100;
}
}, maxDegreeOfParallelism : st.parallelism);
if (st.wipe)
{
counter = 0;
var currentFiles = st.destination.ToDirectoryInfo().GetFiles("*", recursive: st.recursive);
foreach (var file in currentFiles)
{
if (!files.Any(x => x.FullName == file.FullName))
{
Console.WriteLine($"Removing File [{++counter}/{currentFiles.Length - files.Count}] '{file.FullName}' ...");
file.Delete();
}
}
}
downloadStatus.finalized = true;
var uploadResult = await _S3Helper.UploadJsonAsync(downloadStatus.bucket, downloadStatus.key, downloadStatus)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry);
var avgSpeed = speedList.IsNullOrEmpty() ? double.NaN : speedList.Average();
Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated.");
Console.WriteLine($"Average Download Speed: {avgSpeed} MB/s");
return(new SyncResult(success: true, speed: avgSpeed));
}
public async Task FunctionHandler(ILambdaContext context)
{
_sw = Stopwatch.StartNew();
_context = context;
_logger = _context.Logger;
_logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Started");
_verbose = Environment.GetEnvironmentVariable("verbose").ToBoolOrDefault(true);
_masterChatId = new ChatId(Environment.GetEnvironmentVariable("MASTER_CHAT_ID").ToLongOrDefault(-1001261081309));
_maxParallelism = Environment.GetEnvironmentVariable("MAX_PARALLELISM").ToIntOrDefault(0);
_cosmosHubClientTimeout = Environment.GetEnvironmentVariable("HUB_CLIENT_TIMEOUT").ToIntOrDefault(7);
_maxMessageAge = Environment.GetEnvironmentVariable("MAX_MESSAGE_AGE").ToDoubleOrDefault(24 * 3600);
_bucket = Environment.GetEnvironmentVariable("BUCKET_NAME");
_lambdaTime = Environment.GetEnvironmentVariable("LAMBDA_TIME").ToIntOrDefault((15 * 60 * 1000) - 5000);
var secretName = Environment.GetEnvironmentVariable("SECRET_NAME") ?? "KiraFaucetBot";
if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false))
{
Log($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available.");
}
var secret = JObject.Parse(await _SM.GetSecret(secretName));
#if (TEST)
var accessToken = secret["test_token"]?.ToString();
//_mnemonic = (secret["test_mnemonic"]?.ToString()).ToSecureString();
_mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString();
_bucket = "kira-faucet-test";
#elif (PUBLISH)
var accessToken = secret["token"]?.ToString();
_mnemonic = (secret["mnemonic"]?.ToString()).ToSecureString();
#endif
_TBC = new TelegramBotClient(accessToken);
_bot = await _TBC.GetMeAsync();
Log($"[INFO] {_bot.FirstName} {_version} started! Bot Name: @{_bot.Username ?? "undefined"}, Bot Id: '{_bot.Id}', Master Chat: '{_masterChatId.Identifier}'");
_TBC.OnMessage += Tbc_OnMessage;
_TBC.OnCallbackQuery += _TBC_OnCallbackQuery;
_TBC.OnInlineQuery += _TBC_OnInlineQuery;
_TBC.OnInlineResultChosen += _TBC_OnInlineResultChosen;
_TBC.StartReceiving();
try
{
Log($"Processing...");
var finalize = false;
while (true)
{
#if (PUBLISH)
if (!finalize && _sw.ElapsedMilliseconds >= _lambdaTime)
{
_TBC.StopReceiving();
finalize = true;
_logger.Log($"Finalizing, elapsed {_sw.ElapsedMilliseconds} / {_lambdaTime} [ms] ...");
}
#endif
if (_messages.IsNullOrEmpty() && _callbacks.IsNullOrEmpty())
{
if (finalize)
{
_logger.Log($"Lambda was finalized gracefully within {_lambdaTime - _sw.ElapsedMilliseconds} ms.");
return;
}
else
{
await Task.Delay(100);
continue;
}
}
Message[] msgArr = null;
_ssMsgLocker.Lock(() =>
{
msgArr = _messages.ToArray().DeepCopy();
_messages.Clear();
});
var t0 = ParallelEx.ForEachAsync(msgArr, async msg => {
async Task ProcessUser(Message m)
{
var user = m.From;
var replyUser = m.ReplyToMessage?.From;
if (user != null)
{
await UpdateUserData(user);
}
if (replyUser != null && user?.Id != replyUser.Id)
{
await UpdateUserData(replyUser);
}
}
#if (TEST)
await ProcessUser(msg);
#elif (PUBLISH)
try
{
await ProcessUser(msg);
}
catch (Exception ex)
{
_logger.Log($"[USR ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to save user status: '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
}
#endif
});
var t1 = ParallelEx.ForEachAsync(msgArr, async msg =>
{
#if (TEST)
await ProcessMessage(msg);
#elif (PUBLISH)
try
{
await ProcessMessage(msg);
}
catch (Exception ex)
{
_logger.Log($"[MSG ERROR] => Filed ('{msg?.Chat?.Id ?? 0}') to process message ({msg?.MessageId}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
await _TBC.SendTextMessageAsync(chatId: msg.Chat,
$"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.",
replyToMessageId: msg.MessageId,
parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown);
}
#endif
}, maxDegreeOfParallelism: _maxParallelism);
CallbackQuery[] cbqArr = null;
_ssCbqLocker.Lock(() =>
{
cbqArr = _callbacks.ToArray().DeepCopy();
_callbacks.Clear();
});
var t2 = ParallelEx.ForEachAsync(cbqArr, async cbq =>
{
#if (TEST)
await ProcessCallbacks(cbq);
#elif (PUBLISH)
try
{
await ProcessCallbacks(cbq);
}
catch (Exception ex)
{
_logger.Log($"[CBQ ERROR] => Filed ('{cbq.Message?.Chat?.Id ?? 0}') to process callback ({cbq.Id}): '{ex.JsonSerializeAsPrettyException(Newtonsoft.Json.Formatting.Indented)}'");
await _TBC.SendTextMessageAsync(chatId: cbq.Message.Chat,
$"Something went wrong, visit {await GetMasterChatInviteLink()} to find help.",
parseMode: Telegram.Bot.Types.Enums.ParseMode.Markdown);
}
#endif
}, maxDegreeOfParallelism: _maxParallelism);
await Task.WhenAll(t0, t1, t2);
}
}
finally
{
_logger.Log($"{context?.FunctionName} => {nameof(FunctionHandler)} => Stopped, Eveluated within: {_sw.ElapsedMilliseconds} [ms]");
}
}
public static PeakTransitionGroupFeatureSet GetPeakFeatures(this SrmDocument document,
IList <IPeakFeatureCalculator> calcs,
IProgressMonitor progressMonitor = null,
bool verbose = false)
{
// Get features for each peptide
int totalPeptides = document.MoleculeCount;
int currentPeptide = 0;
IProgressStatus status = new ProgressStatus(Resources.PeakFeatureEnumerator_GetPeakFeatures_Calculating_peak_group_scores);
// Set up run ID dictionary
var runEnumDict = new Dictionary <int, int>();
var chromatograms = document.Settings.MeasuredResults.Chromatograms;
foreach (var fileInfo in chromatograms.SelectMany(c => c.MSDataFileInfos))
{
runEnumDict.Add(fileInfo.FileIndex, runEnumDict.Count + 1);
}
// Using Parallel.For is quicker, but order needs to be maintained
var moleculeGroupPairs = document.GetMoleculeGroupPairs();
var peakFeatureLists = new PeakTransitionGroupFeatures[moleculeGroupPairs.Length][];
int peakFeatureCount = 0;
ParallelEx.For(0, moleculeGroupPairs.Length, i =>
{
var pair = moleculeGroupPairs[i];
var nodePepGroup = pair.NodeMoleculeGroup;
var nodePep = pair.NodeMolecule;
if (nodePep.TransitionGroupCount == 0)
{
return;
}
// Exclude standard peptides
if (nodePep.GlobalStandardType != null)
{
return;
}
if (progressMonitor != null)
{
if (progressMonitor.IsCanceled)
{
throw new OperationCanceledException();
}
int?percentComplete = ProgressStatus.ThreadsafeIncementPercent(ref currentPeptide, totalPeptides);
if (percentComplete.HasValue && percentComplete.Value < 100)
{
progressMonitor.UpdateProgress(status = status.ChangePercentComplete(percentComplete.Value));
}
}
var peakFeatureList = new List <PeakTransitionGroupFeatures>();
foreach (var peakFeature in document.GetPeakFeatures(nodePepGroup, nodePep, calcs, runEnumDict, verbose))
{
if (peakFeature.PeakGroupFeatures.Any())
{
peakFeatureList.Add(peakFeature);
Interlocked.Increment(ref peakFeatureCount);
}
}
peakFeatureLists[i] = peakFeatureList.ToArray();
});
var result = new PeakTransitionGroupFeatures[peakFeatureCount];
int peakFeatureCurrent = 0;
int decoyCount = 0;
foreach (var peakFeatureList in peakFeatureLists)
{
if (peakFeatureList == null)
{
continue;
}
foreach (var peakFeature in peakFeatureList)
{
result[peakFeatureCurrent++] = peakFeature;
if (peakFeature.IsDecoy)
{
decoyCount++;
}
}
}
if (progressMonitor != null)
{
progressMonitor.UpdateProgress(status.ChangePercentComplete(100));
}
return(new PeakTransitionGroupFeatureSet(decoyCount, result));
}
/// <summary>
/// Constructs batches and makes predictions in parallel
/// </summary>
/// <param name="predictionClient">Client to use for prediction</param>
/// <param name="progressMonitor">Monitor to show progress in UI</param>
/// <param name="progressStatus"/>
/// <param name="settings">Settings to use for constructing inputs and outputs</param>
/// <param name="inputs">List of inputs to predict</param>
/// <param name="token">Token for cancelling prediction</param>
/// <returns>Predictions from Prosit</returns>
public TSkylineOutput PredictBatches(PredictionService.PredictionServiceClient predictionClient,
IProgressMonitor progressMonitor, ref IProgressStatus progressStatus, SrmSettings settings, IList <TSkylineInputRow> inputs, CancellationToken token)
{
const int CONSTRUCTING_INPUTS_FRACTION = 50;
progressMonitor.UpdateProgress(progressStatus = progressStatus
.ChangeMessage(PrositResources.PrositModel_BatchPredict_Constructing_Prosit_inputs)
.ChangePercentComplete(0));
inputs = inputs.Distinct().ToArray();
var processed = 0;
var totalCount = inputs.Count;
var inputLock = new object();
var inputsList =
new List <TPrositIn>();
var validInputsList =
new List <List <TSkylineInputRow> >();
// Construct batch inputs in parallel
var localProgressStatus = progressStatus;
ParallelEx.ForEach(PrositHelpers.EnumerateBatches(inputs, PrositConstants.BATCH_SIZE),
batchEnumerable =>
{
var batch = batchEnumerable.ToArray();
var batchInputs = new List <TPrositInputRow>(batch.Length);
var validSkylineInputs = new List <TSkylineInputRow>(batch.Length);
foreach (var singleInput in batch)
{
var input = CreatePrositInputRow(settings, singleInput, out _);
if (input != null)
{
batchInputs.Add(input);
validSkylineInputs.Add(singleInput);
}
}
lock (inputLock)
{
inputsList.Add(CreatePrositInput(batchInputs));
validInputsList.Add(validSkylineInputs);
// ReSharper disable AccessToModifiedClosure
processed += batch.Length;
progressMonitor.UpdateProgress(localProgressStatus.ChangePercentComplete(CONSTRUCTING_INPUTS_FRACTION * processed / totalCount));
// ReSharper enable AccessToModifiedClosure
}
});
processed = 0;
totalCount = inputsList.Sum(pi => pi.InputRows.Count);
const int REQUESTING_INPUTS_FRACTION = 100 - CONSTRUCTING_INPUTS_FRACTION;
progressStatus = progressStatus
.ChangeMessage(PrositResources.PrositModel_BatchPredict_Requesting_predictions_from_Prosit)
.ChangePercentComplete(CONSTRUCTING_INPUTS_FRACTION);
progressMonitor.UpdateProgress(progressStatus);
// Make predictions batch by batch in sequence and merge the outputs
var prositOutputAll = new TPrositOut();
foreach (var prositIn in inputsList)
{
var prositOutput = Predict(predictionClient, prositIn, token);
prositOutputAll = prositOutputAll.MergeOutputs(prositOutput);
processed += prositIn.InputRows.Count;
progressStatus = progressStatus.ChangeMessage(TextUtil.SpaceSeparate(
PrositResources.PrositModel_BatchPredict_Requesting_predictions_from_Prosit,
processed.ToString(), @"/", totalCount.ToString()))
.ChangePercentComplete(CONSTRUCTING_INPUTS_FRACTION +
REQUESTING_INPUTS_FRACTION * processed / totalCount);
progressMonitor.UpdateProgress(progressStatus);
}
return(CreateSkylineOutput(settings, validInputsList.SelectMany(i => i).ToArray(), prositOutputAll));
}
public async Task <SyncInfo> Download()
{
var bkp = st.source.ToBucketKeyPair();
var bucket = bkp.bucket;
si = new SyncInfo(st);
si.start = DateTimeEx.UnixTimestampNow();
if (bucket.IsNullOrEmpty())
{
throw new Exception($"Source '{st.source ?? "undefined"}' does not contain bucket name.");
}
var destination = st.destination?.ToDirectoryInfo();
if (destination?.TryCreate() != true)
{
throw new Exception($"Destination '{st.destination ?? "undefined"}' does not exist and coudn't be created.");
}
if (st.verbose)
{
Console.WriteLine($"Processing Download Target: '{st?.id ?? "undefined"}'");
}
var status = await S3HashStoreStatus.GetStatusFile(s3h, st, st.minTimestamp, st.maxTimestamp);
var downloadStatus = st.ReadSyncFile();
if (status == null)
{
throw new Exception($"Could not download latest data from the source '{st.source}', status file was not found in '{st?.status ?? "undefined"}' within time range of <{st.minTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()},{st.maxTimestamp.ToDateTimeFromTimestamp().ToLongDateTimeString()}>");
}
status.files = status?.files?.Where(x => x != null)?.ToArray() ?? new SilyFileInfo[0];
si.total = status.files.Sum(x => x?.Length ?? 0);
if (downloadStatus.finalized)
{
var elspased = DateTimeEx.UnixTimestampNow() - si.start;
if (st.verbose)
{
Console.WriteLine($"Download sync file '{st.status}' was already finalized {elspased}s ago.");
}
await Task.Delay(millisecondsDelay : 1000);
si.success = true;
return(si);
}
if (st.verbose)
{
Console.WriteLine($"Download Target: '{st?.id ?? "undefined"}' status indicates that targt is not finalized");
}
int counter = 0;
var directories = new List <DirectoryInfo>();
directories.Add(st.destination.ToDirectoryInfo());
foreach (var dir in status.directories)
{
if (dir == null)
{
continue;
}
var relativeDir = dir.FullName.ToRuntimePath().TrimStart(status.source.ToRuntimePath());
var downloadDir = PathEx.RuntimeCombine(st.destination, relativeDir).ToDirectoryInfo();
if (!downloadDir.Exists && st.verbose)
{
Console.WriteLine($"Creating Directory [{++counter}/{status.directories.Length}] '{downloadDir.FullName}' ...");
}
if (downloadDir?.TryCreate() != true)
{
throw new Exception($"Could not find or create directory '{downloadDir?.FullName ?? "undefined"}'.");
}
directories.Add(downloadDir);
}
if (st.wipe)
{
counter = 0;
var currentDirectories = st.destination.ToDirectoryInfo().GetDirectories(recursive: st.recursive);
foreach (var dir in currentDirectories)
{
if (!directories.Any(x => x.FullName == dir.FullName))
{
Console.WriteLine($"Removing Directory [{++counter}/{currentDirectories.Length - directories.Count}] '{dir.FullName}' ...");
dir.Delete(recursive: st.recursive);
}
}
}
if (st.verbose)
{
Console.WriteLine($"Found {status.files} files and {status.directories} directories for target '{st?.id ?? "undefined"}'.");
}
counter = 1;
var files = new List <FileInfo>();
await ParallelEx.ForEachAsync(status.files, async file =>
{
try
{
var relativePath = file.FullName.ToRuntimePath().TrimStart(status.source.ToRuntimePath());
var downloadPath = PathEx.RuntimeCombine(st.destination, relativePath).ToFileInfo();
files.Add(downloadPath);
if (downloadPath.Exists && downloadPath.MD5().ToHexString() == file.MD5)
{
if (st.verbose)
{
Console.WriteLine($"Found [{counter}/{status.files.Length}][{file.Length.ToPrettyBytes()}], file '{downloadPath.FullName}' ({file.MD5}) already exists.");
}
return; //file already exists
}
var key = $"{st.source.TrimEnd('/')}/{file.MD5}".ToBucketKeyPair().key;
if (st.verbose)
{
Console.WriteLine($"Downloading [{counter}/{status.files.Length}][{file.Length.ToPrettyBytes()}] '{bucket}/{key}' => '{downloadPath.FullName}' ...");
}
var sw = Stopwatch.StartNew();
async Task DownloadFile()
{
downloadPath.Refresh();
if (downloadPath.Exists && downloadPath.TryDelete() != true)
{
throw new Exception($"Obsolete file was found in '{downloadPath?.FullName ?? "undefined"}' but couldn't be deleted.");
}
using (var stream = await s3h.DownloadObjectAsync(bucketName: bucket, key: key, throwIfNotFound: true))
{
var compressed = file.TryGetProperty("compress") == "zip";
if (!downloadPath.Directory.TryCreate())
{
throw new Exception($"Failed to create directory '{downloadPath?.Directory.FullName ?? "undefined"}'.");
}
if (compressed)
{
if (st.verbose)
{
Console.WriteLine($"UnZipping '{downloadPath.FullName}' ...");
}
downloadPath.UnZipStream(stream);
}
else
{
using (var fs = File.Create(downloadPath.FullName))
stream.CopyTo(fs);
}
}
downloadPath.Refresh();
if (!downloadPath.Exists)
{
throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}'.");
}
if (st.verify)
{
var md5 = downloadPath.MD5().ToHexString();
if (md5 != file.MD5)
{
throw new Exception($"Failed download '{bucket}/{key}'-/-> '{downloadPath.FullName}', expected MD5 to be '{md5 ?? "undefined"}' but was '{file.MD5 ?? "undefined"}'.");
}
}
await ss.LockAsync(() =>
{
si.transferred += file.Length;
});
}
await DownloadFile().TryCatchRetryAsync(maxRepeats: st.retry).Timeout(msTimeout: st.timeout);
}
finally
{
await ss.LockAsync(() =>
{
++counter;
si.processed += file.Length;
si.progress = ((double)si.processed / si.total) * 100;
st.WriteInfoFile(si);
});
}
}, maxDegreeOfParallelism : st.parallelism);
if (st.wipe)
{
counter = 0;
var currentFiles = st.destination.ToDirectoryInfo().GetFiles("*", recursive: st.recursive);
foreach (var file in currentFiles)
{
if (!files.Any(x => x.FullName == file.FullName))
{
if (st.verbose)
{
Console.WriteLine($"Removing File [{++counter}/{currentFiles.Length - files.Count}] '{file.FullName}' ...");
}
file.Delete();
}
}
}
downloadStatus.finalized = true;
si.stop = DateTimeEx.UnixTimestampNow();
si.speed = (double)si.transferred / Math.Max(si.stop - si.start, 1);
si.success = true;
st.WriteSyncFile(downloadStatus);
if (st.verbose)
{
Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated.");
Console.WriteLine($"Average Download Speed: {si.speed.ToPrettyBytes()}/s");
}
return(si);
}
private static async Task <bool> executeScheduler(string[] args)
{
var nArgs = CLIHelper.GetNamedArguments(args);
switch (args[1]?.ToLower())
{
case "github":
{
if (Environment.GetEnvironmentVariable("test_connection").ToBoolOrDefault(false))
{
Console.WriteLine($"Your Internet Connection is {(SilyWebClientEx.CheckInternetAccess(timeout: 5000) ? "" : "NOT")} available.");
}
var userWhitelist = nArgs.GetValueOrDefault("users")?.Split(',');
var repositoriesWhitelist = nArgs.GetValueOrDefault("repositories")?.Split(',');
Console.WriteLine($"Fetching scheduler info.");
var workingDirectory = (await GetVariableByKey("working_directory", nArgs: nArgs)).ToDirectoryInfo();
var githubSchedule = await GetVariableByKey("github_schedule", nArgs : nArgs);
var user = GITWrapper.GitHubHelperEx.GetUserFromUrl(githubSchedule);
if (!userWhitelist.IsNullOrEmpty() && !userWhitelist.Any(x => x == user))
{
throw new Exception($"User was specified but, user '{user ?? "undefined"}' was not present among whitelisted users: {userWhitelist.JsonSerialize()}");
}
var accessToken = await GetSecretHexToken("github_token", nArgs);
var repo = GITWrapper.GitHubHelperEx.GetRepoFromUrl(githubSchedule);
if (!repositoriesWhitelist.IsNullOrEmpty() && !repositoriesWhitelist.Any(x => x == user))
{
throw new Exception($"Repository was specified but, repo '{repo ?? "undefined"}' was not present among whitelisted repositories: {repositoriesWhitelist.JsonSerialize()}");
}
var branch = GITWrapper.GitHubHelperEx.GetBranchFromUrl(githubSchedule);
var scheduleLocation = GITWrapper.GitHubHelperEx.GetFileFromUrl(githubSchedule);
var git = new GITWrapper.GitHubHelper(new GITWrapper.Models.GitHubRepoConfig()
{
accessToken = accessToken,
user = user,
repository = repo,
branch = branch
});
var contentDirectory = PathEx.RuntimeCombine(workingDirectory.FullName, repo).ToDirectoryInfo();
var statusDirectory = PathEx.RuntimeCombine(workingDirectory.FullName, "status").ToDirectoryInfo();
var logsDirectory = PathEx.RuntimeCombine(workingDirectory.FullName, "logs").ToDirectoryInfo();
var scheduleFileInfo = PathEx.RuntimeCombine(contentDirectory.FullName, scheduleLocation).ToFileInfo();
contentDirectory.TryDelete(recursive: true, exception: out var contentDirectoryException);
Console.WriteLine($"Removing git directory '{contentDirectory.FullName}' {(contentDirectory.Exists ? $"did NOT suceeded, error: {contentDirectoryException.JsonSerializeAsPrettyException()}" : "succeded")}.");
statusDirectory.TryCreate();
CommandOutput result;
var pullCommand = $"git clone https://{accessToken}@github.com/{user}/{repo}.git --branch {branch}";
result = CLIHelper.Console(pullCommand, workingDirectory: workingDirectory.FullName);
Console.WriteLine(result.JsonSerialize());
var gitDirectory = PathEx.RuntimeCombine(contentDirectory.FullName, ".git").ToDirectoryInfo();
gitDirectory.TryDelete(recursive: true);
Console.WriteLine($"Removing git directory '{gitDirectory.FullName}' {(gitDirectory.Exists ? "did NOT" : "")} succeded.");
if (!RuntimeEx.IsWindows())
{
result = CLIHelper.Console($"chmod 777 -R ./{repo}", workingDirectory: workingDirectory.FullName);
Console.WriteLine(result.JsonSerialize());
}
if (!scheduleFileInfo.Exists)
{
Console.WriteLine($"FAILURE, schedule file '{scheduleFileInfo.FullName}' does not exist or was not defined.");
return(false);
}
var deploymentConfig = scheduleFileInfo.DeserialiseJson <DeploymentConfig>();
var deploymentConfigOld = deploymentConfig.LoadDeploymentConfig(statusDirectory);
if (deploymentConfig?.enable != true || deploymentConfig.schedules.IsNullOrEmpty())
{
Console.WriteLine($"Deployment config '{scheduleFileInfo.FullName}' was not enabled or schedules were not defined.");
return(false);
}
//Defines if schedule executuions should be triggered
var masterTrigger = deploymentConfig.IsTriggered(deploymentConfigOld);
var serialSchedules = deploymentConfig.schedules
.Where(x => !(x?.id).IsNullOrEmpty() && x.parallelizable == false)
?.OrderBy(x => x.priority)?.DistinctBy(x => x.id)?.ToArray();
var parallelSchedules = deploymentConfig.schedules
.Where(x => !(x?.id).IsNullOrEmpty() && x.parallelizable == true)
?.OrderBy(x => x.priority)?.DistinctBy(x => x.id)?.ToArray();
var breakAll = false;
async Task TryCatchExecute(ExecutionSchedule s)
{
var sOld = s.LoadExecutionSchedule(statusDirectory);
if (s == null || sOld == null)
{
Console.WriteLine($"New or old schedule could not be found.");
return;
}
if (!s.IsTriggered(sOld, masterTrigger))
{
Console.WriteLine($"WARNING, schedule '{s?.id ?? "undefined"}' execution was not triggered.");
return;
}
Console.WriteLine($"Processing executioon schedule '{s.id}', parralelized: {s.parallelizable}, cron: {s.cron ?? "null"}, trigger: {s.trigger}/{sOld.trigger}.");
if (s.delay > 0)
{
await Task.Delay(s.delay);
}
if (_debug)
{
Console.WriteLine($"WARNING! github schedule will be processed in DEBUG mode");
await ProcessSchedule(s, sOld, contentDirectory, statusDirectory, logsDirectory, masterTrigger : masterTrigger);
return;
}
try
{
await ProcessSchedule(s, sOld, contentDirectory, statusDirectory, logsDirectory, masterTrigger : masterTrigger);
breakAll = s.breakAllOnFinalize;
if (s.sleep > 0)
{
await Task.Delay(s.sleep);
}
}
catch (Exception ex)
{
try
{
if (deploymentConfig.throwOnFailure == true)
{
if (deploymentConfig.finalizeOnFailure)
{
deploymentConfig.UpdateDeploymentConfig(statusDirectory);
breakAll = s.breakAllOnFinalize;
}
throw;
}
Console.WriteLine($"FAILED! execution of schedule '{s.id}', parralelized: {s.parallelizable}, error: {ex.JsonSerializeAsPrettyException()}.");
}
finally
{
var logPath = PathEx.Combine(logsDirectory.FullName, $"{s.GetFileSafeId() ?? "tmp.log"}.log").ToFileInfo();
if (logPath.TryCreate())
{
logPath.AppendAllText(ex.JsonSerializeAsPrettyException());
}
}
}
}
if (deploymentConfig.delay > 0)
{
await Task.Delay(deploymentConfig.delay);
}
var sum = 0;
if (!serialSchedules.IsNullOrEmpty())
{
sum += serialSchedules.Length;
foreach (var s in serialSchedules)
{
await TryCatchExecute(s);
}
}
if (!parallelSchedules.IsNullOrEmpty())
{
sum += serialSchedules.Length;
await ParallelEx.ForEachAsync(parallelSchedules,
s => TryCatchExecute(s), maxDegreeOfParallelism : parallelSchedules.Count());
}
deploymentConfig.UpdateDeploymentConfig(statusDirectory);
Console.WriteLine($"SUCCESS, {sum} github schedule/s was/were executed out of {deploymentConfig.schedules.Length}.");
if (deploymentConfig.sleep > 0)
{
await Task.Delay(deploymentConfig.sleep);
}
return(true);
}
case "help":
case "--help":
case "-help":
case "-h":
case "h":
{
HelpPrinter($"{args[0]}", "Command Deployment",
("github", "Accepts params: working_directory, github_schedule, github_token"));
return(true);
}
default:
{
Console.WriteLine($"Try '{args[0]} help' to find out list of available commands.");
throw new Exception($"Unknown String command: '{args[0]} {args[1]}'");
}
}
}
public async Task <SyncResult> UploadAWS(SyncTarget st)
{
var bkp = st.destination.ToBucketKeyPair();
var bucket = bkp.bucket;
var key = bkp.key;
var timestamp = DateTimeEx.UnixTimestampNow();
if (bucket.IsNullOrEmpty())
{
throw new Exception($"Destination '{st.destination ?? "undefined"}' does not contain bucket name.");
}
var path = st.destination;
var sourceInfo = st.GetSourceInfo();
if (sourceInfo.rootDirectory == null)
{
return(new SyncResult(success: false)); //failed to get source info
}
var directory = st.source.ToDirectoryInfo();
var prefix = directory.FullName;
var counter = 0;
var status = await GetStatusFile(st, UploadStatusFilePrefix);
var elspased = DateTimeEx.UnixTimestampNow() - status.timestamp;
if (status.finalized)
{
var remaining = st.retention - elspased;
Console.WriteLine($"Upload sync file '{st.status}' was already finalized {elspased}s ago. Next sync in {st.retention - elspased}s.");
await Task.Delay(millisecondsDelay : 1000);
return(new SyncResult(success: true));
}
_syncInfo[st.id] = new SyncInfo(st);
_syncInfo[st.id].total = sourceInfo.files.Sum(x => x?.Length ?? 0);
_syncInfo[st.id].timestamp = timestamp;
var cleanup = st.cleanup ? Cleanup(st, status) : null;
var isStatusFileUpdated = false;
var files = new List <SilyFileInfo>();
var speedList = new List <double>();
await ParallelEx.ForEachAsync(sourceInfo.files, async file =>
{
try
{
var sw = Stopwatch.StartNew();
var uploadedFile = status.files?.FirstOrDefault(x => x.FullNameEqual(file));
string localMD5;
string destination;
if (uploadedFile != null) //file was already uploaded to AWS
{
if (uploadedFile.LastWriteTime == file.LastWriteTime.ToUnixTimestamp())
{
if (st.verbose > 1)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file did not changed since last upload.");
}
lock (_locker)
{
files.Add(uploadedFile);
++counter;
}
return; //do not uplad, file did not changed
}
localMD5 = file.MD5().ToHexString();
destination = $"{key}/{localMD5}";
if (localMD5 == uploadedFile.MD5)
{
if (st.verbose > 1)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file alredy exists in the '{bucket}/{destination}'.");
}
lock (_locker)
{
++counter;
files.Add(uploadedFile);
}
return;
}
}
else //file was not uploaded to AWS yet
{
localMD5 = file.MD5().ToHexString();
destination = $"{key}/{localMD5}";
if (await _S3Helper.ObjectExistsAsync(bucketName: bucket, key: $"{key}/{localMD5}")
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry))
{
lock (_locker)
{
++counter;
files.Add(file.ToSilyFileInfo(md5: localMD5));
}
if (st.verbose > 1)
{
Console.WriteLine($"Skipping upload of '{file.FullName}', file was found in the '{bucket}/{destination}'.");
}
return;
}
}
lock (_locker)
{
if (!isStatusFileUpdated) //update status file
{
status.timestamp = timestamp;
status.version = status.version + 1;
status.finalized = false;
var statusUploadResult = _S3Helper.UploadJsonAsync(status.bucket, status.key, status)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry).Result;
isStatusFileUpdated = true;
}
++counter;
}
async Task <string> UploadFile()
{
file?.Refresh();
if (file == null || !file.Exists)
{
return(null);
}
using (var fs = File.Open( //upload new file to AWS
file.FullName,
FileMode.Open,
FileAccess.Read,
EnumEx.ToEnum <FileShare>(st.filesShare)))
{
var hash = await _S3Helper.UploadStreamAsync(bucketName: bucket,
key: destination,
inputStream: fs,
throwIfAlreadyExists: false, msTimeout: st.timeout).TryCatchRetryAsync(maxRepeats: st.retry);
fs.Close();
return(hash.IsNullOrEmpty() ? null : hash);
}
}
Console.WriteLine($"Uploading [{counter}/{sourceInfo.files.Length}][{file.Length}B] '{file.FullName}' => '{bucket}/{destination}' ...");
var md5 = await UploadFile().TryCatchRetryAsync(maxRepeats: st.retry);
if (md5 != localMD5)
{
Console.WriteLine($"Warning! file changed during upload '{file.FullName}' => '{bucket}/{destination}'.");
}
if (!md5.IsNullOrEmpty())
{
lock (_locker)
{
files.Add(file.ToSilyFileInfo(md5));
var megabytes = (double)(file.Length + (md5.Length + bucket.Length + key.Length) * sizeof(char)) / (1024 * 1024);
var seconds = (double)(sw.ElapsedMilliseconds + 1) / 1000;
var speed = megabytes / seconds;
speedList.Add(speed);
}
}
else
{
Console.WriteLine($"FAILED, Upload '{file.FullName}' => '{bucket}/{destination}'");
}
}
finally
{
_syncInfo[st.id].processed += file.Length;
_syncInfo[st.id].progress = ((double)_syncInfo[st.id].processed / _syncInfo[st.id].total) * 100;
}
}, maxDegreeOfParallelism : st.parallelism);
var directories = sourceInfo.directories.Select(x => x.ToSilyDirectoryInfo()).ToArray();
var avgSpeed = speedList.IsNullOrEmpty() ? double.NaN : speedList.Average();
if (cleanup != null)
{
await cleanup;
}
if (isStatusFileUpdated || //if modifications were made to files
!status.directories.JsonEquals(directories)) // or directories
{
status.files = files.ToArray();
status.finalized = true;
status.directories = directories;
status.source = st.source;
status.destination = st.destination;
var uploadResult = await _S3Helper.UploadJsonAsync(status.bucket, status.key, status)
.Timeout(msTimeout: st.timeout)
.TryCatchRetryAsync(maxRepeats: st.retry);
Console.WriteLine($"SUCCESS, processed '{st.status}', all {status.files.Length} files and {status.directories.Length} directories were updated.");
Console.WriteLine($"Average Upload Speed: {avgSpeed} MB/s");
}
return(new SyncResult(success: true, speed: avgSpeed));
}
public async Task Process()
{
var syncTargets = _cfg.GetSyncTargets();
if (syncTargets.IsNullOrEmpty())
{
return;
}
foreach (var st in syncTargets)
{
if (st.source.IsNullOrEmpty())
{
throw new Exception("SyncTarget 'source' was not defined");
}
if (st.destination.IsNullOrEmpty())
{
throw new Exception("SyncTarget 'destination' was not defined");
}
if (st.id.IsNullOrEmpty())
{
st.id = Guid.NewGuid().ToString();
}
}
if (_syncResult == null || _syncResult.Count != syncTargets.Length || syncTargets.Any(x => !_syncResult.ContainsKey(x.id)))
{
_syncResult = new ConcurrentDictionary <string, SyncResult>();
foreach (var st in syncTargets)
{
_syncResult.Add(st.id, null);
}
}
if (_syncInfo == null || _syncResult.Count != syncTargets.Length || syncTargets.Any(x => !_syncInfo.ContainsKey(x.id)))
{
_syncInfo = new ConcurrentDictionary <string, SyncInfo>();
foreach (var st in syncTargets)
{
_syncInfo.Add(st.id, null);
}
}
++_run;
await ParallelEx.ForEachAsync(syncTargets, async st => {
var sw = Stopwatch.StartNew();
if (st.type == SyncTarget.types.none)
{
return;
}
_S3Helper = st.profile.IsNullOrEmpty() ?
new S3Helper() :
new S3Helper(AWSWrapper.Extensions.Helper.GetAWSCredentials(st.profile));
SyncResult result;
if (st.type == SyncTarget.types.awsUpload)
{
/* // debug only
* result = await UploadAWS(st);
* /*/
result = TryProcessUploadAWS(st);
//*/
}
else if (st.type == SyncTarget.types.awsDownload)
{
/* // debug only
* result = await DownloadAWS(st);
* /*/
result = TryProcessDownloadAWS(st);
//*/
}
else
{
throw new Exception($"SyncTarget type '{st.type.ToString()}' was not defined");
}
result.run = _run;
result.duration = sw.ElapsedMilliseconds / 1000;
_syncResult[st.id] = result;
if (st.sleep >= 0)
{
var sleep = st.sleep + 1000;
Console.WriteLine($"Sync Task {st.id} was compleated, result: {(result.success ? "success":"failure")}, sleep: {st.sleep} [ms].");
await Task.Delay(st.sleep);
}
}, maxDegreeOfParallelism : _cfg.parallelism);
}