public static bool GetFeatureFlagState(FeatureAvailabilityHttpClient featureAvailabilityHttpClient, string FFName, IAsyncCommandContext context)
{
try
{
var featureFlag = featureAvailabilityHttpClient?.GetFeatureFlagByNameAsync(FFName).Result;
if (featureFlag != null && featureFlag.EffectiveState.Equals("Off", StringComparison.OrdinalIgnoreCase))
{
return(false);
}
}
catch
{
context.Debug(StringUtil.Format("Failed to get FF {0} Value. By default, publishing data to TCM.", FFName));
return(true);
}
return(true);
}
private async Task <List <string> > UploadAsync(IAsyncCommandContext context, int uploaderId, CancellationToken token)
{
List <string> failedFiles = new List <string>();
string fileToUpload;
Stopwatch uploadTimer = new Stopwatch();
while (_fileUploadQueue.TryDequeue(out fileToUpload))
{
token.ThrowIfCancellationRequested();
try
{
using (FileStream fs = File.Open(fileToUpload, FileMode.Open, FileAccess.Read, FileShare.Read))
{
string itemPath = (_containerPath.TrimEnd('/') + "/" + fileToUpload.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
uploadTimer.Restart();
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
try
{
response = await _fileContainerHttpClient.UploadFileAsync(_containerId, itemPath, fs, _projectId, cancellationToken : token, chunkSize : 4 * 1024 * 1024);
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", fileToUpload));
if (response != null)
{
response.Dispose();
response = null;
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", fileToUpload, ex.Message));
context.Output(ex.ToString());
}
uploadTimer.Stop();
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created))
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, fileToUpload, itemPath));
}
// output detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Output(StringUtil.Loc("FileUploadDetailTrace", itemPath));
string message;
while (logQueue.TryDequeue(out message))
{
context.Output(message);
}
}
// tracking file that failed to upload.
failedFiles.Add(fileToUpload);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", fileToUpload, uploadTimer.ElapsedMilliseconds));
// debug detail upload trace for the file.
ConcurrentQueue <string> logQueue;
if (_fileUploadTraceLog.TryGetValue(itemPath, out logQueue))
{
context.Debug($"Detail upload trace for file: {itemPath}");
string message;
while (logQueue.TryDequeue(out message))
{
context.Debug(message);
}
}
}
if (response != null)
{
response.Dispose();
response = null;
}
}
Interlocked.Increment(ref _filesProcessed);
}
catch (Exception ex)
{
context.Output(StringUtil.Loc("FileUploadFileOpenFailed", ex.Message, fileToUpload));
throw ex;
}
}
return(failedFiles);
}
private async Task <UploadResult> AssociateAsync(IAsyncCommandContext context, ConcurrentQueue <BlobFileInfo> associateQueue, CancellationToken token)
{
var uploadResult = new UploadResult();
var retryHelper = new RetryHelper(context);
var uploadTimer = new Stopwatch();
while (associateQueue.TryDequeue(out var file))
{
uploadTimer.Restart();
string itemPath = (_containerPath.TrimEnd('/') + "/" + file.Path.Remove(0, _sourceParentDirectory.Length + 1)).Replace('\\', '/');
bool catchExceptionDuringUpload = false;
HttpResponseMessage response = null;
try
{
if (file.Success)
{
var length = (long)file.Node.TransitiveContentBytes;
response = await retryHelper.Retry(async() => await _fileContainerHttpClient.CreateItemForArtifactUpload(_containerId, itemPath, _projectId,
file.DedupId.ValueString, length, token),
(retryCounter) => (int)Math.Pow(retryCounter, 2) * 5,
(exception) => true);
uploadResult.TotalFileSizeUploaded += length;
}
}
catch (OperationCanceledException) when(token.IsCancellationRequested)
{
context.Output(StringUtil.Loc("FileUploadCancelled", itemPath));
if (response != null)
{
response.Dispose();
}
throw;
}
catch (Exception ex)
{
catchExceptionDuringUpload = true;
context.Output(StringUtil.Loc("FileUploadFailed", itemPath, ex.Message));
context.Output(ex.ToString());
}
if (catchExceptionDuringUpload || (response != null && response.StatusCode != HttpStatusCode.Created) || !file.Success)
{
if (response != null)
{
context.Output(StringUtil.Loc("FileContainerUploadFailed", response.StatusCode, response.ReasonPhrase, file.Path, itemPath));
}
if (!file.Success)
{
context.Output(StringUtil.Loc("FileContainerUploadFailedBlob", file.Path, itemPath));
}
// tracking file that failed to upload.
uploadResult.FailedFiles.Add(file.Path);
}
else
{
context.Debug(StringUtil.Loc("FileUploadFinish", file.Path, uploadTimer.ElapsedMilliseconds));
}
if (response != null)
{
response.Dispose();
}
Interlocked.Increment(ref _filesProcessed);
}
return(uploadResult);
}
public RetryHelper(IAsyncCommandContext commandContext, int maxRetries = 3)
{
Debug = (str) => commandContext.Debug(str);
Warning = (str) => commandContext.Output(str);
MaxRetries = maxRetries;
}