private async Task <T> DoWithRetry <T>(FileEntryItem item, Func <Task <T> > method)
{
item.IsRetry = false;
Exception lastException = null;
if (!await m_taskreader.TransferProgressAsync)
{
throw new OperationCanceledException();
}
if (m_workerSource.IsCancellationRequested)
{
throw new OperationCanceledException();
}
for (var i = 0; i < m_options.NumberOfRetries; i++)
{
if (m_options.RetryDelay.Ticks != 0 && i != 0)
{
await Task.Delay(m_options.RetryDelay).ConfigureAwait(false);
}
if (!await m_taskreader.TransferProgressAsync)
{
throw new OperationCanceledException();
}
if (m_workerSource.IsCancellationRequested)
{
throw new OperationCanceledException();
}
try
{
if (m_backend == null)
{
m_backend = DynamicLoader.BackendLoader.GetBackend(m_backendurl, m_options.RawOptions);
}
if (m_backend == null)
{
throw new Exception("Backend failed to re-load");
}
var r = await method().ConfigureAwait(false);
return(r);
}
catch (Exception ex)
{
item.IsRetry = true;
lastException = ex;
Logging.Log.WriteRetryMessage(LOGTAG, $"Retry{item.Operation}", ex, "Operation {0} with file {1} attempt {2} of {3} failed with message: {4}", item.Operation, item.RemoteFilename, i + 1, m_options.NumberOfRetries, ex.Message);
// If the thread is aborted, we exit here
if (ex is System.Threading.ThreadAbortException || ex is OperationCanceledException)
{
break;
}
await m_stats.SendEventAsync(item.Operation, i < m_options.NumberOfRetries?BackendEventType.Retrying : BackendEventType.Failed, item.RemoteFilename, item.Size);
bool recovered = false;
if (!m_uploadSuccess && ex is Duplicati.Library.Interface.FolderMissingException && m_options.AutocreateFolders)
{
try
{
// If we successfully create the folder, we can re-use the connection
m_backend.CreateFolder();
recovered = true;
}
catch (Exception dex)
{
Logging.Log.WriteWarningMessage(LOGTAG, "FolderCreateError", dex, "Failed to create folder: {0}", ex.Message);
}
}
if (!recovered)
{
ResetBackend(ex);
}
}
finally
{
if (m_options.NoConnectionReuse)
{
ResetBackend(null);
}
}
}
throw lastException;
}