private async void validarTodos()
{
if (Utility.AccesoInternet())
{
progressBar1.Visible = true;
try
{
var progress = new System.Progress <ProgressReport>();
progress.ProgressChanged += (o, report) => {
progressBar1.Value = report.PorcentComplete;
progressBar1.Update();
};
await ProcessData(dtgvDetalleFactura, progress);
}
catch (Exception)
{
}
cargarDatos();
progressBar1.Visible = false;
}
else
{
MessageBox.Show("No hay acceso a internet, no se validarán los documentos", "Sin Internet", MessageBoxButtons.OK, MessageBoxIcon.Error);
}
progressBar1.Visible = false;
}
public string Download()
{
var zipFile = Path.Combine(_binariesDir, _filename);
// check if zip file exists in the destination folder
// return the file path and don't require to download again
if (File.Exists(zipFile))
{
return(zipFile);
}
var progress = new System.Progress <double>();
progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value);
try
{
// download the file
var cs = new CancellationTokenSource();
Utils.DownloadAsync(_config.DownloadUrl, zipFile, progress, cs.Token).Wait();
}
catch (Exception ex)
{
throw new Exception($"Failed to download {_config.DownloadUrl}", ex);
}
return(zipFile);
}
public string Download()
{
var zipFile = GetOutputPath();
// check if zip file exists in the destination folder
// return the file path and don't require to download again
if (File.Exists(zipFile))
{
return(zipFile);
}
var cs = new CancellationTokenSource();
var url = GetNugetUri(cs.Token).Result;
if (url.IsFile)
{
// Use the global package folder
return(ExtractContent(url.LocalPath, zipFile));
}
else
{
// Download from the nuget repository
var downloadPath = Path.Combine(_destDir, $@"PostgreSql.Binaries.Lite.{_pgVersion}.nupkg");
var progress = new System.Progress <double>();
progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value);
Utils.DownloadAsync(url.AbsoluteUri, downloadPath, progress, cs.Token).Wait();
return(ExtractContent(downloadPath, zipFile));
}
}
public string Download()
{
var versionParts = _pgVersion.Split('.');
string zipFilename = "";
if (versionParts.Length > 3)
{
zipFilename = string.Format(FILE_NAME, $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}-{versionParts[3]}");
}
else
{
zipFilename = string.Format(FILE_NAME, $"{versionParts[0]}.{versionParts[1]}-{versionParts[2]}");
}
var zipFile = Path.Combine(_destDir, zipFilename);
// check if zip file exists in the destination folder
// return the file path and don't require to download again
if (File.Exists(zipFile))
{
return(zipFile);
}
// first step is to download the nupkg file
var url = string.Format(NUGET_URL, _pgVersion);
var nupkgFile = Path.Combine(_destDir, $@"PostgreSql.Binaries.Lite.{_pgVersion}.nupkg");
var progress = new System.Progress <double>();
progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value);
// download the file
var cs = new CancellationTokenSource();
Utils.DownloadAsync(url, nupkgFile, progress, cs.Token).Wait();
// extract the PG binary zip file
using (var archive = ZipFile.OpenRead(nupkgFile))
{
var result = from entry in archive.Entries
where Path.GetDirectoryName(entry.FullName) == "content"
where !string.IsNullOrEmpty(entry.Name)
select entry;
var pgBinaryZipFile = result.FirstOrDefault();
if (pgBinaryZipFile != null)
{
pgBinaryZipFile.ExtractToFile(zipFile, true);
return(zipFile);
}
return(string.Empty);
}
}
private async Task Install_Download()
{
CanCancel = true;
try
{
// actually start downloading
string downloadUrl = string.Empty;
CalculatedFileSize = 0;
Progress <long> progress = new System.Progress <long>();
string progressTitle = GetResourceString("Installer_ProgressBar_Downloading");
progress.ProgressChanged += (sender, e) => UpdateProgressBar(progressTitle, e, CalculatedFileSize, false);
var release_page_url = (IsBeta ? GithubAPI.BETA_URL : GithubAPI.RELEASE_URL);
var httpRequest = (HttpWebRequest)WebRequest.Create(release_page_url);
httpRequest.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";
httpRequest.Headers["Authorization"] = "Bearer " + GithubAPI.ACCESS_TOKEN;
var httpResponse = (HttpWebResponse)httpRequest.GetResponse();
using (var streamReader = new StreamReader(httpResponse.GetResponseStream()))
{
var contentsJson = streamReader.ReadToEnd();
var contents = JsonConvert.DeserializeObject <UpdateNote>(contentsJson);
downloadUrl = contents.assets[0].url;
CalculatedFileSize = contents.assets[0].size;
}
var file = System.IO.Path.Combine(this.Path, "build.zip");
var httpRequest2 = (HttpWebRequest)WebRequest.Create(downloadUrl);
httpRequest2.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36";
httpRequest2.Headers["Authorization"] = "Bearer " + GithubAPI.ACCESS_TOKEN;
httpRequest2.Accept = "application/octet-stream";
using (Stream output = System.IO.File.OpenWrite(file))
using (WebResponse response = httpRequest2.GetResponse())
using (Stream stream = response.GetResponseStream())
{
DownloadTask = stream.CopyToAsync(output, progress, ICancel.Token, 81920);
await DownloadTask;
}
}
catch (Exception e)
{
if (ICancel.IsCancellationRequested)
{
return;
}
if (e != null)
{
Console.WriteLine(e.ToString());
MessageBox.Show(GetResourceString("Installer_Dialog_ErrorOccured_Text") + "\n" + e.ToString());
Install_Cancel();
return;
}
}
CanCancel = false;
}
public async Task <int> EnsureServerSync(Action <int> setProgress, bool syncOldData)
{
var progress = new System.Progress <int>();
progress.ProgressChanged += (sender, e) => { setProgress(e); };
var asIProgress = progress as IProgress <int>;
if (isRunning == 1)
{
return(0);
}
isRunning = 1;
await Task.Run(async() => await doServerSync(asIProgress.Report, syncOldData)
);
isRunning = 0;
return(0);
}
private async void StartButton_Click(object sender, RoutedEventArgs e)
{
SetState(false);
SaveFileDialog saveFile = new SaveFileDialog()
{
Filter = "XLSX|*.xlsx",
Title = "Укажите путь до Excel-документа"
};
if (saveFile.ShowDialog() != System.Windows.Forms.DialogResult.OK)
{
SetState(true);
return;
}
progressBar.Value = 0;
progressBar.Visibility = Visibility.Visible;
Progress <Tuple <int, int> > progress = new System.Progress <Tuple <int, int> >((x) =>
{
progressBar.Maximum = x.Item2;
progressBar.Value = x.Item1;
});
try
{
bool result = await DataImporter.ImportDataAsync(textBox.Text, saveFile.FileName, progress);
if (result)
{
new MessageWindow("Успех", "Данные успешно записаны").ShowDialog();
}
else
{
new MessageWindow("Неудача", "Не удалось записать данные").ShowDialog();
}
}
catch (Exception ex)
{
new MessageWindow("Успех", ex.Message).ShowDialog();
}
progressBar.Visibility = Visibility.Hidden;
SetState(true);
}
public static async void Download(string downloadUrl, string filePath, string fileName)
{
var httpclient = HttpDownloadHelper.HttpClient;
Progress <float> progress = new System.Progress <float>(x => NotifyProgress(x));
//progress.ProgressChanged += Progress_ProgressChanged;
// Create a file stream to store the downloaded data. This really can be any type of writeable stream.
string filePathTemp = filePath + "//" + fileName;
if (File.Exists(filePathTemp))
{
File.Delete(filePathTemp);
}
using (var file = new FileStream(filePathTemp, FileMode.Create, FileAccess.Write, FileShare.None))
{
// Use the custom extension method below to download the data. The passed progress-instance will receive the download status updates.
await httpclient.DownloadAsync(downloadUrl, file, progress, source.Token);
}
}
private async void button_Click(object sender, RoutedEventArgs e)
{
button.IsEnabled = false;
//warning
MessageBox.Show("Next you'll be asked to choose a folder. Keep in mind that if you choose \"C:\\LOL_4.20\" the result will be \"C:\\LOL_4.20\\League of Legends\"");
//Select folder
path = ShowFolderBrowserDialog();
//Start the download
label1.Content = "Stats: Downloading ...";
var client = new MegaApiClient();
client.LoginAnonymous();
Progress <double> ze = new System.Progress <double>(p => UpdateStatus(p));
if (File.Exists(appData + "\\ze.rar"))
{
File.Delete(appData + "\\ze.rar");
}
await client.DownloadFileAsync(new Uri("https://mega.nz/#!pFRVxBJQ!AMbsJnS9kqhvQ-tfP8QxoBikbrjlGQ4MdzNYGo0fIKM"), appData + "\\ze.rar", ze); //smaller test file https://mega.nz/#!J90AhTAI!Piq-76v6tB6l6W2HexqoN9XU8qvGdBJ6CONFMEyCPqE
StartExtracting();
}
/// <summary>
/// Performs the bulk copy and tracks the progress
/// </summary>
/// <param name="destinationFolderPath">Destination folder path</param>
/// <param name="accountName">Account name</param>
/// <param name="sourcePath">Source folder or file path</param>
/// <param name="cmdletCancellationToken">Commandlet cancellation token</param>
/// <param name="fileThreadCount">PerFileThreadCount</param>
/// <param name="recursive">If true then Enumeration of the subdirectories are done recursively</param>
/// <param name="overwrite">True if we want to overwrite existing destinations</param>
/// <param name="resume">Indicates that the file(s) being copied are a continuation of a previous file transfer</param>
/// <param name="isDownload">True if it is download else upload</param>
/// <param name="cmdletRunningRequest">Current Commandlet</param>
/// <param name="isBinary">Indicates that the file(s) being uploaded should be copied with no concern for new line preservation across appends</param>
public void BulkCopy(string destinationFolderPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken,
int fileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool isDownload = false,
Cmdlet cmdletRunningRequest = null, bool isBinary = false)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
var progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Enumerating the source and preparing the copy.",
sourcePath, recursive ? " recursively" : string.Empty), "Copy in progress...")
{
PercentComplete = 0
};
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <TransferStatus>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * (e.ChunksTransfered + e.FilesTransfered + e.DirectoriesTransferred) / (e.TotalChunksToTransfer + e.TotalFilesToTransfer + e.TotalDirectoriesToTransfer) * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
progress.Activity =
$"Copying Folder: {sourcePath}{(recursive ? " recursively" : string.Empty)}. Bytes remaining: {e.TotalSizeToTransfer - e.SizeTransfered}{(e.TotalChunksToTransfer > 0 ? $", Chunks remaining: {e.TotalChunksToTransfer - e.ChunksTransfered}" : "")}{(e.TotalNonChunkedFileToTransfer > 0 ? $", Non-chunked files remaining: {e.TotalNonChunkedFileToTransfer - e.NonChunkedFileTransferred}" : "")}" +
$"{(e.TotalDirectoriesToTransfer > 0 ? $", Directories remaining: {e.TotalDirectoriesToTransfer - e.DirectoriesTransferred}" : "")}.";
}
};
TransferStatus status = null;
Task transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
if (isDownload)
{
status = client.BulkDownload(sourcePath, destinationFolderPath, fileThreadCount,
overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, cmdletCancellationToken);
}
else
{
status = client.BulkUpload(sourcePath, destinationFolderPath, fileThreadCount,
overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, isBinary, cmdletCancellationToken);
}
}, cmdletCancellationToken);
TrackTaskProgress(transferTask, cmdletRunningRequest, progress, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
if (status != null && cmdletRunningRequest != null)
{
foreach (var failedEntry in status.EntriesFailed)
{
cmdletRunningRequest.WriteObject($"FailedTransfer: {failedEntry.EntryName} {failedEntry.Errors}");
}
}
}
}
/// <summary>
/// Changes Acl recursively
/// </summary>
/// <param name="path">Input path</param>
/// <param name="accountName">Account name</param>
/// <param name="aclToSet">List of acl to set</param>
/// <param name="aclChangeType">Type of al change- Modify, Set, Remove</param>
/// <param name="concurrency">Concurrency- number of parallel operations</param>
/// <param name="aclCmdlet">Cmdlet for acl change. This is only for printing progress. If passed null, then no progress tracking is done</param>
/// <param name="trackProgress"></param>
/// <param name="cmdletCancellationToken">Cancellationtoken for cmdlet</param>
public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet,
RequestedAclType aclChangeType, int concurrency, Cmdlet aclCmdlet, bool trackProgress, CancellationToken cmdletCancellationToken)
{
var client = AdlsClientFactory.GetAdlsClient(accountName, _context);
// Currently mockadlsclient signature is different, once that gets fixed, we can remove this
if (client.GetType() != typeof(MockAdlsClient))
{
System.Progress <AclProcessorStats> progressTracker = null;
ProgressRecord progress = null;
// If passing null, then we do not want progreess tracking
if (trackProgress)
{
progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000),
string.Format($"Recursive acl change for path {path}"),
$"Type of Acl Change: {aclChangeType}")
{
PercentComplete = 0
};
// On update from the Data Lake store uploader, capture the progress.
progressTracker = new System.Progress <AclProcessorStats>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = 0;
progress.Activity =
$"Files enumerated: {e.FilesProcessed} Directories enumerated:{e.DirectoryProcessed}";
}
};
}
AclProcessorStats status = null;
Task aclTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
status = client.ChangeAcl(path, aclToSet, aclChangeType, concurrency, progressTracker,
cmdletCancellationToken);
}, cmdletCancellationToken);
if (trackProgress || _isDebugEnabled)
{
TrackTaskProgress(aclTask, aclCmdlet, progress, cmdletCancellationToken);
if (trackProgress && !cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, aclCmdlet);
}
}
else
{
WaitForTask(aclTask, cmdletCancellationToken);
}
return(status);
}
return(client.ChangeAcl(path, aclToSet, aclChangeType, concurrency));
}
public void CopyFile(string destinationPath, string accountName, string sourcePath,
CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false,
bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null)
{
FileType ignoredType;
if (!overwrite && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType))
{
throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath));
}
//TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in
// default the number of threads to use to the processor count
if (threadCount < 1)
{
threadCount = Environment.ProcessorCount;
}
// Progress bar indicator.
var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}",
sourcePath, destinationPath, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
"Upload to DataLakeStore Store",
description)
{
PercentComplete = 0
};
if (parentProgress != null)
{
progress.ParentActivityId = parentProgress.ActivityId;
}
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress<UploadProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100);
}
};
var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount,
overwrite, resume, isBinary);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
progressTracker);
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.Expect100Continue = false;
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
}
finally
{
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyFile(string destinationPath, string accountName, string sourcePath,
CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false,
bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null)
{
var originalValue = TracingAdapter.IsEnabled;
try
{
//TODO: Remove this logic when defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) is resolved
TracingAdapter.IsEnabled = false;
// default the number of threads to use to the processor count
if (threadCount < 1)
{
threadCount = Environment.ProcessorCount;
}
// Progress bar indicator.
var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}",
sourcePath, destinationPath, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
"Upload to DataLakeStore Store",
description)
{
PercentComplete = 0
};
if (parentProgress != null)
{
progress.ParentActivityId = parentProgress.ActivityId;
}
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress<UploadProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100);
}
};
var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount,
overwrite, resume, isBinary);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
}
finally
{
TracingAdapter.IsEnabled = originalValue;
}
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int concurrentFileCount = -1,
int perFileThreadCount = -1,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
bool isDownload = false,
Cmdlet cmdletRunningRequest = null)
{
long totalBytes;
long totalFiles;
if (isDownload)
{
var summary = _client.FileSystem.GetContentSummary(accountName, sourceFolderPath);
totalBytes = summary.ContentSummary.SpaceConsumed.GetValueOrDefault();
totalFiles = summary.ContentSummary.FileCount.GetValueOrDefault();
}
else
{
totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, accountName);
totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, accountName);
}
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{
PercentComplete = 0
};
UpdateProgress(progress, cmdletRunningRequest);
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
// Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively.
ServicePointManager.Expect100Continue = false;
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <TransferFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * e.TransferredByteCount / e.TotalFileLength * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.TransferredByteCount, e.TotalFileCount - e.TransferredFileCount);
}
};
Task transferTask;
if (isDownload)
{
transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
_client.FileSystem.DownloadFolder(
accountName,
sourceFolderPath,
destinationFolderPath,
perFileThreadCount,
concurrentFileCount,
resume,
overwrite,
recursive,
progressTracker,
cmdletCancellationToken);
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
}
else
{
transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
_client.FileSystem.UploadFolder(
accountName,
sourceFolderPath,
destinationFolderPath,
perFileThreadCount,
concurrentFileCount,
resume,
overwrite,
isBinary,
recursive,
progressTracker,
cmdletCancellationToken);
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
}
TrackUploadProgress(transferTask, progress, cmdletRunningRequest, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyFile(string destinationPath, string accountName, string sourcePath,
CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false,
bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null)
{
// Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively.
FileType ignoredType;
if (!overwrite && (!isDownload && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType) || (isDownload && File.Exists(destinationPath))))
{
throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath));
}
if (threadCount < 1)
{
threadCount = 10; // 10 is the default per our documentation.
}
// Progress bar indicator.
var description = string.Format("Copying {0} File: {1} {2} Location: {3} for account: {4}",
isDownload ? "Data Lake Store" : "Local",
sourcePath,
isDownload ? "to local" : "to Data Lake Store",
destinationPath, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("{0} Data Lake Store Store", isDownload ? "Download from" : "Upload to"),
description)
{
PercentComplete = 0
};
if (parentProgress != null)
{
progress.ParentActivityId = parentProgress.ActivityId;
}
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <TransferProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * e.TransferredByteCount / e.TotalFileLength * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
}
};
Task transferTask;
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.Expect100Continue = false;
if (isDownload)
{
transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
_client.FileSystem.DownloadFile(
accountName,
sourcePath,
destinationPath,
threadCount,
resume,
overwrite,
progressTracker,
cmdletCancellationToken);
cmdletCancellationToken.ThrowIfCancellationRequested();
},
cmdletCancellationToken);
}
else
{
transferTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
_client.FileSystem.UploadFile(
accountName,
sourcePath,
destinationPath,
threadCount,
resume,
overwrite,
isBinary,
progressTracker,
cmdletCancellationToken);
cmdletCancellationToken.ThrowIfCancellationRequested();
},
cmdletCancellationToken);
}
TrackUploadProgress(transferTask, progress, cmdletRunningRequest, cmdletCancellationToken);
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int concurrentFileCount = 5,
int perFileThreadCount = 10,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
bool isDownload = false,
Cmdlet cmdletRunningRequest = null)
{
var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, isDownload, accountName);
var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, isDownload, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{ PercentComplete = 0 };
UpdateProgress(progress, cmdletRunningRequest);
var internalFolderThreads = concurrentFileCount <= 0 ? 5 : concurrentFileCount;
var internalFileThreads = perFileThreadCount <= 0 ? 10 : perFileThreadCount;
// we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
var previousExpect100 = ServicePointManager.Expect100Continue;
var previousTracing = ServiceClientTracing.IsEnabled;
try
{
// disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved.
ServiceClientTracing.IsEnabled = false;
ServicePointManager.DefaultConnectionLimit =
Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads,
ServicePointManager.DefaultConnectionLimit);
ServicePointManager.Expect100Continue = false;
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress<UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount);
}
};
var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads,
isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive, isDownload: isDownload);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
folderProgressTracker: progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServiceClientTracing.IsEnabled = previousTracing;
ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit;
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public static bool UploadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false, bool testCancel = false)
{
var cancelSource = new CancellationTokenSource();
var myToken = cancelSource.Token;
var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive);
var progressTracker = new System.Progress <UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
if (e.TotalFileCount == 0)
{
Console.WriteLine("we are done!");
}
};
var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient, myToken);
var uploader = new DataLakeStoreUploader(parameters, frontend, myToken, folderProgressTracker: progressTracker);
if (testCancel)
{
var uploadTask = Task.Run(() =>
{
myToken.ThrowIfCancellationRequested();
uploader.Execute();
myToken.ThrowIfCancellationRequested();
}, myToken);
try
{
while (!uploadTask.IsCompleted && !uploadTask.IsCanceled)
{
if (myToken.IsCancellationRequested)
{
// we are done tracking progress and will just break and let the task clean itself up.
try
{
uploadTask.Wait();
}
catch (OperationCanceledException)
{
if (uploadTask.IsCanceled)
{
uploadTask.Dispose();
}
}
catch (AggregateException ex)
{
if (ex.InnerExceptions.OfType <OperationCanceledException>().Any())
{
if (uploadTask.IsCanceled)
{
uploadTask.Dispose();
}
}
else
{
throw;
}
}
catch (Exception ex)
{
// swallow this for debugging to see what it is.
}
break;
}
Thread.Sleep(60000);
// run for 60 seconds and then cancel out and see what happens
cancelSource.Cancel();
}
}
catch (OperationCanceledException)
{
// do nothing since we successfully cancelled out
}
catch (Exception ex)
{
// see what the heck is going on.
}
}
else
{
uploader.Execute();
}
return(true);
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int folderThreadCount = -1,
int perFileThreadCount = -1,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
Cmdlet cmdletRunningRequest = null)
{
var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive);
var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{
PercentComplete = 0
};
UpdateProgress(progress, cmdletRunningRequest);
var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount;
var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount;
// we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.DefaultConnectionLimit =
Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads,
ServicePointManager.DefaultConnectionLimit);
ServicePointManager.Expect100Continue = false;
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount);
}
};
var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads,
isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
folderProgressTracker: progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit;
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyFile(string destinationPath, string accountName, string sourcePath,
CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false,
bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null)
{
var originalValue = TracingAdapter.IsEnabled;
try
{
//TODO: Remove this logic when defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) is resolved
TracingAdapter.IsEnabled = false;
// default the number of threads to use to the processor count
if (threadCount < 1)
{
threadCount = Environment.ProcessorCount;
}
// Progress bar indicator.
var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}",
sourcePath, destinationPath, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
"Upload to DataLakeStore Store",
description)
{
PercentComplete = 0
};
if (parentProgress != null)
{
progress.ParentActivityId = parentProgress.ActivityId;
}
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <UploadProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
}
};
var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount,
overwrite, resume, isBinary);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
}
finally
{
TracingAdapter.IsEnabled = originalValue;
}
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int concurrentFileCount = 5,
int perFileThreadCount = 10,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
bool isDownload = false,
Cmdlet cmdletRunningRequest = null)
{
var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, isDownload, accountName);
var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, isDownload, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{
PercentComplete = 0
};
UpdateProgress(progress, cmdletRunningRequest);
var internalFolderThreads = concurrentFileCount <= 0 ? 5 : concurrentFileCount;
var internalFileThreads = perFileThreadCount <= 0 ? 10 : perFileThreadCount;
// we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
// Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively.
ServicePointManager.DefaultConnectionLimit =
Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads,
ServicePointManager.DefaultConnectionLimit);
ServicePointManager.Expect100Continue = false;
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress <UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount);
}
};
var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads,
isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive, isDownload: isDownload);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
folderProgressTracker: progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit;
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyDirectory(
string destinationFolderPath,
string accountName,
string sourceFolderPath,
CancellationToken cmdletCancellationToken,
int folderThreadCount = -1,
int perFileThreadCount = -1,
bool recursive = false,
bool overwrite = false,
bool resume = false,
bool forceBinaryOrText = false,
bool isBinary = false,
Cmdlet cmdletRunningRequest = null)
{
var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive);
var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles),
"Copy in progress...")
{ PercentComplete = 0 };
UpdateProgress(progress, cmdletRunningRequest);
var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount;
var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount;
// we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want)
var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit;
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.DefaultConnectionLimit =
Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads,
ServicePointManager.DefaultConnectionLimit);
ServicePointManager.Expect100Continue = false;
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress<UploadFolderProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}",
sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount);
}
};
var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads,
isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
folderProgressTracker: progressTracker);
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
if (!cmdletCancellationToken.IsCancellationRequested)
{
progress.PercentComplete = 100;
progress.RecordType = ProgressRecordType.Completed;
UpdateProgress(progress, cmdletRunningRequest);
}
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit;
ServicePointManager.Expect100Continue = previousExpect100;
}
}
public void CopyFile(string destinationPath, string accountName, string sourcePath,
CancellationToken cmdletCancellationToken, int threadCount = 10, bool overwrite = false, bool resume = false,
bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null)
{
var previousTracing = ServiceClientTracing.IsEnabled;
try
{
// disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved.
ServiceClientTracing.IsEnabled = false;
FileType ignoredType;
if (!overwrite && (!isDownload && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType) || (isDownload && File.Exists(destinationPath))))
{
throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath));
}
if (threadCount < 1)
{
threadCount = 10; // 10 is the default per our documentation.
}
// Progress bar indicator.
var description = string.Format("Copying {0} File: {1} {2} Location: {3} for account: {4}",
isDownload ? "Data Lake Store" : "Local",
sourcePath,
isDownload ? "to local" : "to Data Lake Store",
destinationPath, accountName);
var progress = new ProgressRecord(
uniqueActivityIdGenerator.Next(0, 10000000),
string.Format("{0} Data Lake Store Store", isDownload ? "Download from" : "Upload to"),
description)
{
PercentComplete = 0
};
if (parentProgress != null)
{
progress.ParentActivityId = parentProgress.ActivityId;
}
// On update from the Data Lake store uploader, capture the progress.
var progressTracker = new System.Progress<UploadProgress>();
progressTracker.ProgressChanged += (s, e) =>
{
lock (ConsoleOutputLock)
{
var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100);
// powershell defect protection. If, through some defect in
// our progress tracking, the number is outside of 0 - 100,
// powershell will crash if it is set to that value. Instead
// just keep the value unchanged in that case.
if (toSet < 0 || toSet > 100)
{
progress.PercentComplete = progress.PercentComplete;
}
else
{
progress.PercentComplete = toSet;
}
}
};
var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount,
isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isDownload: isDownload);
var uploader = new DataLakeStoreUploader(uploadParameters,
new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken),
cmdletCancellationToken,
progressTracker);
var previousExpect100 = ServicePointManager.Expect100Continue;
try
{
ServicePointManager.Expect100Continue = false;
// Execute the uploader.
var uploadTask = Task.Run(() =>
{
cmdletCancellationToken.ThrowIfCancellationRequested();
uploader.Execute();
cmdletCancellationToken.ThrowIfCancellationRequested();
}, cmdletCancellationToken);
TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken);
}
catch (Exception e)
{
throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e));
}
finally
{
ServicePointManager.Expect100Continue = previousExpect100;
}
}
finally
{
ServiceClientTracing.IsEnabled = previousTracing;
}
}
