private async void validarTodos() { if (Utility.AccesoInternet()) { progressBar1.Visible = true; try { var progress = new System.Progress <ProgressReport>(); progress.ProgressChanged += (o, report) => { progressBar1.Value = report.PorcentComplete; progressBar1.Update(); }; await ProcessData(dtgvDetalleFactura, progress); } catch (Exception) { } cargarDatos(); progressBar1.Visible = false; } else { MessageBox.Show("No hay acceso a internet, no se validarán los documentos", "Sin Internet", MessageBoxButtons.OK, MessageBoxIcon.Error); } progressBar1.Visible = false; }
public string Download() { var zipFile = Path.Combine(_binariesDir, _filename); // check if zip file exists in the destination folder // return the file path and don't require to download again if (File.Exists(zipFile)) { return(zipFile); } var progress = new System.Progress <double>(); progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value); try { // download the file var cs = new CancellationTokenSource(); Utils.DownloadAsync(_config.DownloadUrl, zipFile, progress, cs.Token).Wait(); } catch (Exception ex) { throw new Exception($"Failed to download {_config.DownloadUrl}", ex); } return(zipFile); }
public string Download() { var zipFile = GetOutputPath(); // check if zip file exists in the destination folder // return the file path and don't require to download again if (File.Exists(zipFile)) { return(zipFile); } var cs = new CancellationTokenSource(); var url = GetNugetUri(cs.Token).Result; if (url.IsFile) { // Use the global package folder return(ExtractContent(url.LocalPath, zipFile)); } else { // Download from the nuget repository var downloadPath = Path.Combine(_destDir, $@"PostgreSql.Binaries.Lite.{_pgVersion}.nupkg"); var progress = new System.Progress <double>(); progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value); Utils.DownloadAsync(url.AbsoluteUri, downloadPath, progress, cs.Token).Wait(); return(ExtractContent(downloadPath, zipFile)); } }
public string Download() { var versionParts = _pgVersion.Split('.'); string zipFilename = ""; if (versionParts.Length > 3) { zipFilename = string.Format(FILE_NAME, $"{versionParts[0]}.{versionParts[1]}.{versionParts[2]}-{versionParts[3]}"); } else { zipFilename = string.Format(FILE_NAME, $"{versionParts[0]}.{versionParts[1]}-{versionParts[2]}"); } var zipFile = Path.Combine(_destDir, zipFilename); // check if zip file exists in the destination folder // return the file path and don't require to download again if (File.Exists(zipFile)) { return(zipFile); } // first step is to download the nupkg file var url = string.Format(NUGET_URL, _pgVersion); var nupkgFile = Path.Combine(_destDir, $@"PostgreSql.Binaries.Lite.{_pgVersion}.nupkg"); var progress = new System.Progress <double>(); progress.ProgressChanged += (sender, value) => Console.WriteLine("\r %{0:N0}", value); // download the file var cs = new CancellationTokenSource(); Utils.DownloadAsync(url, nupkgFile, progress, cs.Token).Wait(); // extract the PG binary zip file using (var archive = ZipFile.OpenRead(nupkgFile)) { var result = from entry in archive.Entries where Path.GetDirectoryName(entry.FullName) == "content" where !string.IsNullOrEmpty(entry.Name) select entry; var pgBinaryZipFile = result.FirstOrDefault(); if (pgBinaryZipFile != null) { pgBinaryZipFile.ExtractToFile(zipFile, true); return(zipFile); } return(string.Empty); } }
private async Task Install_Download() { CanCancel = true; try { // actually start downloading string downloadUrl = string.Empty; CalculatedFileSize = 0; Progress <long> progress = new System.Progress <long>(); string progressTitle = GetResourceString("Installer_ProgressBar_Downloading"); progress.ProgressChanged += (sender, e) => UpdateProgressBar(progressTitle, e, CalculatedFileSize, false); var release_page_url = (IsBeta ? GithubAPI.BETA_URL : GithubAPI.RELEASE_URL); var httpRequest = (HttpWebRequest)WebRequest.Create(release_page_url); httpRequest.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"; httpRequest.Headers["Authorization"] = "Bearer " + GithubAPI.ACCESS_TOKEN; var httpResponse = (HttpWebResponse)httpRequest.GetResponse(); using (var streamReader = new StreamReader(httpResponse.GetResponseStream())) { var contentsJson = streamReader.ReadToEnd(); var contents = JsonConvert.DeserializeObject <UpdateNote>(contentsJson); downloadUrl = contents.assets[0].url; CalculatedFileSize = contents.assets[0].size; } var file = System.IO.Path.Combine(this.Path, "build.zip"); var httpRequest2 = (HttpWebRequest)WebRequest.Create(downloadUrl); httpRequest2.UserAgent = @"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36"; httpRequest2.Headers["Authorization"] = "Bearer " + GithubAPI.ACCESS_TOKEN; httpRequest2.Accept = "application/octet-stream"; using (Stream output = System.IO.File.OpenWrite(file)) using (WebResponse response = httpRequest2.GetResponse()) using (Stream stream = response.GetResponseStream()) { DownloadTask = stream.CopyToAsync(output, progress, ICancel.Token, 81920); await DownloadTask; } } catch (Exception e) { if (ICancel.IsCancellationRequested) { return; } if (e != null) { Console.WriteLine(e.ToString()); MessageBox.Show(GetResourceString("Installer_Dialog_ErrorOccured_Text") + "\n" + e.ToString()); Install_Cancel(); return; } } CanCancel = false; }
public async Task <int> EnsureServerSync(Action <int> setProgress, bool syncOldData) { var progress = new System.Progress <int>(); progress.ProgressChanged += (sender, e) => { setProgress(e); }; var asIProgress = progress as IProgress <int>; if (isRunning == 1) { return(0); } isRunning = 1; await Task.Run(async() => await doServerSync(asIProgress.Report, syncOldData) ); isRunning = 0; return(0); }
private async void StartButton_Click(object sender, RoutedEventArgs e) { SetState(false); SaveFileDialog saveFile = new SaveFileDialog() { Filter = "XLSX|*.xlsx", Title = "Укажите путь до Excel-документа" }; if (saveFile.ShowDialog() != System.Windows.Forms.DialogResult.OK) { SetState(true); return; } progressBar.Value = 0; progressBar.Visibility = Visibility.Visible; Progress <Tuple <int, int> > progress = new System.Progress <Tuple <int, int> >((x) => { progressBar.Maximum = x.Item2; progressBar.Value = x.Item1; }); try { bool result = await DataImporter.ImportDataAsync(textBox.Text, saveFile.FileName, progress); if (result) { new MessageWindow("Успех", "Данные успешно записаны").ShowDialog(); } else { new MessageWindow("Неудача", "Не удалось записать данные").ShowDialog(); } } catch (Exception ex) { new MessageWindow("Успех", ex.Message).ShowDialog(); } progressBar.Visibility = Visibility.Hidden; SetState(true); }
public static async void Download(string downloadUrl, string filePath, string fileName) { var httpclient = HttpDownloadHelper.HttpClient; Progress <float> progress = new System.Progress <float>(x => NotifyProgress(x)); //progress.ProgressChanged += Progress_ProgressChanged; // Create a file stream to store the downloaded data. This really can be any type of writeable stream. string filePathTemp = filePath + "//" + fileName; if (File.Exists(filePathTemp)) { File.Delete(filePathTemp); } using (var file = new FileStream(filePathTemp, FileMode.Create, FileAccess.Write, FileShare.None)) { // Use the custom extension method below to download the data. The passed progress-instance will receive the download status updates. await httpclient.DownloadAsync(downloadUrl, file, progress, source.Token); } }
private async void button_Click(object sender, RoutedEventArgs e) { button.IsEnabled = false; //warning MessageBox.Show("Next you'll be asked to choose a folder. Keep in mind that if you choose \"C:\\LOL_4.20\" the result will be \"C:\\LOL_4.20\\League of Legends\""); //Select folder path = ShowFolderBrowserDialog(); //Start the download label1.Content = "Stats: Downloading ..."; var client = new MegaApiClient(); client.LoginAnonymous(); Progress <double> ze = new System.Progress <double>(p => UpdateStatus(p)); if (File.Exists(appData + "\\ze.rar")) { File.Delete(appData + "\\ze.rar"); } await client.DownloadFileAsync(new Uri("https://mega.nz/#!pFRVxBJQ!AMbsJnS9kqhvQ-tfP8QxoBikbrjlGQ4MdzNYGo0fIKM"), appData + "\\ze.rar", ze); //smaller test file https://mega.nz/#!J90AhTAI!Piq-76v6tB6l6W2HexqoN9XU8qvGdBJ6CONFMEyCPqE StartExtracting(); }
/// <summary> /// Performs the bulk copy and tracks the progress /// </summary> /// <param name="destinationFolderPath">Destination folder path</param> /// <param name="accountName">Account name</param> /// <param name="sourcePath">Source folder or file path</param> /// <param name="cmdletCancellationToken">Commandlet cancellation token</param> /// <param name="fileThreadCount">PerFileThreadCount</param> /// <param name="recursive">If true then Enumeration of the subdirectories are done recursively</param> /// <param name="overwrite">True if we want to overwrite existing destinations</param> /// <param name="resume">Indicates that the file(s) being copied are a continuation of a previous file transfer</param> /// <param name="isDownload">True if it is download else upload</param> /// <param name="cmdletRunningRequest">Current Commandlet</param> /// <param name="isBinary">Indicates that the file(s) being uploaded should be copied with no concern for new line preservation across appends</param> public void BulkCopy(string destinationFolderPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int fileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, bool isBinary = false) { var client = AdlsClientFactory.GetAdlsClient(accountName, _context); var progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Enumerating the source and preparing the copy.", sourcePath, recursive ? " recursively" : string.Empty), "Copy in progress...") { PercentComplete = 0 }; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <TransferStatus>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * (e.ChunksTransfered + e.FilesTransfered + e.DirectoriesTransferred) / (e.TotalChunksToTransfer + e.TotalFilesToTransfer + e.TotalDirectoriesToTransfer) * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } progress.Activity = $"Copying Folder: {sourcePath}{(recursive ? " recursively" : string.Empty)}. Bytes remaining: {e.TotalSizeToTransfer - e.SizeTransfered}{(e.TotalChunksToTransfer > 0 ? $", Chunks remaining: {e.TotalChunksToTransfer - e.ChunksTransfered}" : "")}{(e.TotalNonChunkedFileToTransfer > 0 ? $", Non-chunked files remaining: {e.TotalNonChunkedFileToTransfer - e.NonChunkedFileTransferred}" : "")}" + $"{(e.TotalDirectoriesToTransfer > 0 ? $", Directories remaining: {e.TotalDirectoriesToTransfer - e.DirectoriesTransferred}" : "")}."; } }; TransferStatus status = null; Task transferTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); if (isDownload) { status = client.BulkDownload(sourcePath, destinationFolderPath, fileThreadCount, overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, cmdletCancellationToken); } else { status = client.BulkUpload(sourcePath, destinationFolderPath, fileThreadCount, overwrite ? IfExists.Overwrite : IfExists.Fail, progressTracker, !recursive, resume, isBinary, cmdletCancellationToken); } }, cmdletCancellationToken); TrackTaskProgress(transferTask, cmdletRunningRequest, progress, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); if (status != null && cmdletRunningRequest != null) { foreach (var failedEntry in status.EntriesFailed) { cmdletRunningRequest.WriteObject($"FailedTransfer: {failedEntry.EntryName} {failedEntry.Errors}"); } } } }
/// <summary> /// Changes Acl recursively /// </summary> /// <param name="path">Input path</param> /// <param name="accountName">Account name</param> /// <param name="aclToSet">List of acl to set</param> /// <param name="aclChangeType">Type of al change- Modify, Set, Remove</param> /// <param name="concurrency">Concurrency- number of parallel operations</param> /// <param name="aclCmdlet">Cmdlet for acl change. This is only for printing progress. If passed null, then no progress tracking is done</param> /// <param name="trackProgress"></param> /// <param name="cmdletCancellationToken">Cancellationtoken for cmdlet</param> public AclProcessorStats ChangeAclRecursively(string path, string accountName, List <AclEntry> aclToSet, RequestedAclType aclChangeType, int concurrency, Cmdlet aclCmdlet, bool trackProgress, CancellationToken cmdletCancellationToken) { var client = AdlsClientFactory.GetAdlsClient(accountName, _context); // Currently mockadlsclient signature is different, once that gets fixed, we can remove this if (client.GetType() != typeof(MockAdlsClient)) { System.Progress <AclProcessorStats> progressTracker = null; ProgressRecord progress = null; // If passing null, then we do not want progreess tracking if (trackProgress) { progress = new ProgressRecord(_uniqueActivityIdGenerator.Next(0, 10000000), string.Format($"Recursive acl change for path {path}"), $"Type of Acl Change: {aclChangeType}") { PercentComplete = 0 }; // On update from the Data Lake store uploader, capture the progress. progressTracker = new System.Progress <AclProcessorStats>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = 0; progress.Activity = $"Files enumerated: {e.FilesProcessed} Directories enumerated:{e.DirectoryProcessed}"; } }; } AclProcessorStats status = null; Task aclTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); status = client.ChangeAcl(path, aclToSet, aclChangeType, concurrency, progressTracker, cmdletCancellationToken); }, cmdletCancellationToken); if (trackProgress || _isDebugEnabled) { TrackTaskProgress(aclTask, aclCmdlet, progress, cmdletCancellationToken); if (trackProgress && !cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, aclCmdlet); } } else { WaitForTask(aclTask, cmdletCancellationToken); } return(status); } return(client.ChangeAcl(path, aclToSet, aclChangeType, concurrency)); }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { FileType ignoredType; if (!overwrite && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType)) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } //TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { var originalValue = TracingAdapter.IsEnabled; try { //TODO: Remove this logic when defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) is resolved TracingAdapter.IsEnabled = false; // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { TracingAdapter.IsEnabled = originalValue; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int concurrentFileCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null) { long totalBytes; long totalFiles; if (isDownload) { var summary = _client.FileSystem.GetContentSummary(accountName, sourceFolderPath); totalBytes = summary.ContentSummary.SpaceConsumed.GetValueOrDefault(); totalFiles = summary.ContentSummary.FileCount.GetValueOrDefault(); } else { totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, accountName); totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, accountName); } var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var previousExpect100 = ServicePointManager.Expect100Continue; try { // Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively. ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <TransferFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.TransferredByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.TransferredByteCount, e.TotalFileCount - e.TransferredFileCount); } }; Task transferTask; if (isDownload) { transferTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); _client.FileSystem.DownloadFolder( accountName, sourceFolderPath, destinationFolderPath, perFileThreadCount, concurrentFileCount, resume, overwrite, recursive, progressTracker, cmdletCancellationToken); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); } else { transferTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); _client.FileSystem.UploadFolder( accountName, sourceFolderPath, destinationFolderPath, perFileThreadCount, concurrentFileCount, resume, overwrite, isBinary, recursive, progressTracker, cmdletCancellationToken); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); } TrackUploadProgress(transferTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { // Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively. FileType ignoredType; if (!overwrite && (!isDownload && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType) || (isDownload && File.Exists(destinationPath)))) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } if (threadCount < 1) { threadCount = 10; // 10 is the default per our documentation. } // Progress bar indicator. var description = string.Format("Copying {0} File: {1} {2} Location: {3} for account: {4}", isDownload ? "Data Lake Store" : "Local", sourcePath, isDownload ? "to local" : "to Data Lake Store", destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("{0} Data Lake Store Store", isDownload ? "Download from" : "Upload to"), description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <TransferProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.TransferredByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } } }; Task transferTask; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; if (isDownload) { transferTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); _client.FileSystem.DownloadFile( accountName, sourcePath, destinationPath, threadCount, resume, overwrite, progressTracker, cmdletCancellationToken); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); } else { transferTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); _client.FileSystem.UploadFile( accountName, sourcePath, destinationPath, threadCount, resume, overwrite, isBinary, progressTracker, cmdletCancellationToken); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); } TrackUploadProgress(transferTask, progress, cmdletRunningRequest, cmdletCancellationToken); } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int concurrentFileCount = 5, int perFileThreadCount = 10, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = concurrentFileCount <= 0 ? 5 : concurrentFileCount; var internalFileThreads = perFileThreadCount <= 0 ? 10 : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; var previousTracing = ServiceClientTracing.IsEnabled; try { // disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved. ServiceClientTracing.IsEnabled = false; ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive, isDownload: isDownload); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServiceClientTracing.IsEnabled = previousTracing; ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public static bool UploadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false, bool testCancel = false) { var cancelSource = new CancellationTokenSource(); var myToken = cancelSource.Token; var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive); var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { if (e.TotalFileCount == 0) { Console.WriteLine("we are done!"); } }; var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient, myToken); var uploader = new DataLakeStoreUploader(parameters, frontend, myToken, folderProgressTracker: progressTracker); if (testCancel) { var uploadTask = Task.Run(() => { myToken.ThrowIfCancellationRequested(); uploader.Execute(); myToken.ThrowIfCancellationRequested(); }, myToken); try { while (!uploadTask.IsCompleted && !uploadTask.IsCanceled) { if (myToken.IsCancellationRequested) { // we are done tracking progress and will just break and let the task clean itself up. try { uploadTask.Wait(); } catch (OperationCanceledException) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } catch (AggregateException ex) { if (ex.InnerExceptions.OfType <OperationCanceledException>().Any()) { if (uploadTask.IsCanceled) { uploadTask.Dispose(); } } else { throw; } } catch (Exception ex) { // swallow this for debugging to see what it is. } break; } Thread.Sleep(60000); // run for 60 seconds and then cancel out and see what happens cancelSource.Cancel(); } } catch (OperationCanceledException) { // do nothing since we successfully cancelled out } catch (Exception ex) { // see what the heck is going on. } } else { uploader.Execute(); } return(true); }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int folderThreadCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount; var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { var originalValue = TracingAdapter.IsEnabled; try { //TODO: Remove this logic when defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) is resolved TracingAdapter.IsEnabled = false; // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { TracingAdapter.IsEnabled = originalValue; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int concurrentFileCount = 5, int perFileThreadCount = 10, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive, isDownload, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = concurrentFileCount <= 0 ? 5 : concurrentFileCount; var internalFileThreads = perFileThreadCount <= 0 ? 10 : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { // Service client tracing is enabled, however issue: https://github.com/Azure/azure-powershell/issues/2499 is not yet resolved, so debug functionality can still potentially affect performance negatively. ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive, isDownload: isDownload); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int folderThreadCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount; var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = 10, bool overwrite = false, bool resume = false, bool isBinary = false, bool isDownload = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { var previousTracing = ServiceClientTracing.IsEnabled; try { // disable this due to performance issues during download until issue: https://github.com/Azure/azure-powershell/issues/2499 is resolved. ServiceClientTracing.IsEnabled = false; FileType ignoredType; if (!overwrite && (!isDownload && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType) || (isDownload && File.Exists(destinationPath)))) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } if (threadCount < 1) { threadCount = 10; // 10 is the default per our documentation. } // Progress bar indicator. var description = string.Format("Copying {0} File: {1} {2} Location: {3} for account: {4}", isDownload ? "Data Lake Store" : "Local", sourcePath, isDownload ? "to local" : "to Data Lake Store", destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("{0} Data Lake Store Store", isDownload ? "Download from" : "Upload to"), description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { var toSet = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); // powershell defect protection. If, through some defect in // our progress tracking, the number is outside of 0 - 100, // powershell will crash if it is set to that value. Instead // just keep the value unchanged in that case. if (toSet < 0 || toSet > 100) { progress.PercentComplete = progress.PercentComplete; } else { progress.PercentComplete = toSet; } } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isDownload: isDownload); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.Expect100Continue = previousExpect100; } } finally { ServiceClientTracing.IsEnabled = previousTracing; } }