public UploadParameters ValidateParameters() { BlobUri destinationUri; if (!BlobUri.TryParseUri(Destination, out destinationUri)) { throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); } BlobUri baseImageUri = null; if (this.BaseImageUriToPatch != null) { if (!BlobUri.TryParseUri(BaseImageUriToPatch, out baseImageUri)) { throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); } if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) { var message = String.Format(Resources.AddAzureVhdCommandSASUriNotSupportedInPatchMode, destinationUri.Uri); throw new ArgumentOutOfRangeException("Destination", message); } } var storageCredentialsFactory = CreateStorageCredentialsFactory(); var parameters = new UploadParameters(destinationUri, baseImageUri, LocalFilePath, OverWrite.IsPresent, NumberOfUploaderThreads) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) }; return(parameters); }
public BookUploadParameters(UploadParameters options) { Folder = options.Path; ExcludeMusic = options.ExcludeMusicAudio; PreserveThumbnails = options.PreserveThumbnails; ForceUpload = options.ForceUpload; }
public static int Handle(UploadParameters options) { // This task will be all the program does. We need to do enough setup so that // the upload code can work, then tear it down. Program.SetUpErrorHandling(); try { using (var applicationContainer = new ApplicationContainer()) { Program.SetUpLocalization(applicationContainer); Browser.SetUpXulRunner(); Browser.XulRunnerShutdown += Program.OnXulRunnerShutdown; LocalizationManager.SetUILanguage(Settings.Default.UserInterfaceLanguage, false); var transfer = new BookTransfer(new BloomParseClient(), ProjectContext.CreateBloomS3Client(), applicationContainer.BookThumbNailer, new BookDownloadStartingEvent()); // Since Bloom is not a normal console app, when run from a command line, the new command prompt // appears at once. The extra newlines here are attempting to separate this from our output. Console.WriteLine("\nstarting upload"); transfer.UploadFolder(options.Path, applicationContainer, options.ExcludeAudio); Console.WriteLine(("\nupload complete\n")); } return(0); } catch (Exception ex) { Debug.WriteLine(ex.Message); Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); return(1); } }
public void DataLakeUploader_UploadDownloadSingleSegment() { var frontEnd = new InMemoryFrontEnd(); var up = new UploadParameters( inputFilePath: _smallFilePath, targetStreamPath: "1", perFileThreadCount: ThreadCount, accountName: "foo", isResume: false, maxSegmentLength: 4 * 1024 * 1024, localMetadataLocation: Path.GetTempPath()); File.WriteAllBytes(_smallFilePath, _smallFileData); var uploader = new DataLakeStoreUploader(up, frontEnd); uploader.Execute(); VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData); up = new UploadParameters( inputFilePath: "1", targetStreamPath: _downloadFilePath, perFileThreadCount: ThreadCount, accountName: "foo", isResume: false, isOverwrite: true, isDownload: true, maxSegmentLength: 4 * 1024 * 1024, localMetadataLocation: Path.GetTempPath()); // now download uploader = new DataLakeStoreUploader(up, frontEnd); uploader.Execute(); VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData); }
public async Task <ResultData> UploadAsync(UploadParameters parameters) { return(await this.PostDataAsync(new SynologyPostParameters(this) { Version = 2, Additional = parameters })); }
public ResultData Upload(UploadParameters parameters) { return PostData(new SynologyPostParameters { Version = 2, Additional = parameters }); }
// Upload a file public static void UploadFile(string srcFilePath, string destFilePath, bool force = true) { var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force); var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient); var uploader = new DataLakeStoreUploader(parameters, frontend); uploader.Execute(); }
public ResultData Upload(UploadParameters parameters) { return(PostData(new SynologyPostParameters(this) { Version = 2, Additional = parameters })); }
public ResultData Upload(UploadParameters parameters) { return(PostData(new SynologyPostParameters { Version = 2, Method = "upload", Additional = parameters })); }
public static bool DownloadFile(DataLakeStoreFileSystemManagementClient dataLakeStoreFileSystemClient, string dlAccountName, string srcPath, string destPath, bool force = false, bool recursive = false) { var parameters = new UploadParameters(srcPath, destPath, dlAccountName, isOverwrite: force, isBinary: true, isDownload: true, perFileThreadCount: 40, concurrentFileCount: 100, isRecursive: recursive); var frontend = new DataLakeStoreFrontEndAdapter(dlAccountName, dataLakeStoreFileSystemClient); var uploader = new DataLakeStoreUploader(parameters, frontend); uploader.Execute(); return(true); }
// Upload the file public static void UploadFile(string srcFilePath, string destFilePath, bool force = true) { //TODO: 5. Configure the file upload var parameters = new UploadParameters(/*...*/); var frontend = new DataLakeStoreFrontEndAdapter(/*...*/); var uploader = new DataLakeStoreUploader(/*...*/); //TODO: 6. Upload the file uploader.//... }
/// <summary> /// Verifies the file was successfully uploaded. /// </summary> /// <param name="up">The upload parameters.</param> /// <param name="frontEnd">The front end.</param> /// <param name="fileContents">The file contents.</param> private void VerifyFileUploadedSuccessfully(UploadParameters up, InMemoryFrontEnd frontEnd, byte[] fileContents) { Assert.True(frontEnd.StreamExists(up.TargetStreamPath), "Uploaded stream does not exist"); Assert.Equal(1, frontEnd.StreamCount); Assert.Equal(fileContents.Length, frontEnd.GetStreamLength(up.TargetStreamPath)); var uploadedData = frontEnd.GetStreamContents(up.TargetStreamPath); AssertExtensions.AreEqual(fileContents, uploadedData, "Uploaded stream is not binary identical to input file"); }
public UploadParameters ValidateParameters() { BlobUri destinationUri; if (!BlobUri.TryParseUri(Destination, out destinationUri)) { throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); } BlobUri baseImageUri = null; if (this.BaseImageUriToPatch != null) { if (!BlobUri.TryParseUri(BaseImageUriToPatch, out baseImageUri)) { throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); } if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) { var message = String.Format(Rsrc.AddAzureVhdCommandSASUriNotSupportedInPatchMode, destinationUri.Uri); throw new ArgumentOutOfRangeException("Destination", message); } } var storageCredentialsFactory = CreateStorageCredentialsFactory(); PathIntrinsics currentPath = SessionState.Path; var filePath = new FileInfo(currentPath.GetUnresolvedProviderPathFromPSPath(LocalFilePath.ToString())); using (var vds = new VirtualDiskStream(filePath.FullName)) { if (vds.DiskType == DiskType.Fixed) { long divisor = Convert.ToInt64(Math.Pow(2, 9)); long rem = 0; Math.DivRem(filePath.Length, divisor, out rem); if (rem != 0) { throw new ArgumentOutOfRangeException("LocalFilePath", "Given vhd file is a corrupted fixed vhd"); } } } var parameters = new UploadParameters( destinationUri, baseImageUri, filePath, OverWrite.IsPresent, (NumberOfUploaderThreads) ?? DefaultNumberOfUploaderThreads) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) }; return(parameters); }
public static int Handle(UploadParameters options) { bool valid = true; if (String.IsNullOrWhiteSpace(options.UploadUser)) { valid = String.IsNullOrWhiteSpace(options.UploadPassword); } else { valid = !String.IsNullOrWhiteSpace(options.UploadPassword); } if (!valid) { Console.WriteLine("Error: upload -u user and -p password must be used together"); return(1); } IsUploading = true; // This task will be all the program does. We need to do enough setup so that // the upload code can work, then tear it down. Program.SetUpErrorHandling(); try { using (var applicationContainer = new ApplicationContainer()) { Program.SetUpLocalization(applicationContainer); Browser.SetUpXulRunner(); Browser.XulRunnerShutdown += Program.OnXulRunnerShutdown; LocalizationManager.SetUILanguage(Settings.Default.UserInterfaceLanguage, false); var transfer = new BookTransfer(new BloomParseClient(), ProjectContext.CreateBloomS3Client(), applicationContainer.BookThumbNailer, new BookDownloadStartingEvent()); // Since Bloom is not a normal console app, when run from a command line, the new command prompt // appears at once. The extra newlines here are attempting to separate this from our output. Console.WriteLine("\nstarting upload"); transfer.UploadFolder(options.Path, applicationContainer, options.ExcludeNarrationAudio, options.UploadUser, options.UploadPassword, options.SingleBookshelfLevel, options.PreserveThumbnails); Console.WriteLine(("\nupload complete\n")); } return(0); } catch (Exception ex) { Debug.WriteLine(ex.Message); Console.WriteLine(ex.Message); Console.WriteLine(ex.StackTrace); return(1); } }
public void ProcessRequest(HttpContext context) { try { // Prepare parameters UploadParameters parameters = GetParametersFromRequest(context); if (!VerifyTicket(parameters.RequestToken, parameters.Ticket)) { throw new Exception("No valid ticket for this request was found."); } // Check request content length and actual stream length // If they're not equal, it means the client canceled the uploading bool lengthsMatched = context.Request.ContentLength == context.Request.InputStream.Length; if (!lengthsMatched) { return; } var storage = SharpBoxSupport.OpenDropBoxStorage(); var newFileName = ObjectId.GenerateNewId() + parameters.FileExtension; string downloadURI = UploadDownlaodable(context, storage, newFileName, parameters.AlbumId); string photoURI = UploadPhoto(context, storage, newFileName, parameters.AlbumId); string thumbURI = UploadThumbnail(context, storage, newFileName, parameters.AlbumId); storage.Close(); string[] coverURIs = AlbumRepository.UpdateCover(parameters.AlbumId, thumbURI); Photo photo = new Photo() { PhotoURI = photoURI, ThumbURI = thumbURI, DownloadURI = downloadURI, AlbumId = parameters.AlbumId, CreatedBy = parameters.User }; PhotoRepository.CreatePhoto(photo); context.Response.Write(string.Join(";", coverURIs)); } catch (Exception ex) { Logger.Error(ex.ToString()); context.Request.InputStream.Close(); context.Response.Write("error"); } }
/// <summary> /// Verifies the file was successfully uploaded. /// </summary> /// <param name="up">The upload parameters.</param> /// <param name="frontEnd">The front end.</param> private void VerifyFileUploadedSuccessfully(UploadParameters up, InMemoryFrontEnd frontEnd) { if (up.IsRecursive) { var fileList = new Dictionary <string, byte[]> { { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_largeFilePath)), _largeFileData }, { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_smallFilePath)), _smallFileData }, { string.Format("{0}/{1}", up.TargetStreamPath, Path.GetFileName(_downloadFilePath)), _smallFileData } }; VerifyFileUploadedSuccessfully(fileList, frontEnd, up.IsDownload); } else { VerifyFileUploadedSuccessfully(up, frontEnd, _largeFileData); } }
public void UploadMetadataGenerator_AlignSegmentsToRecordBoundaries() { //We keep creating a file, by appending a number of bytes to it (taken from FileLengthsInMB). //At each iteration, we append a new blob of data, and then run the whole test on the entire file var rnd = new Random(0); string folderPath = string.Format(@"{0}\uploadtest", Environment.CurrentDirectory); string filePath = Path.Combine(folderPath, "verifymetadata.txt"); try { if (!Directory.Exists(folderPath)) { Directory.CreateDirectory(folderPath); } if (File.Exists(filePath)) { File.Delete(filePath); } foreach (var lengthMB in FileLengthsMB) { var appendLength = (int)(lengthMB * 1024 * 1024); AppendToFile(filePath, appendLength, rnd, 0, MaxAppendLength); string metadataFilePath = filePath + ".metadata.txt"; var up = new UploadParameters(filePath, filePath, null, isBinary: false, maxSegmentLength: 4 * 1024 * 1024); var mg = new UploadMetadataGenerator(up, new InMemoryFrontEnd()); var metadata = mg.CreateNewMetadata(metadataFilePath); VerifySegmentsAreOnRecordBoundaries(metadata, filePath); } } finally { if (Directory.Exists(folderPath)) { Directory.Delete(folderPath, true); } } }
public void UploadFile(string srcFilePath, string destFilePath, bool force = true) { var ok = false; while (!ok) { try { _adlsFileSystemClient.FileSystem.Mkdirs(_adlsAccountName, destFilePath); var parameters = new UploadParameters(srcFilePath, destFilePath, _adlsAccountName, isOverwrite: force); var frontend = new DataLakeStoreFrontEndAdapter(_adlsAccountName, _adlsFileSystemClient); progress = new MyProgress(); var uploader = new DataLakeStoreUploader(parameters, frontend, progress); uploader.Execute(); ok = true; } catch (Exception ex) { Console.WriteLine(ex.ToString()); Login(); } } }
private static UploadParameters CreateUploadParametersFromOptionalParameters( byte[] binary, IUploadOptionalParameters parameters) { var uploadParameters = new UploadParameters(binary); if (parameters != null) { uploadParameters.AdditionalOwnerIds = parameters.AdditionalOwnerIds; uploadParameters.AppendCustomRequestParameters = parameters.AppendCustomRequestParameters; uploadParameters.InitCustomRequestParameters = parameters.InitCustomRequestParameters; uploadParameters.MaxChunkSize = parameters.MaxChunkSize; uploadParameters.QueryMediaCategory = parameters.QueryMediaCategory; uploadParameters.QueryMediaType = parameters.QueryMediaType; uploadParameters.Timeout = parameters.Timeout; uploadParameters.UploadStateChanged = parameters.UploadStateChanged; uploadParameters.WaitForTwitterProcessing = parameters.WaitForTwitterProcessing; } return(uploadParameters); }
public async Task <IActionResult> OnPostAsync([FromServices] IMediator mediator) { if (Files?.Count > 0) { SourceFile[] uploads = Files .Select(x => new SourceFile(x.OpenReadStream(), new MimeType(x.ContentType), x.FileName)) .ToArray(); // TODO: should we dispose streams after upload? var parameters = new UploadParameters(uploads, new UploadDescription(Description ?? string.Empty)); UploadId uploadId = await mediator.Send(new UploadFilesCommand( parameters )); try { await mediator.Send(new UpdateUploadDetailsFromMetadataCommand( uploadId )); } catch { // TODO: error logging } UploadDetails result = await mediator.Send(new GetUploadDetailsQuery( uploadId )); if (result.Files.Count == 1 && result.RejectedDuplicates.Count == 0) { TempData["OperationResultMessage"] = $"File uploaded successfully."; return(RedirectToPage("View", new { fileId = result.Files[0].FileId.Value })); } TempData["OperationResultMessage"] = $"{result.Files.Count} files uploaded successfully."; return(RedirectToPage("UploadResult", new { uploadId = result.Id.Value })); } ModelState.AddModelError(string.Empty, "Select at least one file."); return(Page()); }
/// <summary> /// Uploads the stream /// </summary> /// <param name="stream"> /// The stream to upload. The location must be reset to the start of the stream if it's already been accessed. /// </param> /// <param name="request"> /// The parameters for the upload. /// </param> /// <param name="chunkSizeBytes"> /// The size (in bytes) to include in each chunk. This defaults to about 3 megabytes per chunk. /// </param> /// <param name="emsSystem"> /// The unique identifier of the EMS system. /// </param> public virtual async Task <UploadRecord> UploadStreamAsync(Stream stream, UploadRequest request, int chunkSizeBytes = DefaultChunkSize, int emsSystem = NoEmsServerSpecified) { UploadParameters info = await StartUploadAsync(request, emsSystem); int bytesRead = 0, totalBytesRead = 0; byte[] buffer = new byte[chunkSizeBytes]; while ((bytesRead = await stream.ReadAsync(buffer, 0, chunkSizeBytes)) > 0) { // The start and end byte indexes for our upload call. int uploadStart = totalBytesRead; int uploadEnd = totalBytesRead + bytesRead - 1; // If we couldn't read a full chunk from the stream, this must be the end // of it. Truncate the byte array that we pass to the request. if (bytesRead < chunkSizeBytes) { buffer = buffer.Take(bytesRead).ToArray(); } // Send the chunk. UploadResult chunkResult = await UploadChunkAsync(info.TransferId, uploadStart, uploadEnd, buffer, emsSystem); // Cancel the upload if the chunk failed. if (chunkResult.TransferSuccessful.HasValue && !chunkResult.TransferSuccessful.Value) { return(await CancelUploadAsync(info.TransferId, emsSystem)); } // Track our current position. totalBytesRead += bytesRead; } // Finish the transfer. return(await FinishUploadAsync(info.TransferId, emsSystem)); }
private async Task UploadFiles() { var upload1 = new UploadParameters( new[] { CreateFakeImage(100, 100, Color.Red, "red.jpg"), CreateFakeImage(25, 25, Color.Green, "green.jpg"), CreateFakeImage(100, 20, Color.Blue, "blue.jpg"), CreateFakeImage(20, 80, Color.Black, "black.jpg"), }, new UploadDescription("Small rectangles")); var upload2 = new UploadParameters( new[] { CreateFakeImage(400, 400, Color.AliceBlue, null), CreateFakeImage(500, 200, Color.BlueViolet, "blue-violet.jpg"), CreateFakeImage(300, 400, Color.Crimson, "crimson.jpg"), }, new UploadDescription("Big rectangles")); var upload3 = new UploadParameters( Enumerable.Repeat(CreateFakeImage(50, 50, Color.BurlyWood, "_.jpg"), 10), new UploadDescription("Many small squares")); var upload4 = new UploadParameters( new[] { CreateFakeTextFile("hello", "hello.txt") }, new UploadDescription("1 text file")); await mediator.Send(new UploadFilesCommand( upload1 )); await mediator.Send(new UploadFilesCommand( upload2 )); await mediator.Send(new UploadFilesCommand( upload3 )); await mediator.Send(new UploadFilesCommand( upload4 )); }
public void DataLakeUploader_UploadSingleSegment() { var frontEnd = new InMemoryFrontEnd(); var mockFrontEnd = new MockableFrontEnd(frontEnd); mockFrontEnd.ConcatenateImplementation = (target, inputs) => { Assert.True(false, "Concatenate should not be called when using 1 segment"); }; var up = new UploadParameters( inputFilePath: _smallFilePath, targetStreamPath: "1", threadCount: ThreadCount, accountName: "foo", isResume: false, maxSegmentLength: 4 * 1024 * 1024, localMetadataLocation: Path.GetTempPath()); File.WriteAllBytes(_smallFilePath, _smallFileData); var uploader = new DataLakeStoreUploader(up, frontEnd); uploader.Execute(); VerifyFileUploadedSuccessfully(up, frontEnd, _smallFileData); }
/// <summary> /// Upload bloom books in the specified folder to the bloom library. /// Folders that contain exactly one .htm file are interpreted as books and uploaded. /// Other folders are searched recursively for children that appear to be bloom books. /// The parent folder of a bloom book is searched for a .bloomCollection file and, if one is found, /// the book is treated as part of that collection (e.g., for determining vernacular language). /// If the .bloomCollection file is not found, the book is not uploaded. /// N.B. The bulk upload process will go ahead and upload templates and books that are already on the server /// (over-writing the existing book) without informing the user. /// </summary> /// <remarks>This method is triggered by starting Bloom with "upload" on the cmd line.</remarks> public void BulkUpload(ApplicationContainer container, UploadParameters options) { BookUpload.Destination = options.Dest; using (var progress = new MultiProgress()) { var logFilePath = Path.Combine(options.Path, "BloomBulkUploadLog.txt"); progress.Add(new Bloom.Utils.ConsoleProgress()); progress.Add(new FileLogProgress(logFilePath)); if (!_singleBookUploader.IsThisVersionAllowedToUpload()) { var oldVersionMsg = LocalizationManager.GetString("PublishTab.Upload.OldVersion", "Sorry, this version of Bloom Desktop is not compatible with the current version of BloomLibrary.org. Please upgrade to a newer version."); progress.WriteMessage(oldVersionMsg); return; } Debug.Assert(!String.IsNullOrWhiteSpace(options.UploadUser)); if (!_singleBookUploader.ParseClient.AttemptSignInAgainForCommandLine(options.UploadUser, options.Dest, progress)) { progress.WriteError("Problem logging in. See messages above."); System.Environment.Exit(1); } progress.WriteMessage("Uploading books as user {0}", options.UploadUser); var bookParams = new BookUploadParameters(options); BulkRepairInstanceIds(options.Path); ProjectContext context = null; // Expensive to create; hold each one we make until we find a book that needs a different one. try { _collectionFoldersUploaded = new HashSet <string>(); _newBooksUploaded = 0; _booksUpdated = 0; _booksSkipped = 0; _booksWithErrors = 0; progress.WriteMessageWithColor("green", $"\n\nStarting upload at {DateTime.Now.ToString()}\n"); progress.WriteMessageWithColor("Magenta", $"Looking in '{bookParams.Folder}'..."); UploadCollectionOrKeepLookingDeeper(progress, container, bookParams, ref context); if (_collectionFoldersUploaded.Count > 0) { progress.WriteMessageWithColor("green", "\n\nAll finished!"); progress.WriteMessage("Processed {0} collection folders.", _collectionFoldersUploaded.Count); } else { progress.WriteError("Did not find any collections to upload."); } progress.WriteMessage("Uploaded {0} new books.", _newBooksUploaded); progress.WriteMessage("Updated {0} books that had changed.", _booksUpdated); progress.WriteMessage("Skipped {0} books that had not changed.", _booksSkipped); if (_booksSkipped > 0) { progress.WriteMessage("(If you don't want Bloom to skip books it thinks have not changed, you can use the --force argument to force all books to re-upload, or just use the Bloom UI to force upload this one book)."); } if (_booksWithErrors > 0) { progress.WriteError("Failed to upload {0} books. See \"{1}\" for details.", _booksWithErrors, logFilePath); } } finally { context?.Dispose(); } } }
async Task StrategyExecuteCmdletAsync(IAsyncCmdlet asyncCmdlet) { var client = new Client(DefaultProfile.DefaultContext); ResourceGroupName = ResourceGroupName ?? Name; VirtualNetworkName = VirtualNetworkName ?? Name; SubnetName = SubnetName ?? Name; PublicIpAddressName = PublicIpAddressName ?? Name; SecurityGroupName = SecurityGroupName ?? Name; var resourceClient = AzureSession.Instance.ClientFactory.CreateArmClient <ResourceManagementClient>( DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.ResourceManager); var parameters = new Parameters(this, client, resourceClient); if (DiskFile != null) { if (!resourceClient.ResourceGroups.CheckExistence(ResourceGroupName)) { Location = Location ?? parameters.DefaultLocation; var st0 = resourceClient.ResourceGroups.CreateOrUpdate( ResourceGroupName, new ResourceGroup { Location = Location, Name = ResourceGroupName }); } parameters.ImageAndOsType = new ImageAndOsType( Linux ? OperatingSystemTypes.Linux : OperatingSystemTypes.Windows, null, null); var storageClient = AzureSession.Instance.ClientFactory.CreateArmClient <StorageManagementClient>( DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.ResourceManager); var st1 = storageClient.StorageAccounts.Create( ResourceGroupName, Name, new StorageAccountCreateParameters { Sku = new Microsoft.Azure.Management.Storage.Version2017_10_01.Models.Sku { Name = SkuName.PremiumLRS }, Location = Location }); var filePath = new FileInfo(SessionState.Path.GetUnresolvedProviderPathFromPSPath(DiskFile)); using (var vds = new VirtualDiskStream(filePath.FullName)) { // 2 ^ 9 == 512 if (vds.DiskType == DiskType.Fixed && filePath.Length % 512 != 0) { throw new ArgumentOutOfRangeException( "filePath", string.Format("Given vhd file '{0}' is a corrupted fixed vhd", filePath)); } } var storageAccount = storageClient.StorageAccounts.GetProperties(ResourceGroupName, Name); // BlobUri destinationUri = null; BlobUri.TryParseUri( new Uri(string.Format( "{0}{1}/{2}{3}", storageAccount.PrimaryEndpoints.Blob, ResourceGroupName.ToLower(), Name.ToLower(), ".vhd")), out parameters.DestinationUri); if (parameters.DestinationUri?.Uri == null) { throw new ArgumentNullException("destinationUri"); } var storageCredentialsFactory = new StorageCredentialsFactory( ResourceGroupName, storageClient, DefaultContext.Subscription); var uploadParameters = new UploadParameters(parameters.DestinationUri, null, filePath, true, 2) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) }; if (!string.Equals( Environment.GetEnvironmentVariable("AZURE_TEST_MODE"), "Playback", StringComparison.OrdinalIgnoreCase)) { var st2 = VhdUploaderModel.Upload(uploadParameters); } } var result = await client.RunAsync(client.SubscriptionId, parameters, asyncCmdlet); if (result != null) { var fqdn = PublicIPAddressStrategy.Fqdn(DomainNameLabel, Location); var psResult = ComputeAutoMapperProfile.Mapper.Map <PSVirtualMachine>(result); psResult.FullyQualifiedDomainName = fqdn; var connectionString = parameters.ImageAndOsType.GetConnectionString( fqdn, Credential?.UserName); asyncCmdlet.WriteVerbose( Resources.VirtualMachineUseConnectionString, connectionString); asyncCmdlet.WriteObject(psResult); } }
async Task StrategyExecuteCmdletAsync(IAsyncCmdlet asyncCmdlet) { ResourceGroupName = ResourceGroupName ?? Name; VirtualNetworkName = VirtualNetworkName ?? Name; SubnetName = SubnetName ?? Name; PublicIpAddressName = PublicIpAddressName ?? Name; SecurityGroupName = SecurityGroupName ?? Name; var imageAndOsType = new ImageAndOsType(OperatingSystemTypes.Windows, null); var resourceGroup = ResourceGroupStrategy.CreateResourceGroupConfig(ResourceGroupName); var virtualNetwork = resourceGroup.CreateVirtualNetworkConfig( name: VirtualNetworkName, addressPrefix: AddressPrefix); var subnet = virtualNetwork.CreateSubnet(SubnetName, SubnetAddressPrefix); var publicIpAddress = resourceGroup.CreatePublicIPAddressConfig( name: PublicIpAddressName, getDomainNameLabel: () => DomainNameLabel, allocationMethod: AllocationMethod); var networkSecurityGroup = resourceGroup.CreateNetworkSecurityGroupConfig( name: SecurityGroupName, openPorts: OpenPorts, getOsType: () => imageAndOsType.OsType); var networkInterface = resourceGroup.CreateNetworkInterfaceConfig( Name, subnet, publicIpAddress, networkSecurityGroup); var availabilitySet = AvailabilitySetName == null ? null : resourceGroup.CreateAvailabilitySetConfig(name: AvailabilitySetName); ResourceConfig <VirtualMachine> virtualMachine = null; if (DiskFile == null) { virtualMachine = resourceGroup.CreateVirtualMachineConfig( name: Name, networkInterface: networkInterface, getImageAndOsType: () => imageAndOsType, adminUsername: Credential.UserName, adminPassword: new NetworkCredential(string.Empty, Credential.Password).Password, size: Size, availabilitySet: availabilitySet); } else { var resourceClient = AzureSession.Instance.ClientFactory.CreateArmClient <ResourceManagementClient>(DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.ResourceManager); if (!resourceClient.ResourceGroups.CheckExistence(ResourceGroupName)) { var st0 = resourceClient.ResourceGroups.CreateOrUpdate(ResourceGroupName, new ResourceGroup { Location = Location, Name = ResourceGroupName }); } imageAndOsType = new ImageAndOsType( Linux ? OperatingSystemTypes.Linux : OperatingSystemTypes.Windows, null); var storageClient = AzureSession.Instance.ClientFactory.CreateArmClient <StorageManagementClient>(DefaultProfile.DefaultContext, AzureEnvironment.Endpoint.ResourceManager); var st1 = storageClient.StorageAccounts.Create( ResourceGroupName, Name, new StorageAccountCreateParameters { #if !NETSTANDARD AccountType = AccountType.PremiumLRS, #else Sku = new Microsoft.Azure.Management.Storage.Models.Sku { Name = SkuName.PremiumLRS }, #endif Location = Location }); var filePath = new FileInfo(SessionState.Path.GetUnresolvedProviderPathFromPSPath(DiskFile)); using (var vds = new VirtualDiskStream(filePath.FullName)) { if (vds.DiskType == DiskType.Fixed) { long divisor = Convert.ToInt64(Math.Pow(2, 9)); long rem = 0; Math.DivRem(filePath.Length, divisor, out rem); if (rem != 0) { throw new ArgumentOutOfRangeException( "filePath", string.Format("Given vhd file '{0}' is a corrupted fixed vhd", filePath)); } } } var storageAccount = storageClient.StorageAccounts.GetProperties(ResourceGroupName, Name); BlobUri destinationUri = null; BlobUri.TryParseUri( new Uri(string.Format( "{0}{1}/{2}{3}", storageAccount.PrimaryEndpoints.Blob, ResourceGroupName.ToLower(), Name.ToLower(), ".vhd")), out destinationUri); if (destinationUri == null || destinationUri.Uri == null) { throw new ArgumentNullException("destinationUri"); } var storageCredentialsFactory = new StorageCredentialsFactory( this.ResourceGroupName, storageClient, DefaultContext.Subscription); var parameters = new UploadParameters(destinationUri, null, filePath, true, 2) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) }; if (!string.Equals( Environment.GetEnvironmentVariable("AZURE_TEST_MODE"), "Playback", StringComparison.OrdinalIgnoreCase)) { var st2 = VhdUploaderModel.Upload(parameters); } var disk = resourceGroup.CreateManagedDiskConfig( name: Name, sourceUri: destinationUri.Uri.ToString() ); virtualMachine = resourceGroup.CreateVirtualMachineConfig( name: Name, networkInterface: networkInterface, osType: imageAndOsType.OsType, disk: disk, size: Size, availabilitySet: availabilitySet); } var client = new Client(DefaultProfile.DefaultContext); // get current Azure state var current = await virtualMachine.GetStateAsync(client, new CancellationToken()); Location = current.UpdateLocation(Location, virtualMachine); // generate a domain name label if it's not specified. DomainNameLabel = await PublicIPAddressStrategy.UpdateDomainNameLabelAsync( domainNameLabel : DomainNameLabel, name : Name, location : Location, client : client); var fqdn = PublicIPAddressStrategy.Fqdn(DomainNameLabel, Location); if (DiskFile == null) { imageAndOsType = await client.UpdateImageAndOsTypeAsync(ImageName, Location); } // create target state var target = virtualMachine.GetTargetState(current, client.SubscriptionId, Location); if (target.Get(availabilitySet) != null) { throw new InvalidOperationException("Availability set doesn't exist."); } // apply target state var newState = await virtualMachine .UpdateStateAsync( client, target, new CancellationToken(), new ShouldProcess(asyncCmdlet), asyncCmdlet.ReportTaskProgress); var result = newState.Get(virtualMachine); if (result == null) { result = current.Get(virtualMachine); } if (result != null) { var psResult = ComputeAutoMapperProfile.Mapper.Map <PSVirtualMachine>(result); psResult.FullyQualifiedDomainName = fqdn; asyncCmdlet.WriteVerbose(imageAndOsType.OsType == OperatingSystemTypes.Windows ? "Use 'mstsc /v:" + fqdn + "' to connect to the VM." : "Use 'ssh " + Credential.UserName + "@" + fqdn + "' to connect to the VM."); asyncCmdlet.WriteObject(psResult); } }
public void CopyFile(string destinationPath, string accountName, string sourcePath, CancellationToken cmdletCancellationToken, int threadCount = -1, bool overwrite = false, bool resume = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null, ProgressRecord parentProgress = null) { FileType ignoredType; if (!overwrite && TestFileOrFolderExistence(destinationPath, accountName, out ignoredType)) { throw new InvalidOperationException(string.Format(Properties.Resources.LocalFileAlreadyExists, destinationPath)); } //TODO: defect: 4259238 (located here: http://vstfrd:8080/Azure/RD/_workitems/edit/4259238) needs to be resolved or the tracingadapter work around needs to be put back in // default the number of threads to use to the processor count if (threadCount < 1) { threadCount = Environment.ProcessorCount; } // Progress bar indicator. var description = string.Format("Copying File: {0} to DataLakeStore Location: {1} for account: {2}", sourcePath, destinationPath, accountName); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), "Upload to DataLakeStore Store", description) { PercentComplete = 0 }; if (parentProgress != null) { progress.ParentActivityId = parentProgress.ActivityId; } // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress<UploadProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int) (1.0*e.UploadedByteCount/e.TotalFileLength*100); } }; var uploadParameters = new UploadParameters(sourcePath, destinationPath, accountName, threadCount, overwrite, resume, isBinary); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, progressTracker); var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.Expect100Continue = false; // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); } finally { ServicePointManager.Expect100Continue = previousExpect100; } }
/// <summary> /// Verifies the file was successfully uploaded. /// </summary> /// <param name="up">The upload parameters.</param> /// <param name="frontEnd">The front end.</param> private void VerifyFileUploadedSuccessfully(UploadParameters up, InMemoryFrontEnd frontEnd) { VerifyFileUploadedSuccessfully(up, frontEnd, _largeFileData); }
private static void Main(string[] args) { try { Console.WriteLine("------------------------------------------------------------------------------"); Console.WriteLine($"AdlTransfer {Assembly.GetEntryAssembly().GetName().Version} Copyright (c) 2016 Sascha Dittmann. All Rights Reserved."); Console.WriteLine("------------------------------------------------------------------------------"); if (ParseArguments(args)) { return; } Console.WriteLine(); Console.WriteLine($"Source: {_sourcePath}"); Console.WriteLine($"Target: {_targetPath}"); Console.WriteLine($"Account Name: {_accountName}"); if (_verbose) { Console.WriteLine(); Console.WriteLine($"Per File Thread Count: {_perFileThreadCount}"); Console.WriteLine($"Concurrent File Count: {_concurrentFileCount}"); Console.WriteLine($"Segment Length: {_maxSegmentLength.ToSizeString()}"); Console.WriteLine(); Console.WriteLine($"Overwrite: {_isOverwrite}"); Console.WriteLine($"Binary: {_isBinary}"); Console.WriteLine($"Recursive: {_isRecursive}"); } Console.WriteLine(); var credentials = Authenticate(); var client = new DataLakeStoreFileSystemManagementClient(credentials); var frontEndAdapter = new DataLakeStoreFrontEndAdapter(_accountName, client); var uploadParameters = new UploadParameters( _sourcePath, _targetPath, _accountName, _perFileThreadCount, _concurrentFileCount, _isOverwrite, _isResume, _isBinary, _isRecursive, _isDownload, _maxSegmentLength, _localMetadataLocation); var progressTracker = new Progress <UploadProgress>(); progressTracker.ProgressChanged += UploadProgressChanged; var folderProgressTracker = new Progress <UploadFolderProgress>(); folderProgressTracker.ProgressChanged += UploadFolderProgressChanged; var uploader = new DataLakeStoreUploader(uploadParameters, frontEndAdapter, progressTracker, folderProgressTracker); Console.WriteLine($"{(_isResume ? "Resuming" : "Starting")} {(_isDownload ? "Download" : "Upload")}..."); uploader.Execute(); Console.WriteLine($"{(_isDownload ? "Download" : "Upload")} completed."); } catch (Exception e) { Console.ForegroundColor = ConsoleColor.Red; Console.Error.WriteLine(e.Message); Console.ResetColor(); Environment.ExitCode = -1; } }
public void CopyDirectory( string destinationFolderPath, string accountName, string sourceFolderPath, CancellationToken cmdletCancellationToken, int folderThreadCount = -1, int perFileThreadCount = -1, bool recursive = false, bool overwrite = false, bool resume = false, bool forceBinaryOrText = false, bool isBinary = false, Cmdlet cmdletRunningRequest = null) { var totalBytes = GetByteCountInDirectory(sourceFolderPath, recursive); var totalFiles = GetFileCountInDirectory(sourceFolderPath, recursive); var progress = new ProgressRecord( uniqueActivityIdGenerator.Next(0, 10000000), string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, totalBytes, totalFiles), "Copy in progress...") { PercentComplete = 0 }; UpdateProgress(progress, cmdletRunningRequest); var internalFolderThreads = folderThreadCount <= 0 ? Environment.ProcessorCount : folderThreadCount; var internalFileThreads = perFileThreadCount <= 0 ? Environment.ProcessorCount : perFileThreadCount; // we need to override the default .NET value for max connections to a host to our number of threads, if necessary (otherwise we won't achieve the parallelism we want) var previousDefaultConnectionLimit = ServicePointManager.DefaultConnectionLimit; var previousExpect100 = ServicePointManager.Expect100Continue; try { ServicePointManager.DefaultConnectionLimit = Math.Max((internalFolderThreads * internalFileThreads) + internalFolderThreads, ServicePointManager.DefaultConnectionLimit); ServicePointManager.Expect100Continue = false; // On update from the Data Lake store uploader, capture the progress. var progressTracker = new System.Progress <UploadFolderProgress>(); progressTracker.ProgressChanged += (s, e) => { lock (ConsoleOutputLock) { progress.PercentComplete = (int)(1.0 * e.UploadedByteCount / e.TotalFileLength * 100); progress.Activity = string.Format("Copying Folder: {0}{1}. Total bytes remaining: {2}. Total files remaining: {3}", sourceFolderPath, recursive ? " recursively" : string.Empty, e.TotalFileLength - e.UploadedByteCount, e.TotalFileCount - e.UploadedFileCount); } }; var uploadParameters = new UploadParameters(sourceFolderPath, destinationFolderPath, accountName, internalFileThreads, internalFolderThreads, isOverwrite: overwrite, isResume: resume, isBinary: isBinary, isRecursive: recursive); var uploader = new DataLakeStoreUploader(uploadParameters, new DataLakeStoreFrontEndAdapter(accountName, _client, cmdletCancellationToken), cmdletCancellationToken, folderProgressTracker: progressTracker); // Execute the uploader. var uploadTask = Task.Run(() => { cmdletCancellationToken.ThrowIfCancellationRequested(); uploader.Execute(); cmdletCancellationToken.ThrowIfCancellationRequested(); }, cmdletCancellationToken); TrackUploadProgress(uploadTask, progress, cmdletRunningRequest, cmdletCancellationToken); if (!cmdletCancellationToken.IsCancellationRequested) { progress.PercentComplete = 100; progress.RecordType = ProgressRecordType.Completed; UpdateProgress(progress, cmdletRunningRequest); } } catch (Exception e) { throw new CloudException(string.Format(Properties.Resources.UploadFailedMessage, e)); } finally { ServicePointManager.DefaultConnectionLimit = previousDefaultConnectionLimit; ServicePointManager.Expect100Continue = previousExpect100; } }
public UploadParameters ValidateParameters() { BlobUri destinationUri; if (!BlobUri.TryParseUri(Destination, out destinationUri)) { throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); } BlobUri baseImageUri = null; if (this.BaseImageUriToPatch != null) { if (!BlobUri.TryParseUri(BaseImageUriToPatch, out baseImageUri)) { throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); } if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) { var message = String.Format(Resources.AddAzureVhdCommandSASUriNotSupportedInPatchMode, destinationUri.Uri); throw new ArgumentOutOfRangeException("Destination", message); } } var storageCredentialsFactory = CreateStorageCredentialsFactory(); var parameters = new UploadParameters(destinationUri, baseImageUri, LocalFilePath, OverWrite.IsPresent, NumberOfUploaderThreads) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(storageCredentialsFactory, TimeSpan.FromMinutes(1)) }; return parameters; }
public UploadParameters ValidateParameters() { BlobUri destinationUri; if (!BlobUri.TryParseUri(Destination, out destinationUri)) { throw new ArgumentOutOfRangeException("Destination", this.Destination.ToString()); } BlobUri baseImageUri = null; if (this.BaseImageUriToPatch != null) { if (!BlobUri.TryParseUri(BaseImageUriToPatch, out baseImageUri)) { throw new ArgumentOutOfRangeException("BaseImageUriToPatch", this.BaseImageUriToPatch.ToString()); } if (!String.IsNullOrEmpty(destinationUri.Uri.Query)) { var message = String.Format("SAS Uri for the destination blob is not supported in patch mode:{0}", destinationUri.Uri); throw new ArgumentOutOfRangeException("Destination", message); } } var parameters = new UploadParameters(destinationUri, baseImageUri, LocalFilePath, OverWrite.IsPresent, NumberOfUploaderThreads) { Cmdlet = this, BlobObjectFactory = new CloudPageBlobObjectFactory(this.Channel, this.CurrentSubscription.SubscriptionId, TimeSpan.FromMinutes(1)) }; return parameters; }
/// <summary> /// Encrypts and uploads file attachment. /// </summary> /// <param name="record">Keeper record.</param> /// <param name="uploadTask">Upload task</param> /// <returns>Awaitable task.</returns> /// <seealso cref="IVaultFileAttachment.UploadAttachment"/> public async Task UploadAttachment(PasswordRecord record, IAttachmentUploadTask uploadTask) { var fileStream = uploadTask.Stream; if (fileStream == null) { throw new KeeperInvalidParameter("Vault::UploadAttachment", "uploadTask", "GetStream()", "null"); } var thumbStream = uploadTask.Thumbnail?.Stream; var command = new RequestUploadCommand { FileCount = 1, ThumbnailCount = thumbStream != null ? 1 : 0 }; var rs = await Auth.ExecuteAuthCommand <RequestUploadCommand, RequestUploadResponse>(command); if (rs.FileUploads == null || rs.FileUploads.Length < 1) { throw new KeeperInvalidParameter("Vault::UploadAttachment", "request_upload", "file_uploads", "empty"); } var fileUpload = rs.FileUploads[0]; UploadParameters thumbUpload = null; if (rs.ThumbnailUploads != null && rs.ThumbnailUploads.Length > 0) { thumbUpload = rs.ThumbnailUploads[0]; } var key = CryptoUtils.GenerateEncryptionKey(); var atta = new AttachmentFile { Id = fileUpload.FileId, Name = uploadTask.Name, Title = uploadTask.Title, Key = key.Base64UrlEncode(), Type = uploadTask.MimeType, LastModified = DateTimeOffset.Now, }; var transform = new EncryptAesV1Transform(key); using (var cryptoStream = new CryptoStream(fileStream, transform, CryptoStreamMode.Read)) { await UploadSingleFile(fileUpload, cryptoStream); atta.Size = transform.EncryptedBytes; } if (thumbUpload != null && thumbStream != null) { try { transform = new EncryptAesV1Transform(key); using (var cryptoStream = new CryptoStream(thumbStream, transform, CryptoStreamMode.Read)) { await UploadSingleFile(thumbUpload, cryptoStream); } var thumbnail = new AttachmentFileThumb { Id = thumbUpload.FileId, Type = uploadTask.Thumbnail.MimeType, Size = uploadTask.Thumbnail.Size }; var ts = new[] { thumbnail }; atta.Thumbnails = atta.Thumbnails == null ? ts : atta.Thumbnails.Concat(ts).ToArray(); } catch (Exception e) { Trace.TraceError("Upload Thumbnail: {0}: \"{1}\"", e.GetType().Name, e.Message); } } record.Attachments.Add(atta); }