/// <summary> /// Creates a new file in the current folder. This operation is only valid for folders. /// The file is not added to any existing filesystem snapshot, but you can use the returned object to operate on it. /// </summary> /// <param name="name">The name of the file to add.</param> /// <param name="contents">A stream with the contents that the file will have.</param> /// <param name="feedbackChannel">Allows you to receive feedback about the operation while it is running.</param> /// <param name="cancellationToken">Allows you to cancel the operation.</param> public async Task<CloudItem> NewFileAsync(string name, Stream contents, IFeedbackChannel feedbackChannel = null, CancellationToken cancellationToken = default(CancellationToken)) { Argument.ValidateIsNotNullOrWhitespace(name, "name"); Argument.ValidateIsNotNull(contents, "contents"); if (name.IndexOfAny(new[] { '/', '\\' }) != -1) throw new ArgumentException("A file name cannot contain path separator characters.", "name"); if (!IsContainer) throw new InvalidOperationException("This item cannot contain child items."); if (!contents.CanRead) throw new ArgumentException("The contents stream is not readable.", "contents"); if (!contents.CanSeek) throw new ArgumentException("The contents stream is not seekable.", "contents"); PatternHelper.LogMethodCall("NewFileAsync", feedbackChannel, cancellationToken); PatternHelper.EnsureFeedbackChannel(ref feedbackChannel); using (await _client.AcquireLock(feedbackChannel, cancellationToken)) { feedbackChannel.Status = "Preparing for upload"; var beginUploadResult = await _client.ExecuteCommandInternalAsync<BeginUploadResult>(feedbackChannel, cancellationToken, new BeginUploadCommand { Size = contents.Length }); cancellationToken.ThrowIfCancellationRequested(); feedbackChannel.Status = "Uploading file contents"; Base64Data? completionToken = null; // Set when last chunk has been uploaded. var chunkSizes = Algorithms.MeasureChunks(contents.Length); var chunkCount = chunkSizes.Length; var chunkMacs = new byte[chunkCount][]; var dataKey = Algorithms.GetRandomBytes(16); var nonce = Algorithms.GetRandomBytes(8); // Limit number of chunks in flight at the same time. var concurrentUploadSemaphore = new SemaphoreSlim(4); // Only one file read operation can take place at a time. var concurrentReadSemaphore = new SemaphoreSlim(1); // For progress calculations. long completedBytes = 0; CancellationTokenSource chunkUploadsCancellationSource = new CancellationTokenSource(); var uploadTasks = new List<Task>(); for (int i = 0; i < chunkCount; i++) { int chunkIndex = i; long startOffset = chunkSizes.Take(i).Select(size => (long)size).Sum(); uploadTasks.Add(Task.Run(async delegate { var operationName = string.Format("Uploading chunk {0} of {1}", chunkIndex + 1, chunkSizes.Length); using (var chunkFeedback = feedbackChannel.BeginSubOperation(operationName)) { byte[] bytes = new byte[chunkSizes[chunkIndex]]; using (await SemaphoreLock.TakeAsync(concurrentUploadSemaphore)) { chunkUploadsCancellationSource.Token.ThrowIfCancellationRequested(); using (await SemaphoreLock.TakeAsync(concurrentReadSemaphore)) { chunkUploadsCancellationSource.Token.ThrowIfCancellationRequested(); chunkFeedback.Status = "Reading contents"; // Read in the raw bytes for this chunk. contents.Position = startOffset; contents.Read(bytes, 0, bytes.Length); } chunkFeedback.Status = "Encrypting contents"; byte[] chunkMac; Algorithms.EncryptNodeDataChunk(bytes, dataKey, nonce, out chunkMac, startOffset); chunkMacs[chunkIndex] = chunkMac; await RetryHelper.ExecuteWithRetryAsync(async delegate { chunkUploadsCancellationSource.Token.ThrowIfCancellationRequested(); chunkFeedback.Status = string.Format("Uploading {0} bytes", chunkSizes[chunkIndex]); var url = beginUploadResult.UploadUrl + "/" + startOffset; HttpResponseMessage response; using (var client = new HttpClient()) response = await client.PostAsyncCancellationSafe(url, new ByteArrayContent(bytes), chunkUploadsCancellationSource.Token); response.EnsureSuccessStatusCode(); var responseBody = await response.Content.ReadAsStringAsync(); // Result from last chunk is: base64-encoded completion handle to give to NewItemsCommand // Negative ASCII integer in case of error. Standard-ish stuff? // Empty is just OK but not last chunk. if (responseBody.StartsWith("[")) { // Error result! // Assuming it is formatted like this, I never got it to return an error result. // It always just hangs if I do anything funny... var errorResult = JObject.Parse(responseBody); Channel.ThrowOnFailureResult(errorResult); throw new ProtocolViolationException("Got an unexpected result from chunk upload: " + responseBody); } else if (!string.IsNullOrWhiteSpace(responseBody)) { // Completion token! completionToken = responseBody; } if (bytes.Length != chunkSizes[chunkIndex]) throw new MegaException(string.Format("Expected {0} bytes in chunk but got {1}.", chunkSizes[chunkIndex], bytes.Length)); }, ChunkUploadRetryPolicy, chunkFeedback, chunkUploadsCancellationSource.Token); } Interlocked.Add(ref completedBytes, chunkSizes[chunkIndex]); } }, chunkUploadsCancellationSource.Token)); } // Wait for all tasks to finish. Stop immediately on cancel or if any single task fails. while (uploadTasks.Any(d => !d.IsCompleted)) { feedbackChannel.Progress = Interlocked.Read(ref completedBytes) * 1.0 / contents.Length; Exception failureReason = null; if (cancellationToken.IsCancellationRequested) { failureReason = new OperationCanceledException(); } else { var failedTask = uploadTasks.FirstOrDefault(d => d.IsFaulted || d.IsCanceled); if (failedTask != null) { if (failedTask.Exception != null) failureReason = failedTask.Exception.GetBaseException(); else failureReason = new MegaException("The file could not be uploaded."); } } if (failureReason == null) { await Task.Delay(1000); continue; } chunkUploadsCancellationSource.Cancel(); feedbackChannel.Status = "Stopping upload due to subtask failure"; try { // Wait for all of the tasks to complete, just so we do not leave any dangling activities in the background. Task.WaitAll(uploadTasks.ToArray()); } catch { // It will throw something no notify us of the cancellation. Whatever, do not care. } // Rethrow the failure causing exception. ExceptionDispatchInfo.Capture(failureReason).Throw(); } if (!completionToken.HasValue) throw new ProtocolViolationException("Mega did not provide upload completion token."); feedbackChannel.Progress = 1; feedbackChannel.Progress = null; feedbackChannel.Status = "Creating filesystem entry"; var metaMac = Algorithms.CalculateMetaMac(chunkMacs, dataKey); var itemKey = Algorithms.CreateNodeKey(dataKey, nonce, metaMac); var attributesKey = Algorithms.DeriveNodeAttributesKey(itemKey); // Create the file from the uploaded data. var result = await _client.ExecuteCommandInternalAsync<NewItemsResult>(feedbackChannel, cancellationToken, new NewItemsCommand { ClientInstanceID = _client._clientInstanceID, ParentID = ID, Items = new[] { new NewItemsCommand.NewItem { Attributes = new ItemAttributes { { "n", name } }.SerializeAndEncrypt(attributesKey), Type = KnownItemTypes.File, EncryptedItemKey = Algorithms.EncryptKey(itemKey, _client.AesKey), ItemContentsReference = completionToken.Value } } }); _client.InvalidateFilesystemInternal(); return FromTemplate(result.Items.Single(), _client); } }