private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null) { server ??= ClientContext.Current.Server; // force set values uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes; if (uploadData.FileLength == 0) { uploadData.FileLength = binaryStream.Length; } requestData.Parameters.Add("create", "1"); dynamic uploadedContent = null; // Get ChunkToken try { SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", requestData); var retryCount = 0; await Retrier.RetryAsync(10, 1000, async() => { retryCount++; var retryText = retryCount > 1 ? $" (retry {retryCount})" : string.Empty; server.Logger?.LogTrace($"Uploading initial data of {uploadData.FileName}{retryText}."); var httpContent = new StringContent(uploadData.ToString()); httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType); await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent, response => { uploadData.ChunkToken = response.Content.ReadAsStringAsync().GetAwaiter().GetResult(); }, CancellationToken.None).ConfigureAwait(false); }, (i, exception) => { // choose the exceptions when we can retry the operation return(exception switch { null => true, ClientException cex when (int) cex.StatusCode == 429 || cex.ErrorData?.ExceptionType == "NodeIsOutOfDateException" => false, _ => throw exception }); });
private static async Task <Content> UploadTextInternalAsync(string text, UploadData uploadData, ODataRequest requestData, CancellationToken cancellationToken, ServerContext server = null) { // force set values if (text.Length > ClientContext.Current.ChunkSizeInBytes) { throw new InvalidOperationException($"Cannot upload a text that longer than the chunk size " + $"({ClientContext.Current.ChunkSizeInBytes})."); } if (uploadData.FileLength == 0) { uploadData.FileLength = text.Length; } uploadData.FileText = text; dynamic uploadedContent = null; var model = JsonHelper.GetJsonPostModel(uploadData.ToDictionary()); var httpContent = new StringContent(model); httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType); await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent, async response => { if (response != null) { var rs = await response.Content.ReadAsStringAsync().ConfigureAwait(false); uploadedContent = JsonHelper.Deserialize(rs); } }, cancellationToken).ConfigureAwait(false); if (uploadedContent == null) { return(null); } int contentId = uploadedContent.Id; var content = Content.Create(contentId); content.Name = uploadedContent.Name; content.Path = uploadedContent.Url; return(content); }
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null) { // force set values uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes; if (uploadData.FileLength == 0) { uploadData.FileLength = binaryStream.Length; } requestData.Parameters["create"] = "1"; dynamic uploadedContent = null; var retryCount = 0; // send initial request while (retryCount < REQUEST_RETRY_COUNT) { try { var myReq = CreateInitUploadWebRequest(requestData.ToString(), server, uploadData); SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", myReq.RequestUri); using (var wr = await myReq.GetResponseAsync()) { uploadData.ChunkToken = await ReadResponseStringAsync(wr); } // succesful request: skip out from retry loop break; } catch (WebException ex) { if (retryCount >= REQUEST_RETRY_COUNT - 1) { var ce = new ClientException("Error during binary upload.", ex); ce.Data["SiteUrl"] = requestData.SiteUrl; ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path; ce.Data["FileName"] = uploadData.FileName; ce.Data["ContentType"] = uploadData.ContentType; throw ce; } else { Thread.Sleep(50); } } retryCount++; } var boundary = "---------------------------" + DateTime.UtcNow.Ticks.ToString("x"); var trailer = Encoding.ASCII.GetBytes("\r\n--" + boundary + "--\r\n"); // send subsequent requests var buffer = new byte[ClientContext.Current.ChunkSizeInBytes]; int bytesRead; var start = 0; // reuse previous request data, but remove unnecessary parameters requestData.Parameters.Remove("create"); while ((bytesRead = binaryStream.Read(buffer, 0, buffer.Length)) != 0) { retryCount = 0; //get the request object for the actual chunk while (retryCount < REQUEST_RETRY_COUNT) { Stream requestStream = null; HttpWebRequest chunkRequest; try { chunkRequest = CreateChunkUploadWebRequest(requestData.ToString(), server, uploadData, boundary, out requestStream); SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", chunkRequest.RequestUri); if (uploadData.UseChunk) { chunkRequest.Headers.Set("Content-Range", string.Format("bytes {0}-{1}/{2}", start, start + bytesRead - 1, binaryStream.Length)); } //write the chunk into the request stream requestStream.Write(buffer, 0, bytesRead); requestStream.Write(trailer, 0, trailer.Length); await requestStream.FlushAsync(); } finally { if (requestStream != null) { requestStream.Close(); } } //send the request try { using (var wr = await chunkRequest.GetResponseAsync()) { var rs = await ReadResponseStringAsync(wr); uploadedContent = JsonHelper.Deserialize(rs); } // successful request: skip out from the retry loop break; } catch (WebException ex) { if (retryCount >= REQUEST_RETRY_COUNT - 1) { var ce = new ClientException("Error during binary upload.", ex); ce.Data["SiteUrl"] = requestData.SiteUrl; ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path; ce.Data["FileName"] = uploadData.FileName; ce.Data["ContentType"] = uploadData.ContentType; throw ce; } else { Thread.Sleep(50); } } retryCount++; } start += bytesRead; // notify the caller about every chunk that was uploaded successfully if (progressCallback != null) { progressCallback(start); } } if (uploadedContent == null) { return(null); } int contentId = uploadedContent.Id; var content = Content.Create(contentId); content.Name = uploadedContent.Name; content.Path = uploadedContent.Url; return(content); }
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null) { // force set values uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes; if (uploadData.FileLength == 0) { uploadData.FileLength = binaryStream.Length; } requestData.Parameters.Add("create", "1"); dynamic uploadedContent = null; // Get ChunkToken try { SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", requestData); var httpContent = new StringContent(uploadData.ToString()); httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType); await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent, async response => { uploadData.ChunkToken = await response.Content.ReadAsStringAsync().ConfigureAwait(false); }, CancellationToken.None).ConfigureAwait(false); } catch (WebException ex) { var ce = new ClientException("Error during binary upload.", ex); ce.Data["SiteUrl"] = requestData.SiteUrl; ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path; ce.Data["FileName"] = uploadData.FileName; ce.Data["ContentType"] = uploadData.ContentType; throw ce; } // Reuse previous request data, but remove unnecessary parameters requestData.Parameters.Remove("create"); // Send subsequent requests var boundary = "---------------------------" + DateTime.UtcNow.Ticks.ToString("x"); var uploadFormData = uploadData.ToKeyValuePairs(); var contentDispositionHeaderValue = new ContentDispositionHeaderValue("attachment") { FileName = uploadData.FileName }; var buffer = new byte[ClientContext.Current.ChunkSizeInBytes]; int bytesRead; var start = 0; while ((bytesRead = binaryStream.Read(buffer, 0, buffer.Length)) != 0) { // Prepare the current chunk request var httpContent = new MultipartFormDataContent(boundary); foreach (var item in uploadFormData) { httpContent.Add(new StringContent(item.Value), item.Key); } httpContent.Headers.ContentDisposition = contentDispositionHeaderValue; if (uploadData.UseChunk) { httpContent.Headers.ContentRange = new ContentRangeHeaderValue(start, start + bytesRead - 1, binaryStream.Length); } // Add the chunk as a stream into the request content var postedStream = new MemoryStream(buffer, 0, bytesRead); httpContent.Add(new StreamContent(postedStream), "files[]", uploadData.FileName); // Process await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent, async response => { var rs = await response.Content.ReadAsStringAsync().ConfigureAwait(false); uploadedContent = JsonHelper.Deserialize(rs); }, CancellationToken.None).ConfigureAwait(false); start += bytesRead; // Notify the caller about every chunk that was uploaded successfully progressCallback?.Invoke(start); } if (uploadedContent == null) { return(null); } int contentId = uploadedContent.Id; var content = Content.Create(contentId); content.Name = uploadedContent.Name; content.Path = uploadedContent.Url; return(content); }