//============================================================================= Upload API
/// <summary>
/// Uploads a file to the server into the provided container.
/// </summary>
/// <param name="binaryStream">File contents.</param>
/// <param name="uploadData">Upload parameters.</param>
/// <param name="parentId">Parent id.</param>
/// <param name="server">Target server.</param>
/// <param name="progressCallback">An optional callback method that is called after each chunk is uploaded to the server.</param>
/// <returns>The uploaded file content returned at the end of the upload request.</returns>
public static async Task <Content> UploadAsync(Stream binaryStream, UploadData uploadData, int parentId, ServerContext server = null, Action <int> progressCallback = null)
{
var requestData = new ODataRequest(server)
{
ActionName = "Upload",
ContentId = parentId
};
return(await UploadInternalAsync(binaryStream, uploadData, requestData, server, progressCallback).ConfigureAwait(false));
}
/// <summary>
/// Uploads a file to the server into the provided container.
/// </summary>
/// <param name="binaryStream">File contents.</param>
/// <param name="uploadData">Upload parameters.</param>
/// <param name="parentPath">Parent path.</param>
/// <param name="server">Target server.</param>
/// <param name="progressCallback">An optional callback method that is called after each chunk is uploaded to the server.</param>
/// <returns>The uploaded file content returned at the end of the upload request.</returns>
public static async Task <Content> UploadAsync(Stream binaryStream, UploadData uploadData, string parentPath, ServerContext server = null, Action <int> progressCallback = null)
{
var requestData = new ODataRequest()
{
SiteUrl = ServerContext.GetUrl(server),
ActionName = "Upload",
Path = parentPath
};
return(await UploadInternalAsync(binaryStream, uploadData, requestData, server, progressCallback));
}
/// <summary>
/// Uploads a file to the server into the provided container.
/// </summary>
/// <param name="text">File contents.</param>
/// <param name="uploadData">Upload parameters.</param>
/// <param name="parentPath">Parent path.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <param name="server">Target server.</param>
/// <returns>The uploaded file content returned at the end of the upload request.</returns>
public static async Task <Content> UploadTextAsync(string text, UploadData uploadData, string parentPath,
CancellationToken cancellationToken, ServerContext server = null)
{
var requestData = new ODataRequest(server)
{
ActionName = "Upload",
Path = parentPath
};
return(await UploadTextInternalAsync(text, uploadData, requestData, cancellationToken, server).ConfigureAwait(false));
}
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null)
{
server ??= ClientContext.Current.Server;
// force set values
uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes;
if (uploadData.FileLength == 0)
{
uploadData.FileLength = binaryStream.Length;
}
requestData.Parameters.Add("create", "1");
dynamic uploadedContent = null;
// Get ChunkToken
try
{
SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", requestData);
var retryCount = 0;
await Retrier.RetryAsync(10, 1000, async() =>
{
retryCount++;
var retryText = retryCount > 1 ? $" (retry {retryCount})" : string.Empty;
server.Logger?.LogTrace($"Uploading initial data of {uploadData.FileName}{retryText}.");
var httpContent = new StringContent(uploadData.ToString());
httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType);
await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent,
response =>
{
uploadData.ChunkToken = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
}, CancellationToken.None).ConfigureAwait(false);
}, (i, exception) =>
{
// choose the exceptions when we can retry the operation
return(exception switch
{
null => true,
ClientException cex when
(int) cex.StatusCode == 429 ||
cex.ErrorData?.ExceptionType == "NodeIsOutOfDateException"
=> false,
_ => throw exception
});
});
private static async Task <Content> UploadTextInternalAsync(string text, UploadData uploadData, ODataRequest requestData,
CancellationToken cancellationToken, ServerContext server = null)
{
// force set values
if (text.Length > ClientContext.Current.ChunkSizeInBytes)
{
throw new InvalidOperationException($"Cannot upload a text that longer than the chunk size " +
$"({ClientContext.Current.ChunkSizeInBytes}).");
}
if (uploadData.FileLength == 0)
{
uploadData.FileLength = text.Length;
}
uploadData.FileText = text;
dynamic uploadedContent = null;
var model = JsonHelper.GetJsonPostModel(uploadData.ToDictionary());
var httpContent = new StringContent(model);
httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType);
await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent,
async response =>
{
if (response != null)
{
var rs = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
uploadedContent = JsonHelper.Deserialize(rs);
}
}, cancellationToken).ConfigureAwait(false);
if (uploadedContent == null)
{
return(null);
}
int contentId = uploadedContent.Id;
var content = Content.Create(contentId);
content.Name = uploadedContent.Name;
content.Path = uploadedContent.Url;
return(content);
}
/// <summary>
/// Uploads a file to the server into the provided container.
/// </summary>
/// <param name="parentId">Parent id.</param>
/// <param name="fileName">Name of the file to upload.</param>
/// <param name="stream">File contents.</param>
/// <param name="contentType">Content type of the file. Default is determined by the container.</param>
/// <param name="propertyName">Name of the field to upload to. Default is Binary.</param>
/// <param name="server">Target server.</param>
/// <param name="progressCallback">An optional callback method that is called after each chunk is uploaded to the server.</param>
/// <returns>The uploaded file content returned at the end of the upload request.</returns>
public static async Task <Content> UploadAsync(int parentId, string fileName, Stream stream, string contentType = null, string propertyName = null, ServerContext server = null, Action <int> progressCallback = null)
{
var uploadData = new UploadData()
{
FileName = fileName,
FileLength = stream.Length
};
if (!string.IsNullOrEmpty(contentType))
{
uploadData.ContentType = contentType;
}
if (!string.IsNullOrEmpty(propertyName))
{
uploadData.PropertyName = propertyName;
}
return(await RESTCaller.UploadAsync(stream, uploadData, parentId, server, progressCallback));
}
/// <summary>
/// Uploads a file to the server into the provided container.
/// </summary>
/// <param name="parentId">Parent id.</param>
/// <param name="fileName">Name of the file to upload.</param>
/// <param name="fileText">File content.</param>
/// <param name="cancellationToken">The token to monitor for cancellation requests.</param>
/// <param name="contentType">Content type of the file.</param>
/// <param name="propertyName">Name of the field to upload to. Default is Binary.</param>
/// <param name="server">Target server.</param>
/// <returns>The uploaded file content returned at the end of the upload request.</returns>
public static async Task <Content> UploadTextAsync(int parentId, string fileName, string fileText,
CancellationToken cancellationToken,
string contentType = null, string propertyName = null, ServerContext server = null)
{
var uploadData = new UploadData()
{
FileName = fileName,
ContentType = contentType,
};
if (!string.IsNullOrEmpty(contentType))
{
uploadData.ContentType = contentType;
}
if (!string.IsNullOrEmpty(propertyName))
{
uploadData.PropertyName = propertyName;
}
return(await RESTCaller.UploadTextAsync(fileText, uploadData, parentId, cancellationToken, server)
.ConfigureAwait(false));
}
private static HttpWebRequest CreateChunkUploadWebRequest(string url, ServerContext server, UploadData uploadData, string boundary, out Stream requestStream)
{
var myRequest = GetRequest(url, server);
myRequest.Method = "POST";
myRequest.ContentType = "multipart/form-data; boundary=" + boundary;
myRequest.Headers.Add("Content-Disposition", "attachment; filename=\"" + Uri.EscapeUriString(uploadData.FileName) + "\"");
var boundarybytes = Encoding.ASCII.GetBytes("\r\n--" + boundary + "\r\n");
// we must not close the stream after this as we need to write the chunk into it in the caller method
requestStream = myRequest.GetRequestStream();
//write form data values
foreach (var kvp in uploadData.ToDictionary())
{
requestStream.Write(boundarybytes, 0, boundarybytes.Length);
var formitem = string.Format(UPLOAD_FORMDATA_TEMPLATE, kvp.Key, kvp.Value);
var formitembytes = Encoding.UTF8.GetBytes(formitem);
requestStream.Write(formitembytes, 0, formitembytes.Length);
}
//write a boundary
requestStream.Write(boundarybytes, 0, boundarybytes.Length);
//write file name and content type
var header = string.Format(UPLOAD_HEADER_TEMPLATE, "files[]", Uri.EscapeUriString(uploadData.FileName));
var headerbytes = Encoding.UTF8.GetBytes(header);
requestStream.Write(headerbytes, 0, headerbytes.Length);
return(myRequest);
}
private static WebRequest CreateInitUploadWebRequest(string url, ServerContext server, UploadData uploadData)
{
var myRequest = GetRequest(url, server);
myRequest.Method = "POST";
var postDataBytes = Encoding.ASCII.GetBytes(uploadData.ToString());
myRequest.ContentLength = postDataBytes.Length;
myRequest.ContentType = UPLOAD_CONTENTTYPE;
using (var reqStream = myRequest.GetRequestStream())
{
reqStream.Write(postDataBytes, 0, postDataBytes.Length);
}
return(myRequest);
}
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null)
{
// force set values
uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes;
if (uploadData.FileLength == 0)
{
uploadData.FileLength = binaryStream.Length;
}
requestData.Parameters["create"] = "1";
dynamic uploadedContent = null;
var retryCount = 0;
// send initial request
while (retryCount < REQUEST_RETRY_COUNT)
{
try
{
var myReq = CreateInitUploadWebRequest(requestData.ToString(), server, uploadData);
SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", myReq.RequestUri);
using (var wr = await myReq.GetResponseAsync())
{
uploadData.ChunkToken = await ReadResponseStringAsync(wr);
}
// succesful request: skip out from retry loop
break;
}
catch (WebException ex)
{
if (retryCount >= REQUEST_RETRY_COUNT - 1)
{
var ce = new ClientException("Error during binary upload.", ex);
ce.Data["SiteUrl"] = requestData.SiteUrl;
ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path;
ce.Data["FileName"] = uploadData.FileName;
ce.Data["ContentType"] = uploadData.ContentType;
throw ce;
}
else
{
Thread.Sleep(50);
}
}
retryCount++;
}
var boundary = "---------------------------" + DateTime.UtcNow.Ticks.ToString("x");
var trailer = Encoding.ASCII.GetBytes("\r\n--" + boundary + "--\r\n");
// send subsequent requests
var buffer = new byte[ClientContext.Current.ChunkSizeInBytes];
int bytesRead;
var start = 0;
// reuse previous request data, but remove unnecessary parameters
requestData.Parameters.Remove("create");
while ((bytesRead = binaryStream.Read(buffer, 0, buffer.Length)) != 0)
{
retryCount = 0;
//get the request object for the actual chunk
while (retryCount < REQUEST_RETRY_COUNT)
{
Stream requestStream = null;
HttpWebRequest chunkRequest;
try
{
chunkRequest = CreateChunkUploadWebRequest(requestData.ToString(), server, uploadData, boundary, out requestStream);
SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", chunkRequest.RequestUri);
if (uploadData.UseChunk)
{
chunkRequest.Headers.Set("Content-Range", string.Format("bytes {0}-{1}/{2}", start, start + bytesRead - 1, binaryStream.Length));
}
//write the chunk into the request stream
requestStream.Write(buffer, 0, bytesRead);
requestStream.Write(trailer, 0, trailer.Length);
await requestStream.FlushAsync();
}
finally
{
if (requestStream != null)
{
requestStream.Close();
}
}
//send the request
try
{
using (var wr = await chunkRequest.GetResponseAsync())
{
var rs = await ReadResponseStringAsync(wr);
uploadedContent = JsonHelper.Deserialize(rs);
}
// successful request: skip out from the retry loop
break;
}
catch (WebException ex)
{
if (retryCount >= REQUEST_RETRY_COUNT - 1)
{
var ce = new ClientException("Error during binary upload.", ex);
ce.Data["SiteUrl"] = requestData.SiteUrl;
ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path;
ce.Data["FileName"] = uploadData.FileName;
ce.Data["ContentType"] = uploadData.ContentType;
throw ce;
}
else
{
Thread.Sleep(50);
}
}
retryCount++;
}
start += bytesRead;
// notify the caller about every chunk that was uploaded successfully
if (progressCallback != null)
{
progressCallback(start);
}
}
if (uploadedContent == null)
{
return(null);
}
int contentId = uploadedContent.Id;
var content = Content.Create(contentId);
content.Name = uploadedContent.Name;
content.Path = uploadedContent.Url;
return(content);
}
private static async Task <Content> UploadInternalAsync(Stream binaryStream, UploadData uploadData, ODataRequest requestData, ServerContext server = null, Action <int> progressCallback = null)
{
// force set values
uploadData.UseChunk = binaryStream.Length > ClientContext.Current.ChunkSizeInBytes;
if (uploadData.FileLength == 0)
{
uploadData.FileLength = binaryStream.Length;
}
requestData.Parameters.Add("create", "1");
dynamic uploadedContent = null;
// Get ChunkToken
try
{
SnTrace.Category(ClientContext.TraceCategory).Write("###>REQ: {0}", requestData);
var httpContent = new StringContent(uploadData.ToString());
httpContent.Headers.ContentType = new MediaTypeHeaderValue(JsonContentMimeType);
await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server, httpContent,
async response =>
{
uploadData.ChunkToken = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
}, CancellationToken.None).ConfigureAwait(false);
}
catch (WebException ex)
{
var ce = new ClientException("Error during binary upload.", ex);
ce.Data["SiteUrl"] = requestData.SiteUrl;
ce.Data["Parent"] = requestData.ContentId != 0 ? requestData.ContentId.ToString() : requestData.Path;
ce.Data["FileName"] = uploadData.FileName;
ce.Data["ContentType"] = uploadData.ContentType;
throw ce;
}
// Reuse previous request data, but remove unnecessary parameters
requestData.Parameters.Remove("create");
// Send subsequent requests
var boundary = "---------------------------" + DateTime.UtcNow.Ticks.ToString("x");
var uploadFormData = uploadData.ToKeyValuePairs();
var contentDispositionHeaderValue = new ContentDispositionHeaderValue("attachment")
{
FileName = uploadData.FileName
};
var buffer = new byte[ClientContext.Current.ChunkSizeInBytes];
int bytesRead;
var start = 0;
while ((bytesRead = binaryStream.Read(buffer, 0, buffer.Length)) != 0)
{
// Prepare the current chunk request
var httpContent = new MultipartFormDataContent(boundary);
foreach (var item in uploadFormData)
{
httpContent.Add(new StringContent(item.Value), item.Key);
}
httpContent.Headers.ContentDisposition = contentDispositionHeaderValue;
if (uploadData.UseChunk)
{
httpContent.Headers.ContentRange = new ContentRangeHeaderValue(start, start + bytesRead - 1, binaryStream.Length);
}
// Add the chunk as a stream into the request content
var postedStream = new MemoryStream(buffer, 0, bytesRead);
httpContent.Add(new StreamContent(postedStream), "files[]", uploadData.FileName);
// Process
await ProcessWebResponseAsync(requestData.ToString(), HttpMethod.Post, server,
httpContent,
async response =>
{
var rs = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
uploadedContent = JsonHelper.Deserialize(rs);
}, CancellationToken.None).ConfigureAwait(false);
start += bytesRead;
// Notify the caller about every chunk that was uploaded successfully
progressCallback?.Invoke(start);
}
if (uploadedContent == null)
{
return(null);
}
int contentId = uploadedContent.Id;
var content = Content.Create(contentId);
content.Name = uploadedContent.Name;
content.Path = uploadedContent.Url;
return(content);
}
