private async Task <HttpResponseMessage> ContainerGetRequestAsync(
Int64 containerId,
String itemPath,
String contentType,
CancellationToken cancellationToken,
Guid scopeIdentifier,
Object userState = null)
{
if (containerId < 1)
{
throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
}
List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier);
HttpRequestMessage requestMessage = await CreateRequestMessageAsync(HttpMethod.Get, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false);
if (!String.IsNullOrEmpty(contentType))
{
requestMessage.Headers.Accept.Clear();
var header = new System.Net.Http.Headers.MediaTypeWithQualityHeaderValue(contentType);
header.Parameters.Add(new System.Net.Http.Headers.NameValueHeaderValue(ApiResourceVersionExtensions.c_apiVersionHeaderKey, "1.0"));
header.Parameters.Add(new System.Net.Http.Headers.NameValueHeaderValue(ApiResourceVersionExtensions.c_legacyResourceVersionHeaderKey, "1"));
requestMessage.Headers.Accept.Add(header);
}
return(await SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, userState, cancellationToken).ConfigureAwait(false));
}
/// <summary>
/// Queries for container items in a container.
/// </summary>
/// <param name="containerId">Id of the container to query.</param>
/// <param name="scopeIdentifier">Id of the scope to query</param>
/// <param name="isShallow">Whether to just return immediate children items under the itemPath</param>
/// <param name="itemPath">Path to folder or file. Can be empty or null to query from container root.</param>
/// <param name="userState">User state</param>
/// <param name="includeDownloadTickets">Whether to include download ticket(s) for the container item(s) in the result</param>
/// <param name="cancellationToken">CancellationToken to cancel the task</param>
public Task <List <FileContainerItem> > QueryContainerItemsAsync(Int64 containerId, Guid scopeIdentifier, Boolean isShallow, String itemPath = null, Object userState = null, Boolean includeDownloadTickets = false, CancellationToken cancellationToken = default(CancellationToken))
{
if (containerId < 1)
{
throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
}
List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier, includeDownloadTickets, isShallow);
return(SendAsync <List <FileContainerItem> >(HttpMethod.Get, FileContainerResourceIds.FileContainer, routeValues: new { containerId = containerId }, version: s_currentApiVersion, queryParameters: query, userState: userState, cancellationToken: cancellationToken));
}
public async Task <HttpResponseMessage> UploadFileAsync(
Int64 containerId,
String itemPath,
Stream fileStream,
byte[] contentId,
Int64 fileLength,
Boolean isGzipped,
Guid scopeIdentifier,
CancellationToken cancellationToken = default(CancellationToken),
int chunkSize = c_defaultChunkSize,
int chunkRetryTimes = c_defaultChunkRetryTimes,
bool uploadFirstChunk = false,
Object userState = null)
{
if (containerId < 1)
{
throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
}
if (chunkSize > c_maxChunkSize)
{
chunkSize = c_maxChunkSize;
}
// if a contentId is specified but the chunk size is not a 2mb multiple error
if (contentId != null && (chunkSize % c_ContentChunkMultiple) != 0)
{
throw new ArgumentException(FileContainerResources.ChunksizeWrongWithContentId(c_ContentChunkMultiple), "chunkSize");
}
ArgumentUtility.CheckForNull(fileStream, "fileStream");
ApiResourceVersion gzipSupportedVersion = new ApiResourceVersion(new Version(1, 0), 2);
ApiResourceVersion requestVersion = await NegotiateRequestVersionAsync(FileContainerResourceIds.FileContainer, s_currentApiVersion, userState, cancellationToken).ConfigureAwait(false);
if (isGzipped &&
(requestVersion.ApiVersion < gzipSupportedVersion.ApiVersion ||
(requestVersion.ApiVersion == gzipSupportedVersion.ApiVersion && requestVersion.ResourceVersion < gzipSupportedVersion.ResourceVersion)))
{
throw new ArgumentException(FileContainerResources.GzipNotSupportedOnServer(), "isGzipped");
}
if (isGzipped && fileStream.Length >= fileLength)
{
throw new ArgumentException(FileContainerResources.BadCompression(), "fileLength");
}
HttpRequestMessage requestMessage = null;
List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier);
if (fileStream.Length == 0)
{
// zero byte upload
FileUploadTrace(itemPath, $"Upload zero byte file '{itemPath}'.");
requestMessage = await CreateRequestMessageAsync(HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false);
return(await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false));
}
bool multiChunk = false;
int totalChunks = 1;
if (fileStream.Length > chunkSize)
{
totalChunks = (int)Math.Ceiling(fileStream.Length / (double)chunkSize);
FileUploadTrace(itemPath, $"Begin chunking upload file '{itemPath}', chunk size '{chunkSize} Bytes', total chunks '{totalChunks}'.");
multiChunk = true;
}
else
{
FileUploadTrace(itemPath, $"File '{itemPath}' will be uploaded in one chunk.");
chunkSize = (int)fileStream.Length;
}
StreamParser streamParser = new StreamParser(fileStream, chunkSize);
SubStream currentStream = streamParser.GetNextStream();
HttpResponseMessage response = null;
Byte[] dataToSend = new Byte[chunkSize];
int currentChunk = 0;
Stopwatch uploadTimer = new Stopwatch();
while (currentStream.Length > 0 && !cancellationToken.IsCancellationRequested)
{
currentChunk++;
for (int attempt = 1; attempt <= chunkRetryTimes && !cancellationToken.IsCancellationRequested; attempt++)
{
if (attempt > 1)
{
TimeSpan backoff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10));
FileUploadTrace(itemPath, $"Backoff {backoff.TotalSeconds} seconds before attempt '{attempt}' chunk '{currentChunk}' of file '{itemPath}'.");
await Task.Delay(backoff, cancellationToken).ConfigureAwait(false);
currentStream.Seek(0, SeekOrigin.Begin);
}
FileUploadTrace(itemPath, $"Attempt '{attempt}' for uploading chunk '{currentChunk}' of file '{itemPath}'.");
// inorder for the upload to be retryable, we need the content to be re-readable
// to ensure this we copy the chunk into a byte array and send that
// chunk size ensures we can convert the length to an int
int bytesToCopy = (int)currentStream.Length;
using (MemoryStream ms = new MemoryStream(dataToSend))
{
await currentStream.CopyToAsync(ms, bytesToCopy, cancellationToken).ConfigureAwait(false);
}
// set the content and the Content-Range header
HttpContent byteArrayContent = new ByteArrayContent(dataToSend, 0, bytesToCopy);
byteArrayContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue("application/octet-stream");
byteArrayContent.Headers.ContentLength = currentStream.Length;
byteArrayContent.Headers.ContentRange = new System.Net.Http.Headers.ContentRangeHeaderValue(currentStream.StartingPostionOnOuterStream,
currentStream.EndingPostionOnOuterStream,
streamParser.Length);
FileUploadTrace(itemPath, $"Generate new HttpRequest for uploading file '{itemPath}', chunk '{currentChunk}' of '{totalChunks}'.");
try
{
if (requestMessage != null)
{
requestMessage.Dispose();
requestMessage = null;
}
requestMessage = await CreateRequestMessageAsync(
HttpMethod.Put,
FileContainerResourceIds.FileContainer,
routeValues : new { containerId = containerId },
version : s_currentApiVersion,
content : byteArrayContent,
queryParameters : query,
userState : userState,
cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// stop re-try on cancellation.
throw;
}
catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to create HttpRequest. Error: {ex.ToString()}.");
continue;
}
if (isGzipped)
{
//add gzip header info
byteArrayContent.Headers.ContentEncoding.Add("gzip");
byteArrayContent.Headers.Add("x-tfs-filelength", fileLength.ToString(System.Globalization.CultureInfo.InvariantCulture));
}
if (contentId != null)
{
byteArrayContent.Headers.Add("x-vso-contentId", Convert.ToBase64String(contentId)); // Base64FormattingOptions.None is default when not supplied
}
FileUploadTrace(itemPath, $"Start uploading file '{itemPath}' to server, chunk '{currentChunk}'.");
uploadTimer.Restart();
try
{
if (response != null)
{
response.Dispose();
response = null;
}
response = await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when(cancellationToken.IsCancellationRequested)
{
// stop re-try on cancellation.
throw;
}
catch (Exception ex) when(attempt < chunkRetryTimes) // not the last attempt
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' fail to send request to server. Error: {ex.ToString()}.");
continue;
}
uploadTimer.Stop();
FileUploadTrace(itemPath, $"Finished upload chunk '{currentChunk}' of file '{itemPath}', elapsed {uploadTimer.ElapsedMilliseconds} (ms), response code '{response.StatusCode}'.");
if (multiChunk)
{
FileUploadProgress(itemPath, currentChunk, (int)Math.Ceiling(fileStream.Length / (double)chunkSize));
}
if (response.IsSuccessStatusCode)
{
break;
}
else if (IsFastFailResponse(response))
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request and cannot continue.");
break;
}
else
{
FileUploadTrace(itemPath, $"Chunk '{currentChunk}' attempt '{attempt}' of file '{itemPath}' received non-success status code {response.StatusCode} for sending request.");
continue;
}
}
// if we don't have success then bail and return the failed response
if (!response.IsSuccessStatusCode)
{
break;
}
if (contentId != null && response.StatusCode == HttpStatusCode.Created)
{
// no need to keep uploading since the server said it has all the content
FileUploadTrace(itemPath, $"Stop chunking upload the rest of the file '{itemPath}', since server already has all the content.");
break;
}
currentStream = streamParser.GetNextStream();
if (uploadFirstChunk)
{
break;
}
}
cancellationToken.ThrowIfCancellationRequested();
return(response);
}
/// <summary>
/// Uploads a file in chunks to the specified uri.
/// </summary>
/// <param name="fileStream">Stream to upload.</param>
/// <param name="cancellationToken">CancellationToken to cancel the task</param>
/// <returns>Http response message.</returns>
public async Task <HttpResponseMessage> UploadFileAsync(
Int64 containerId,
String itemPath,
Stream fileStream,
Guid scopeIdentifier,
CancellationToken cancellationToken = default(CancellationToken),
int chunkSize = c_defaultChunkSize,
bool uploadFirstChunk = false,
Object userState = null,
Boolean compressStream = true)
{
if (containerId < 1)
{
throw new ArgumentException(WebApiResources.ContainerIdMustBeGreaterThanZero(), "containerId");
}
ArgumentUtility.CheckForNull(fileStream, "fileStream");
if (fileStream.Length == 0)
{
HttpRequestMessage requestMessage;
List <KeyValuePair <String, String> > query = AppendItemQueryString(itemPath, scopeIdentifier);
// zero byte upload
requestMessage = await CreateRequestMessageAsync(HttpMethod.Put, FileContainerResourceIds.FileContainer, routeValues : new { containerId = containerId }, version : s_currentApiVersion, queryParameters : query, userState : userState, cancellationToken : cancellationToken).ConfigureAwait(false);
return(await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false));
}
ApiResourceVersion gzipSupportedVersion = new ApiResourceVersion(new Version(1, 0), 2);
ApiResourceVersion requestVersion = await NegotiateRequestVersionAsync(FileContainerResourceIds.FileContainer, s_currentApiVersion, userState, cancellationToken : cancellationToken).ConfigureAwait(false);
if (compressStream &&
(requestVersion.ApiVersion < gzipSupportedVersion.ApiVersion ||
(requestVersion.ApiVersion == gzipSupportedVersion.ApiVersion && requestVersion.ResourceVersion < gzipSupportedVersion.ResourceVersion)))
{
compressStream = false;
}
Stream streamToUpload = fileStream;
Boolean gzipped = false;
long filelength = fileStream.Length;
try
{
if (compressStream)
{
if (filelength > 65535) // if file greater than 64K use a file
{
String tempFile = Path.GetTempFileName();
streamToUpload = File.Create(tempFile, 32768, FileOptions.DeleteOnClose | FileOptions.Asynchronous);
}
else
{
streamToUpload = new MemoryStream((int)filelength + 8);
}
using (GZipStream zippedStream = new GZipStream(streamToUpload, CompressionMode.Compress, true))
{
await fileStream.CopyToAsync(zippedStream).ConfigureAwait(false);
}
if (streamToUpload.Length >= filelength)
{
// compression did not help
streamToUpload.Dispose();
streamToUpload = fileStream;
}
else
{
gzipped = true;
}
streamToUpload.Seek(0, SeekOrigin.Begin);
}
return(await UploadFileAsync(containerId, itemPath, streamToUpload, null, filelength, gzipped, scopeIdentifier, cancellationToken, chunkSize, uploadFirstChunk : uploadFirstChunk, userState : userState));
}
finally
{
if (gzipped && streamToUpload != null)
{
streamToUpload.Dispose();
}
}
}