/// <summary>
/// Downloads the batch job results from a specified URL.
/// </summary>
/// <param name="url">The download URL from a batch job.</param>
/// <returns>The results from the batch job.</returns>
protected string DownloadResults(string url)
{
// Mark the usage.
featureUsageRegistry.MarkUsage(FEATURE_ID);
BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user);
while (true)
{
WebRequest request = HttpUtilities.BuildRequest(url, "GET", user.Config);
WebResponse response = null;
LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider());
logEntry.LogRequest(request, "", HEADERS_TO_MASK);
try {
response = request.GetResponse();
string contents = MediaUtilities.GetStreamContentsAsString(
response.GetResponseStream());
logEntry.LogResponse(response, false, contents);
logEntry.Flush();
return(contents);
} catch (WebException e) {
HandleCloudException(errorHandler, logEntry, e);
} finally {
if (response != null)
{
response.Close();
}
}
}
}
/// <summary>
/// Gets the upload progress.
/// </summary>
/// <param name="url">The resumable upload URL.</param>
/// <returns>The number of bytes uploaded so far.</returns>
protected virtual int GetUploadProgress(string url)
{
int totalLength = 0;
int retval = 0;
BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user);
while (true)
{
WebResponse response = null;
// As per https://cloud.google.com/storage/docs/resumable-uploads-xml#step_4wzxhzdk17query_title_for_the_upload_status,
// one should be passing bytes */Length, where length is the actual
// length of bytes that was being uploaded during the request that was
// interrupted. In practice, passing length as 0 also works.
WebRequest request = HttpUtilities.BuildRangeRequest(url, 0,
string.Format("bytes */{0}", totalLength), user.Config);
LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider());
logEntry.LogRequest(GenerateRequestInfo(request, ""), HEADERS_TO_MASK);
try {
response = request.GetResponse();
// This block of code is hit if the user uploaded without chunking and
// then called this method.
string contents = MediaUtilities.GetStreamContentsAsString(
response.GetResponseStream());
Dictionary <string, object> temp =
JsonConvert.DeserializeObject <Dictionary <string, object> >(contents);
int.TryParse(temp["size"].ToString(), out retval);
logEntry.LogResponse(GenerateResponseInfo(response, "", ""));
logEntry.Flush();
break;
} catch (WebException e) {
// This block of code is hit if if chunking is enabled and the
// operations upload is incomplete. The server responds with a 308
// status code. See
// https://cloud.google.com/storage/docs/resumable-uploads-xml#step_4wzxhzdk17query_title_for_the_upload_status
// for more details.
if (IsPartialUploadSuccessResponse(e))
{
retval = ExtractUpperRange(e.Response.Headers["Range"], retval) + 1;
logEntry.LogResponse(GenerateResponseInfo(e.Response, "", ""));
logEntry.Flush();
break;
}
else
{
HandleCloudException(errorHandler, logEntry, e);
}
} finally {
if (response != null)
{
response.Close();
}
}
}
return(retval);
}
/// <summary>
/// Uploads a chunk of data for the batch job.
/// </summary>
/// <param name="url">The resumable upload URL.</param>
/// <param name="postBody">The post body.</param>
/// <param name="start">The start of range of bytes from the postBody to
/// be uploaded.</param>
/// <param name="end">The end of range of bytes from the postBody to be
/// uploaded.</param>
/// <param name="startOffset">The start offset in the stream to upload to.</param>
/// <param name="totalUploadSize">If specified, this indicates the total
/// size of the upload. When doing a streamed upload, this value will be
/// null for all except the last chunk.</param>
protected virtual void UploadChunk(string url, byte[] postBody, int start, int end,
long startOffset, long?totalUploadSize)
{
BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user);
string totalUploadSizeForRequest = (totalUploadSize == null) ? "*" :
totalUploadSize.ToString();
while (true)
{
WebResponse response = null;
LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider());
long rangeStart = start + startOffset;
long rangeEnd = end + startOffset;
int bytesToWrite = end - start + 1;
string rangeHeader = string.Format("bytes {0}-{1}/{2}",
rangeStart, rangeEnd, totalUploadSizeForRequest);
HttpWebRequest request = (HttpWebRequest)HttpUtilities.BuildRangeRequest(
url, bytesToWrite, rangeHeader, user.Config);
request.ContentType = "application/xml";
string textToLog = GetTextToLog(postBody, start, bytesToWrite);
try {
logEntry.LogRequest(GenerateRequestInfo(request, textToLog), HEADERS_TO_MASK);
using (Stream requestStream = request.GetRequestStream()) {
requestStream.Write(postBody, start, bytesToWrite);
}
response = request.GetResponse();
logEntry.LogResponse(GenerateResponseInfo(response, "", ""));
logEntry.Flush();
return;
} catch (WebException e) {
response = e.Response;
if (IsPartialUploadSuccessResponse(e))
{
logEntry.LogResponse(GenerateResponseInfo(e.Response, "", ""));
logEntry.Flush();
return;
}
else
{
HandleCloudException(errorHandler, logEntry, e);
}
} finally {
if (response != null)
{
response.Close();
}
}
}
}
/// <summary>
/// Uploads a chunk of data for the batch job.
/// </summary>
/// <param name="url">The resumable upload URL.</param>
/// <param name="postBody">The post body.</param>
/// <param name="start">The start of range of bytes to be uploaded.</param>
/// <param name="end">The end of range of bytes to be uploaded.</param>
protected virtual void UploadChunk(string url, byte[] postBody, int start, int end)
{
BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user);
while (true)
{
WebResponse response = null;
LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider());
int bytesToWrite = end - start + 1;
HttpWebRequest request = (HttpWebRequest)HttpUtilities.BuildRangeRequest(
url, bytesToWrite,
string.Format("bytes {0}-{1}/{2}", start, end, postBody.Length), user.Config);
request.ContentType = "application/xml";
try {
logEntry.LogRequest(request, "Truncated", HEADERS_TO_MASK);
using (Stream requestStream = request.GetRequestStream()) {
requestStream.Write(postBody, start, bytesToWrite);
}
response = request.GetResponse();
logEntry.LogResponse(response, true, "");
logEntry.Flush();
return;
} catch (WebException e) {
response = e.Response;
if (IsPartialUploadSuccessResponse(e))
{
logEntry.LogResponse(e.Response, true, "");
logEntry.Flush();
return;
}
else
{
HandleCloudException(errorHandler, logEntry, e);
}
} finally {
if (response != null)
{
response.Close();
}
}
}
}
/// <summary>
/// Handles the exception from Google Cloud Storage servers when uploading
/// operations.
/// </summary>
/// <param name="errorHandler">The error handler.</param>
/// <param name="logEntry">The log entry.</param>
/// <param name="e">The web exception that was thrown by the server.</param>
/// <returns>True if this is a success, false if this was a server error.
/// </returns>
private void HandleCloudException(BulkJobErrorHandler errorHandler, LogEntry logEntry,
WebException e)
{
Exception downloadException = null;
using (WebResponse response = e.Response) {
string contents = HttpUtilities.GetErrorResponseBody(e);
logEntry.LogResponse(response, false, contents);
logEntry.Flush();
downloadException = ParseException(e, contents);
if (errorHandler.ShouldRetry(downloadException))
{
errorHandler.PrepareForRetry(downloadException);
}
else
{
throw downloadException;
}
}
}