/// <summary> /// Downloads the batch job results from a specified URL. /// </summary> /// <param name="url">The download URL from a batch job.</param> /// <returns>The results from the batch job.</returns> protected string DownloadResults(string url) { // Mark the usage. featureUsageRegistry.MarkUsage(FEATURE_ID); BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user); while (true) { WebRequest request = HttpUtilities.BuildRequest(url, "GET", user.Config); WebResponse response = null; LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider()); logEntry.LogRequest(request, "", HEADERS_TO_MASK); try { response = request.GetResponse(); string contents = MediaUtilities.GetStreamContentsAsString( response.GetResponseStream()); logEntry.LogResponse(response, false, contents); logEntry.Flush(); return(contents); } catch (WebException e) { HandleCloudException(errorHandler, logEntry, e); } finally { if (response != null) { response.Close(); } } } }
/// <summary> /// Gets the upload progress. /// </summary> /// <param name="url">The resumable upload URL.</param> /// <returns>The number of bytes uploaded so far.</returns> protected virtual int GetUploadProgress(string url) { int totalLength = 0; int retval = 0; BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user); while (true) { WebResponse response = null; // As per https://cloud.google.com/storage/docs/resumable-uploads-xml#step_4wzxhzdk17query_title_for_the_upload_status, // one should be passing bytes */Length, where length is the actual // length of bytes that was being uploaded during the request that was // interrupted. In practice, passing length as 0 also works. WebRequest request = HttpUtilities.BuildRangeRequest(url, 0, string.Format("bytes */{0}", totalLength), user.Config); LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider()); logEntry.LogRequest(GenerateRequestInfo(request, ""), HEADERS_TO_MASK); try { response = request.GetResponse(); // This block of code is hit if the user uploaded without chunking and // then called this method. string contents = MediaUtilities.GetStreamContentsAsString( response.GetResponseStream()); Dictionary <string, object> temp = JsonConvert.DeserializeObject <Dictionary <string, object> >(contents); int.TryParse(temp["size"].ToString(), out retval); logEntry.LogResponse(GenerateResponseInfo(response, "", "")); logEntry.Flush(); break; } catch (WebException e) { // This block of code is hit if if chunking is enabled and the // operations upload is incomplete. The server responds with a 308 // status code. See // https://cloud.google.com/storage/docs/resumable-uploads-xml#step_4wzxhzdk17query_title_for_the_upload_status // for more details. if (IsPartialUploadSuccessResponse(e)) { retval = ExtractUpperRange(e.Response.Headers["Range"], retval) + 1; logEntry.LogResponse(GenerateResponseInfo(e.Response, "", "")); logEntry.Flush(); break; } else { HandleCloudException(errorHandler, logEntry, e); } } finally { if (response != null) { response.Close(); } } } return(retval); }
/// <summary> /// Uploads a chunk of data for the batch job. /// </summary> /// <param name="url">The resumable upload URL.</param> /// <param name="postBody">The post body.</param> /// <param name="start">The start of range of bytes from the postBody to /// be uploaded.</param> /// <param name="end">The end of range of bytes from the postBody to be /// uploaded.</param> /// <param name="startOffset">The start offset in the stream to upload to.</param> /// <param name="totalUploadSize">If specified, this indicates the total /// size of the upload. When doing a streamed upload, this value will be /// null for all except the last chunk.</param> protected virtual void UploadChunk(string url, byte[] postBody, int start, int end, long startOffset, long?totalUploadSize) { BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user); string totalUploadSizeForRequest = (totalUploadSize == null) ? "*" : totalUploadSize.ToString(); while (true) { WebResponse response = null; LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider()); long rangeStart = start + startOffset; long rangeEnd = end + startOffset; int bytesToWrite = end - start + 1; string rangeHeader = string.Format("bytes {0}-{1}/{2}", rangeStart, rangeEnd, totalUploadSizeForRequest); HttpWebRequest request = (HttpWebRequest)HttpUtilities.BuildRangeRequest( url, bytesToWrite, rangeHeader, user.Config); request.ContentType = "application/xml"; string textToLog = GetTextToLog(postBody, start, bytesToWrite); try { logEntry.LogRequest(GenerateRequestInfo(request, textToLog), HEADERS_TO_MASK); using (Stream requestStream = request.GetRequestStream()) { requestStream.Write(postBody, start, bytesToWrite); } response = request.GetResponse(); logEntry.LogResponse(GenerateResponseInfo(response, "", "")); logEntry.Flush(); return; } catch (WebException e) { response = e.Response; if (IsPartialUploadSuccessResponse(e)) { logEntry.LogResponse(GenerateResponseInfo(e.Response, "", "")); logEntry.Flush(); return; } else { HandleCloudException(errorHandler, logEntry, e); } } finally { if (response != null) { response.Close(); } } } }
/// <summary> /// Uploads a chunk of data for the batch job. /// </summary> /// <param name="url">The resumable upload URL.</param> /// <param name="postBody">The post body.</param> /// <param name="start">The start of range of bytes to be uploaded.</param> /// <param name="end">The end of range of bytes to be uploaded.</param> protected virtual void UploadChunk(string url, byte[] postBody, int start, int end) { BulkJobErrorHandler errorHandler = new BulkJobErrorHandler(user); while (true) { WebResponse response = null; LogEntry logEntry = new LogEntry(User.Config, new DefaultDateTimeProvider()); int bytesToWrite = end - start + 1; HttpWebRequest request = (HttpWebRequest)HttpUtilities.BuildRangeRequest( url, bytesToWrite, string.Format("bytes {0}-{1}/{2}", start, end, postBody.Length), user.Config); request.ContentType = "application/xml"; try { logEntry.LogRequest(request, "Truncated", HEADERS_TO_MASK); using (Stream requestStream = request.GetRequestStream()) { requestStream.Write(postBody, start, bytesToWrite); } response = request.GetResponse(); logEntry.LogResponse(response, true, ""); logEntry.Flush(); return; } catch (WebException e) { response = e.Response; if (IsPartialUploadSuccessResponse(e)) { logEntry.LogResponse(e.Response, true, ""); logEntry.Flush(); return; } else { HandleCloudException(errorHandler, logEntry, e); } } finally { if (response != null) { response.Close(); } } } }
/// <summary> /// Handles the exception from Google Cloud Storage servers when uploading /// operations. /// </summary> /// <param name="errorHandler">The error handler.</param> /// <param name="logEntry">The log entry.</param> /// <param name="e">The web exception that was thrown by the server.</param> /// <returns>True if this is a success, false if this was a server error. /// </returns> private void HandleCloudException(BulkJobErrorHandler errorHandler, LogEntry logEntry, WebException e) { Exception downloadException = null; using (WebResponse response = e.Response) { string contents = HttpUtilities.GetErrorResponseBody(e); logEntry.LogResponse(response, false, contents); logEntry.Flush(); downloadException = ParseException(e, contents); if (errorHandler.ShouldRetry(downloadException)) { errorHandler.PrepareForRetry(downloadException); } else { throw downloadException; } } }