/// <summary>
/// Gets the uploadPartUrl for each worker function.
/// </summary>
/// <returns></returns>
public async Task <UploadPartsUrlDetails> GetUploadPartUrl()
{
//Taken from https://www.backblaze.com/b2/docs/b2_get_upload_part_url.html with edits
//Create an UploadPartsUrlDetails data object to hold the data
UploadPartsUrlDetails uploadPartsUrlDetails = new UploadPartsUrlDetails();
// Get Upload URL
String getUploadUrlJsonStr = "{\"fileId\":\"" + fileDetails.fileId + "\"}";
byte[] getUloadUrlJsonData = Encoding.UTF8.GetBytes(getUploadUrlJsonStr);
//To allow us to count the number of failed attempts
int WebRequestAttempt = 1;
//To allow us to retry the webrequest if it fails (there is probably a better option here but I don't know it
RetryRequest:
HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_url");
//Intentionally generating an error for testing purposes
//HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_ur");
getUploadUrlRequest.Method = "POST";
getUploadUrlRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken);
getUploadUrlRequest.ContentType = "application/json; charset=utf-8";
getUploadUrlRequest.ContentLength = getUloadUrlJsonData.Length;
using (Stream stream = getUploadUrlRequest.GetRequestStream())
{
stream.Write(getUloadUrlJsonData, 0, getUloadUrlJsonData.Length);
stream.Close();
}
// Handle the response and print the json
try
{
HttpWebResponse getUploadUrlResponse = (HttpWebResponse)getUploadUrlRequest.GetResponse();
//I have made heavy changes here so errors are almost certainly due to that!
using (Stream responseStream = getUploadUrlResponse.GetResponseStream())
{
UploadPartUrlResponse uploadPartUrlResponse = await JsonSerializer.DeserializeAsync <UploadPartUrlResponse>(responseStream);
uploadPartsUrlDetails.authorizationToken = uploadPartUrlResponse.authorizationToken;
uploadPartsUrlDetails.uploadUrl = uploadPartUrlResponse.uploadUrl;
}
getUploadUrlResponse.Close();
}
catch (WebException e)
{
//Print error to console before retrying
using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response)
{
StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {errorResponse.StatusCode}. Retrying....", DebugLevel.Warn);
using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream()))
{
String text = reader.ReadToEnd();
StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn);
}
}
//If we have failed less than 5 times
if (WebRequestAttempt < 5)
{
//Log a message
StaticHelpers.DebugLogger("We have failed to get a part upload URL, retrying....", DebugLevel.Verbose);
//Wait a while
int secToWait = WebRequestAttempt * 2;
Thread.Sleep(secToWait * 1000);
//Increment counter
WebRequestAttempt++;
//Go back to retry the request
goto RetryRequest;
}
}
return(uploadPartsUrlDetails);
}
/// <summary>
/// Starts and is the upload worker that does the actual uploads.
/// </summary>
public async void StartUploadWorker()
{
StaticHelpers.DebugLogger("Starting an internal upload worker", DebugLevel.Verbose);
//Get our object containing our authorisation URLs
UploadPartsUrlDetails uploadPartsUrlDetails = await GetUploadPartUrl();
#region Loop where work is done
//While there are bytes still be sent
while (uploadDetails.totalBytesSent < localFileSize)
{
#region Check Bandwidth Monitor
//If the bandwidth monitor requires a reduction in usage
if (bandwidthMonitor.reduceUsage || bandwidthMonitor.urgentReduceUsage)
{
//Check thread count is greater than 1
if (AllThreads.Count(thread => thread.ThreadState != System.Threading.ThreadState.Stopped) > 1)
{
//Log to debug
StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor. Killing self....", DebugLevel.Verbose);
//Set reduceUsage to false
bandwidthMonitor.reduceUsage = false;
//Kill this thread
break;
}
else
{
StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor HOWEVER as the only remaining thread I am ignoring.", DebugLevel.Verbose);
}
}
#endregion
#region Get details from uploadDetails & Data from file
//Create variables outside of the lock so we can set it inside but still access it outside
//For a snapshot of uploadDetails
UploadDetails uploadDetailsSnapshot = new UploadDetails();
//Create the byte array for the data we are going to use
byte[] data = new byte[Singletons.options.PartSize * (1000 * 1000)];
//Lock the uploadDetails
lock (uploadDetailsLock)
{
#region Check if we are on the last part or even if there are no parts left
//If there is nothing left to upload
if ((localFileSize - uploadDetails.totalBytesSent) == 0)
{
//Break out of the loop as there is no more work to do
break;
}
//If the remaining bytes are less the minimum part size
if ((localFileSize - uploadDetails.totalBytesSent) <= uploadDetails.minimumPartSize)
{
//Changes the bytes sent for part to the remaining number of bytes
uploadDetails.bytesSentForPart = (localFileSize - uploadDetails.totalBytesSent);
}
#endregion
#region Read & hash File
// Generate SHA1 Chunk
// Open stream of the file
FileStream f = File.OpenRead(pathToFile);
//Seek to the location in the file we are currently up to
f.Seek(uploadDetails.totalBytesSent, SeekOrigin.Begin);
//Read the data from the file that we are going to use this time
f.Read(data, 0, (int)uploadDetails.bytesSentForPart);
//Create a blank SHA1 hash
SHA1 sha1 = SHA1.Create();
//Hash the bytes in our current data and keep the hash in hashData
byte[] hashData = sha1.ComputeHash(data, 0, (int)uploadDetails.bytesSentForPart);
//Dispose of the hash
sha1.Dispose();
//Create a string builder to manipulate the hash
StringBuilder sb = new StringBuilder();
//Add data to every byte in the range
foreach (byte b in hashData)
{
sb.Append(b.ToString("x2"));
}
//Close the file read because we now have the data
f.Close();
//Add the hash to the hash array
uploadDetails.partSha1Array.Add(sb.ToString());
#endregion
#region Finalise Operations on uploadDetails so we can release lock
//Get all the values we might need to use internally. OR just make a snapshot of Upload Details? (Yes this should work!)
uploadDetailsSnapshot = uploadDetails.CloneMe();
//Update the actual uploadDetails with what we intend to do.
//Increment the partNo
uploadDetails.partNo++;
//Increment the totalBytesSent
uploadDetails.totalBytesSent = uploadDetails.totalBytesSent + uploadDetails.bytesSentForPart;
#endregion
}
#endregion
//To count number of failed attempts
int WebRequestAttempt = 1;
//To allow retry of the failed request
RetryPartUpload:
//Output urls for debugging
//StaticHelpers.DebugLogger("UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug);
//Start a new web request
HttpWebRequest uploadPartRequest = (HttpWebRequest)WebRequest.Create(uploadPartsUrlDetails.uploadUrl);
//Set to post
uploadPartRequest.Method = "POST";
//Set the request timeout to 5 minutes
uploadPartRequest.Timeout = 5 * 60 * 1000;
//Set authorization token (using the one for the current uploadPartUrl)
//Intentionally generating error:
//uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken + "r");
uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken);
//Set the part number
uploadPartRequest.Headers.Add("X-Bz-Part-Number", uploadDetailsSnapshot.partNo.ToString());
//Set the sha1 hash from the array (minus one on the part number because 0-index array
uploadPartRequest.Headers.Add("X-Bz-Content-Sha1", (String)uploadDetailsSnapshot.partSha1Array[(uploadDetailsSnapshot.partNo - 1)]);
//Set content type to json
uploadPartRequest.ContentType = "application/json; charset=utf-8";
//Set the content length to the bytes sent for the part
uploadPartRequest.ContentLength = uploadDetailsSnapshot.bytesSentForPart;
//Create a stream to use for the uploadPartRequest (this may be the one to change to a filestream)
using (Stream stream = uploadPartRequest.GetRequestStream())
{
//Write the data (through the stream?) to the uploadPartRequest
stream.Write(data, 0, (int)uploadDetailsSnapshot.bytesSentForPart);
//Close the stream
stream.Close();
}
//Set upload response to null
HttpWebResponse uploadPartResponse = null;
//Verbose message
StaticHelpers.DebugLogger("Starting upload of part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose);
//Try the upload
try
{
//Try the upload and set the upload part response to the response
uploadPartResponse = (HttpWebResponse)uploadPartRequest.GetResponse();
}
//If theres an exception catch and output it
catch (WebException e)
{
if (e.Response == null)
{
StaticHelpers.DebugLogger("Upload has failed with error: " + e.Message, DebugLevel.Warn);
}
else
{
using (WebResponse r = e.Response)
{
HttpWebResponse httpResponse = (HttpWebResponse)r;
StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn);
using (Stream dataE = r.GetResponseStream())
using (var reader = new StreamReader(dataE))
{
string text = reader.ReadToEnd();
StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn);
}
}
}
//If we have failed less than 5 times
if (WebRequestAttempt < 5)
{
//Log a message
StaticHelpers.DebugLogger("Upload has failed, getting fresh uploadparturl and retrying....", DebugLevel.Verbose);
//Get our object containing our authorisation URLs
uploadPartsUrlDetails = await GetUploadPartUrl();
//Output fresh url for debugging
StaticHelpers.DebugLogger("Fresh UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug);
//Wait a while
int secToWait = WebRequestAttempt * 2;
Thread.Sleep(secToWait * 1000);
//Increment counter
WebRequestAttempt++;
//Retry
goto RetryPartUpload;
}
}
//Close the upload part response
uploadPartResponse.Close();
//Lock so we can work on uploadDetails
lock (uploadDetailsLock)
{
//Update uploadDetails with the fact this part has been completed
uploadDetails.BytesConfirmedSent = uploadDetails.BytesConfirmedSent + uploadDetailsSnapshot.bytesSentForPart;
//Calculate the decimal amount completed
double decimalPercentage = (double)uploadDetails.BytesConfirmedSent / (double)localFileSize;
//Calculate the percentage completed
int percentage = (int)(decimalPercentage * 100);
//Output to the console the percentage completed
StaticHelpers.UpdateSummary($"Progress: {percentage}%");
}
//Log to the debugger what part we've just done
StaticHelpers.DebugLogger("Uploaded Part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose);
}
#endregion
//Check whether we have finished or if just this thread being killed:
if (uploadDetails.totalBytesSent >= localFileSize)
{
//We have reached the end of parts to upload so tell the system not to upload anymore
noMoreThreads = true;
}
StaticHelpers.DebugLogger("Internal upload worker is dead.", DebugLevel.Verbose);
}
