/// <summary> /// Finishes the file upload by calling b2_finish_large_file to combine all parts. /// </summary> public void FinishLargeFile() { // Create a request object and copy it to the memory stream. B2FinishLargeFileRequest finishLargeFileData = new B2FinishLargeFileRequest { fileId = fileDetails.fileId, partSha1Array = uploadDetails.partSha1Array }; //So instead of using the old json serialized things going to use the new one... string FinishLargeFileInfoJsonString = JsonSerializer.Serialize <B2FinishLargeFileRequest>(finishLargeFileData); //Convert the string to a memory stream byte[] byteArray = Encoding.UTF8.GetBytes(FinishLargeFileInfoJsonString); MemoryStream finishLargeFileMemStream = new MemoryStream(byteArray); HttpWebRequest finishLargeFileRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_finish_large_file"); finishLargeFileRequest.Method = "POST"; finishLargeFileRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken); finishLargeFileRequest.ContentType = "application/json; charset=utf-8"; finishLargeFileRequest.ContentLength = finishLargeFileMemStream.Length; finishLargeFileMemStream.WriteTo(finishLargeFileRequest.GetRequestStream()); HttpWebResponse finishLargeFileResponse; try { finishLargeFileResponse = (HttpWebResponse)finishLargeFileRequest.GetResponse(); } catch (WebException e) { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } }
/// <summary> /// Runs & Manages the upload workers. /// </summary> public void RunUploadWorkers() { //Start the bandwidth monitor bandwidthMonitor.startMonitoring(); //Set WorkFinished to false bool WorkFinished = false; //Put up a blank status summary StaticHelpers.UpdateSummary($"Progress: 0%"); //Start a continious loop until we have finished the work while (WorkFinished == false) { //If we are under max threads and bandwidth monitor doesn't forbid it and we haven't already started all parts if (maxThreads > AllThreads.Count() && bandwidthMonitor.CanIncrease == true && noMoreThreads == false) { //Create thread to run StartUploadWorker Thread thread = new Thread(StartUploadWorker); //Start the thread thread.Start(); //Put the thread in our list of threads AllThreads.Add(thread); } //Recalculate number of active threads by removing inactive AllThreads.RemoveAll(thread => thread.ThreadState == System.Threading.ThreadState.Stopped); //Output a debug message StaticHelpers.DebugLogger($"Current number of threads = {AllThreads.Count}", DebugLevel.FullDebug); //If whole file has been uploaded and all threads are stopped if (uploadDetails.totalBytesSent >= localFileSize && AllThreads.Count() == 0) { //We have reached the end of parts to upload so tell the system not to upload anymore WorkFinished = true; } //Add a wait so we aren't too aggressive in adding threads Thread.Sleep(3000); } bandwidthMonitor.StopMonitoring = true; StaticHelpers.DebugLogger("Upload has finished.", DebugLevel.Verbose); }
/// <summary> /// Gets the uploadPartUrl for each worker function. /// </summary> /// <returns></returns> public async Task <UploadPartsUrlDetails> GetUploadPartUrl() { //Taken from https://www.backblaze.com/b2/docs/b2_get_upload_part_url.html with edits //Create an UploadPartsUrlDetails data object to hold the data UploadPartsUrlDetails uploadPartsUrlDetails = new UploadPartsUrlDetails(); // Get Upload URL String getUploadUrlJsonStr = "{\"fileId\":\"" + fileDetails.fileId + "\"}"; byte[] getUloadUrlJsonData = Encoding.UTF8.GetBytes(getUploadUrlJsonStr); //To allow us to count the number of failed attempts int WebRequestAttempt = 1; //To allow us to retry the webrequest if it fails (there is probably a better option here but I don't know it RetryRequest: HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_url"); //Intentionally generating an error for testing purposes //HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_ur"); getUploadUrlRequest.Method = "POST"; getUploadUrlRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken); getUploadUrlRequest.ContentType = "application/json; charset=utf-8"; getUploadUrlRequest.ContentLength = getUloadUrlJsonData.Length; using (Stream stream = getUploadUrlRequest.GetRequestStream()) { stream.Write(getUloadUrlJsonData, 0, getUloadUrlJsonData.Length); stream.Close(); } // Handle the response and print the json try { HttpWebResponse getUploadUrlResponse = (HttpWebResponse)getUploadUrlRequest.GetResponse(); //I have made heavy changes here so errors are almost certainly due to that! using (Stream responseStream = getUploadUrlResponse.GetResponseStream()) { UploadPartUrlResponse uploadPartUrlResponse = await JsonSerializer.DeserializeAsync <UploadPartUrlResponse>(responseStream); uploadPartsUrlDetails.authorizationToken = uploadPartUrlResponse.authorizationToken; uploadPartsUrlDetails.uploadUrl = uploadPartUrlResponse.uploadUrl; } getUploadUrlResponse.Close(); } catch (WebException e) { //Print error to console before retrying using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response) { StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {errorResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) { String text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("We have failed to get a part upload URL, retrying....", DebugLevel.Verbose); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Go back to retry the request goto RetryRequest; } } return(uploadPartsUrlDetails); }
/// <summary> /// Starts and is the upload worker that does the actual uploads. /// </summary> public async void StartUploadWorker() { StaticHelpers.DebugLogger("Starting an internal upload worker", DebugLevel.Verbose); //Get our object containing our authorisation URLs UploadPartsUrlDetails uploadPartsUrlDetails = await GetUploadPartUrl(); #region Loop where work is done //While there are bytes still be sent while (uploadDetails.totalBytesSent < localFileSize) { #region Check Bandwidth Monitor //If the bandwidth monitor requires a reduction in usage if (bandwidthMonitor.reduceUsage || bandwidthMonitor.urgentReduceUsage) { //Check thread count is greater than 1 if (AllThreads.Count(thread => thread.ThreadState != System.Threading.ThreadState.Stopped) > 1) { //Log to debug StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor. Killing self....", DebugLevel.Verbose); //Set reduceUsage to false bandwidthMonitor.reduceUsage = false; //Kill this thread break; } else { StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor HOWEVER as the only remaining thread I am ignoring.", DebugLevel.Verbose); } } #endregion #region Get details from uploadDetails & Data from file //Create variables outside of the lock so we can set it inside but still access it outside //For a snapshot of uploadDetails UploadDetails uploadDetailsSnapshot = new UploadDetails(); //Create the byte array for the data we are going to use byte[] data = new byte[Singletons.options.PartSize * (1000 * 1000)]; //Lock the uploadDetails lock (uploadDetailsLock) { #region Check if we are on the last part or even if there are no parts left //If there is nothing left to upload if ((localFileSize - uploadDetails.totalBytesSent) == 0) { //Break out of the loop as there is no more work to do break; } //If the remaining bytes are less the minimum part size if ((localFileSize - uploadDetails.totalBytesSent) <= uploadDetails.minimumPartSize) { //Changes the bytes sent for part to the remaining number of bytes uploadDetails.bytesSentForPart = (localFileSize - uploadDetails.totalBytesSent); } #endregion #region Read & hash File // Generate SHA1 Chunk // Open stream of the file FileStream f = File.OpenRead(pathToFile); //Seek to the location in the file we are currently up to f.Seek(uploadDetails.totalBytesSent, SeekOrigin.Begin); //Read the data from the file that we are going to use this time f.Read(data, 0, (int)uploadDetails.bytesSentForPart); //Create a blank SHA1 hash SHA1 sha1 = SHA1.Create(); //Hash the bytes in our current data and keep the hash in hashData byte[] hashData = sha1.ComputeHash(data, 0, (int)uploadDetails.bytesSentForPart); //Dispose of the hash sha1.Dispose(); //Create a string builder to manipulate the hash StringBuilder sb = new StringBuilder(); //Add data to every byte in the range foreach (byte b in hashData) { sb.Append(b.ToString("x2")); } //Close the file read because we now have the data f.Close(); //Add the hash to the hash array uploadDetails.partSha1Array.Add(sb.ToString()); #endregion #region Finalise Operations on uploadDetails so we can release lock //Get all the values we might need to use internally. OR just make a snapshot of Upload Details? (Yes this should work!) uploadDetailsSnapshot = uploadDetails.CloneMe(); //Update the actual uploadDetails with what we intend to do. //Increment the partNo uploadDetails.partNo++; //Increment the totalBytesSent uploadDetails.totalBytesSent = uploadDetails.totalBytesSent + uploadDetails.bytesSentForPart; #endregion } #endregion //To count number of failed attempts int WebRequestAttempt = 1; //To allow retry of the failed request RetryPartUpload: //Output urls for debugging //StaticHelpers.DebugLogger("UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug); //Start a new web request HttpWebRequest uploadPartRequest = (HttpWebRequest)WebRequest.Create(uploadPartsUrlDetails.uploadUrl); //Set to post uploadPartRequest.Method = "POST"; //Set the request timeout to 5 minutes uploadPartRequest.Timeout = 5 * 60 * 1000; //Set authorization token (using the one for the current uploadPartUrl) //Intentionally generating error: //uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken + "r"); uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken); //Set the part number uploadPartRequest.Headers.Add("X-Bz-Part-Number", uploadDetailsSnapshot.partNo.ToString()); //Set the sha1 hash from the array (minus one on the part number because 0-index array uploadPartRequest.Headers.Add("X-Bz-Content-Sha1", (String)uploadDetailsSnapshot.partSha1Array[(uploadDetailsSnapshot.partNo - 1)]); //Set content type to json uploadPartRequest.ContentType = "application/json; charset=utf-8"; //Set the content length to the bytes sent for the part uploadPartRequest.ContentLength = uploadDetailsSnapshot.bytesSentForPart; //Create a stream to use for the uploadPartRequest (this may be the one to change to a filestream) using (Stream stream = uploadPartRequest.GetRequestStream()) { //Write the data (through the stream?) to the uploadPartRequest stream.Write(data, 0, (int)uploadDetailsSnapshot.bytesSentForPart); //Close the stream stream.Close(); } //Set upload response to null HttpWebResponse uploadPartResponse = null; //Verbose message StaticHelpers.DebugLogger("Starting upload of part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose); //Try the upload try { //Try the upload and set the upload part response to the response uploadPartResponse = (HttpWebResponse)uploadPartRequest.GetResponse(); } //If theres an exception catch and output it catch (WebException e) { if (e.Response == null) { StaticHelpers.DebugLogger("Upload has failed with error: " + e.Message, DebugLevel.Warn); } else { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("Upload has failed, getting fresh uploadparturl and retrying....", DebugLevel.Verbose); //Get our object containing our authorisation URLs uploadPartsUrlDetails = await GetUploadPartUrl(); //Output fresh url for debugging StaticHelpers.DebugLogger("Fresh UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Retry goto RetryPartUpload; } } //Close the upload part response uploadPartResponse.Close(); //Lock so we can work on uploadDetails lock (uploadDetailsLock) { //Update uploadDetails with the fact this part has been completed uploadDetails.BytesConfirmedSent = uploadDetails.BytesConfirmedSent + uploadDetailsSnapshot.bytesSentForPart; //Calculate the decimal amount completed double decimalPercentage = (double)uploadDetails.BytesConfirmedSent / (double)localFileSize; //Calculate the percentage completed int percentage = (int)(decimalPercentage * 100); //Output to the console the percentage completed StaticHelpers.UpdateSummary($"Progress: {percentage}%"); } //Log to the debugger what part we've just done StaticHelpers.DebugLogger("Uploaded Part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose); } #endregion //Check whether we have finished or if just this thread being killed: if (uploadDetails.totalBytesSent >= localFileSize) { //We have reached the end of parts to upload so tell the system not to upload anymore noMoreThreads = true; } StaticHelpers.DebugLogger("Internal upload worker is dead.", DebugLevel.Verbose); }
/// <summary> /// Confirms a file exists after upload to confirm success. /// </summary> /// <param name="fileId">Id of the file to confirm exists.</param> /// <returns></returns> public bool ConfirmFileExists(string fileId) { //Set variable and goto allowing repeat of process int WebRequestAttempt = 1; RetryFileCheck: HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(authenticationDetails.apiUrl + "/b2api/v2/b2_get_file_info"); string body = "{\"fileId\":\"" + fileId + "\"}"; var data = Encoding.UTF8.GetBytes(body); webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", authenticationDetails.authorizationToken); webRequest.ContentType = "application/json; charset=utf-8"; webRequest.ContentLength = data.Length; using (var stream = webRequest.GetRequestStream()) { stream.Write(data, 0, data.Length); stream.Close(); } try { WebResponse response = (HttpWebResponse)webRequest.GetResponse(); var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); response.Close(); //Console.WriteLine(responseString); //If we haven't had an exception by this point we have a code 200 response meaning file does exist so we can return true return(true); //If theres an exception catch and output it } catch (WebException e) { if (e.Response == null) { StaticHelpers.DebugLogger("Upload has failed with error: " + e.Message, DebugLevel.Error); } else { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("Cannot get upload success confirmation, retrying....", DebugLevel.Verbose); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Retry goto RetryFileCheck; } //If we have got to this point everything has failed for this request and we need to die StaticHelpers.DebugLogger("Sorry, we can't confirm the file has been uploaded successfully, please check manually. fileId: " + fileId, DebugLevel.Error); return(false); } }
/// <summary> /// Monitor function, runs in its own thread doing the monitoring. /// </summary> private void runMonitor() { //So... we will establish a baseline over 5 seconds. Then enter that as our baseline. // an increase from the baseline of 50% or more will be seen as needing us to back off // a failure of a ping will be a full retreat (killing 25% of runnign threads) //Spend 5 seconds looping to get a baseline (done separately to avoid a bunch of extra checks below) for (int i = 0; i < (5000 / pingInterval); i++) { //Add to the list PingReply pingReply = DoPing(); //Add to our ping list Last5Seconds.Add((int)pingReply.RoundtripTime); //Sleep if ping time has been less than the interval time if (pingReply.RoundtripTime < pingInterval) { //Sleep however long is required to have 250ms between pings Thread.Sleep(pingInterval - (int)pingReply.RoundtripTime); } } //Calculate the lowbaseline int LowBaseLine = (int)Last5Seconds.Average() * 5; //Output StaticHelpers.DebugLogger($"Test ping average is: {Last5Seconds.Average()}. Baseline set at: {LowBaseLine}", DebugLevel.Verbose); while (StopMonitoring == false) { //Run the ping PingReply pingReply = DoPing(); //Remove the oldest value and add the new one Last5Seconds.RemoveAt(0); Last5Seconds.Add((int)pingReply.RoundtripTime); //Output the returned ping amount (disabled even for full debug as it generates an insane amount of noise //StaticHelpers.DebugLogger($"Ping roundtrip: {pingReply.RoundtripTime}", DebugLevel.FullDebug); //Check if we need to make a change to our reduceUssage flag if (Last5Seconds.Average() > LowBaseLine) { //Set flag to pull back reduceUsage = true; //Set lastHighAverage to now LastHighAverage = DateTime.Now; //Output thats what we want to log StaticHelpers.DebugLogger($"Current Average Ping: {Last5Seconds.Average()}. Baseline is {LowBaseLine}. Request reduction in threads...", DebugLevel.Verbose); } //If we have a result of zero if (pingReply.RoundtripTime == 0) { //Get the last2Results into a list var Last3Results = Last5Seconds.TakeLast <int>(3); //If the last 2 have been 0 meaning we've had it 3 in a row if (Last3Results.Average() == 0) { //Set urgentReduceUsage to true urgentReduceUsage = true; //Set this as te time we've most recently requested a reduction in threads LastFailedPing = DateTime.Now; //Output thats what we want StaticHelpers.DebugLogger($"Average of last 3 pings: {Last3Results.Average()}. Request urgent reduction in threads...", DebugLevel.Verbose); } } else { //Make sure we aren't requesting an urgent reduction urgentReduceUsage = false; } //Check if the last failed ping was within the last 30 second and prevent an increase in that time int LastFailedPingInSeconds = (int)(DateTime.Now - LastFailedPing).TotalSeconds; int LastHighAverageInSeconds = (int)(DateTime.Now - LastHighAverage).TotalSeconds; if (LastFailedPingInSeconds < 30 || LastHighAverageInSeconds < 30) { CanIncrease = false; } else { CanIncrease = true; } //Sleep if ping time has been less than the interval time if (pingReply.RoundtripTime < pingInterval) { //Sleep however long is required to have 250ms between pings Thread.Sleep(pingInterval - (int)pingReply.RoundtripTime); } } StaticHelpers.DebugLogger($"Bandwidth Monitor has shutdown.", DebugLevel.Verbose); }