/// <summary> /// Fetches the bucketId from Backblaze based on the name provided by the user. /// </summary> /// <param name="BucketName">The name of the bucket to get the id for</param> public async void GetBucketId(string BucketName) { HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(authenticationDetails.apiUrl + "/b2api/v2/b2_list_buckets"); string body = "{\"accountId\":\"" + authenticationDetails.accountId + "\", \"bucketName\":\"" + BucketName + "\"}"; var data = Encoding.UTF8.GetBytes(body); webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", authenticationDetails.authorizationToken); webRequest.ContentType = "application/json; charset=utf-8"; webRequest.ContentLength = data.Length; using (var stream = webRequest.GetRequestStream()) { stream.Write(data, 0, data.Length); stream.Close(); } //Get the response and then a stream of that response WebResponse response = (HttpWebResponse)webRequest.GetResponse(); Stream responseStream = response.GetResponseStream(); //Deserialize onto ListBuckets object as BucketsList ListBuckets BucketsList = await JsonSerializer.DeserializeAsync <ListBuckets>(responseStream); //Check only 1 has been returned if (BucketsList.buckets.Count() != 1) { //Generate a fatal error and die StaticHelpers.DebugLogger("Bucket not found. Exiting.", DebugLevel.Error); } authenticationDetails.bucketId = BucketsList.buckets.First().bucketId; //var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); //response.Close(); //Console.WriteLine(responseString); }
/// <summary> /// Manages every aspect of the upload from start to finish. /// </summary> public void UploadFile() { //Create a timestamp for logging time spent DateTime Start = DateTime.Now; //Create a new upload details to hold the details of the upload uploadDetails = new UploadDetails(); //Start large file upload StartLargeFile(); //Upload the parts RunUploadWorkers(); //Finish Large File upload FinishLargeFile(); #region Output final status message. //Get end datetime DateTime End = DateTime.Now; //Get the difference between the two as a string string diffInSeconds = Math.Round((End - Start).TotalSeconds, 1).ToString(); //Get Mbps //First get MB var MBs = fileInfo.Length * 0.00000095367432; //Then mb var Mbs = MBs * 8; //Then calculate Mbps var Mbps = Mbs / (End - Start).TotalSeconds; StaticHelpers.DebugLogger($"Operation Finished. Operation Took: {diffInSeconds} seconds and transferred {Math.Round(MBs, 2)}MBs at a speed of {Math.Round(Mbps, 2)}Mbps", DebugLevel.Info); #endregion }
static void OptionsToOptions(Options opts) { //Validate any options that need validating: if (File.Exists(opts.filePath) == false) { //Throw an error cause the file doesn't exist, for now just write to console. StaticHelpers.DebugLogger("The file specified does not exist! File specified was: " + Singletons.options.filePath, DebugLevel.Error); } //Set options to our singleton Singletons.options = opts; }
/// <summary> /// Starts the bandwidth monitor /// </summary> public void startMonitoring() { //Log to debug StaticHelpers.DebugLogger("Starting bandwidth monitor, establishing baseline....", DebugLevel.Verbose); //Create thread to run StartUploadWorker Thread thread = new Thread(runMonitor); //Start the thread thread.Start(); //Sleep for 5 seconds to form a baseline Thread.Sleep(5 * 1000); }
/// <summary> /// Starts a multipart upload. /// </summary> /// <param name="pathToFile">Full path to the file to upload.</param> public void MultiPartUpload(string pathToFile) { //Print info message StaticHelpers.DebugLogger($"Starting a multipart upload on: " + pathToFile, DebugLevel.Info); //Get a new PartUploader MultiPartUpload multiPartUpload = new MultiPartUpload(pathToFile); //Begin file upload multiPartUpload.UploadFile(); //Check file upload has been successful: if (ConfirmFileExists(multiPartUpload.fileDetails.fileId)) { StaticHelpers.DebugLogger("File upload has been confirmed."); } }
/* * Test Notes * With Single Thread and 20MB file: Test Took: 50.3603453 seconds * With Single Thread and 20MB file: Test Took: 44.648904099999996 seconds * With 1 thread with 100MB file: Test Took: 220.0786028 seconds * With 3 threads with 100MB file: Test Took: 170.9150036 seconds * With 10 threads with 100MB file: Test Took: 95.6517026 seconds * With 13 threads (max for chunk size) with 100MB file: Test Took: 93.2367336 seconds * * * TODO: Confirm file is there before exiting * */ static void Main(string[] args) { //Get start datetime to calculate time taken DateTime Start = DateTime.Now; //Place the cursor bottom left of the window so summary is always there and status messages roll up Console.CursorTop = Console.WindowTop + Console.WindowHeight - 1; //Parse commandline options This is using this parser (may or may not be best but works for now): https://github.com/commandlineparser/commandline var ParserResult = Parser.Default.ParseArguments <Options>(args).WithParsed <Options>(OptionsToOptions); //Exit if parsed options not sufficient if (ParserResult.Tag.ToString() == "NotParsed") { //Check whether help has been requested... foreach (string arg in args) { //Check if it contains "help" if (arg.Contains("help")) { //If it has exit with a success code Environment.Exit(0); } } //If helps hasn't been requested exit with an error that paramaters are missing. StaticHelpers.DebugLogger("Incorrect or missing commandline paramaters, please check and try again.", DebugLevel.Error); } BackblazeApi Backblaze = new BackblazeApi(); //Login to B2 Backblaze.AuthorizeWithB2(); //Get BucketId Backblaze.GetBucketId(Singletons.options.bucketName); //Start a multipart upload Backblaze.MultiPartUpload(Singletons.options.filePath); //Get end datetime DateTime End = DateTime.Now; //Get the difference between the two as a string string diffInSeconds = (End - Start).TotalSeconds.ToString(); StaticHelpers.DebugLogger("Total program runtime: " + diffInSeconds + " seconds", DebugLevel.FullDebug); //Console.ReadLine(); //Kill the program with a success exit code (needed because of threads left running): Environment.Exit(0); }
/// <summary> /// Authorizes with Backblaze using the API credentials supplised by the user. Results go into <see cref="Singletons.authenticationDetails"/> /// </summary> public async void AuthorizeWithB2() { //Craete a webrequest HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create("https://api.backblazeb2.com/b2api/v2/b2_authorize_account"); //Encode given credentials String credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(Singletons.options.applicationKeyId + ":" + Singletons.options.applicationKey)); //Add authentication details to header webRequest.Headers.Add("Authorization", "Basic " + credentials); //Set request ContentType webRequest.ContentType = "application/json; charset=utf-8"; // Handle the response and push the json to authenticationDetails try { HttpWebResponse authenticationResponse = (HttpWebResponse)webRequest.GetResponse(); //I have made heavy changes here so errors are almost certainly due to that! using (Stream responseStream = authenticationResponse.GetResponseStream()) { //String json = new StreamReader(authenticationResponse.GetResponseStream()).ReadToEnd(); //Deserialize and return us the authenticationDetails object AuthenticationDetails newAuthenticationDetails = await JsonSerializer.DeserializeAsync <AuthenticationDetails>(responseStream); //Set to the singleton instance of authenticationDetails /*this.authenticationDetails.apiUrl = authenticationDetails.apiUrl; * this.authenticationDetails.authorizationToken = authenticationDetails.authorizationToken;*/ //Set the singleton insstance to the one we just got Singletons.authenticationDetails = newAuthenticationDetails; //Set our instance to the singleton instance authenticationDetails = Singletons.authenticationDetails; } authenticationResponse.Close(); StaticHelpers.DebugLogger("Successfully authenticated with Backblaze.", DebugLevel.Verbose); } catch (WebException e) { using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response) { StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {errorResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) { String text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } }
/// <summary> /// Calls b2_start_large_file and gets the required idents to begin upload. /// </summary> public async void StartLargeFile() { // Setup JSON to post. String startLargeFileJsonStr = "{\"bucketId\":\"" + Singletons.authenticationDetails.bucketId + "\",\"fileName\":\"" + fileName + "\",\"contentType\":\"" + fileMime + "\"}"; byte[] jsonData = Encoding.UTF8.GetBytes(startLargeFileJsonStr); // Send over the wire HttpWebRequest startLargeFileRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_start_large_file"); startLargeFileRequest.Method = "POST"; startLargeFileRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken); startLargeFileRequest.ContentType = "application/json; charset=utf-8"; startLargeFileRequest.ContentLength = jsonData.Length; using (Stream stream = startLargeFileRequest.GetRequestStream()) { stream.Write(jsonData, 0, jsonData.Length); stream.Close(); } // Handle the response and print the json try { HttpWebResponse startLargeFileResponse = (HttpWebResponse)startLargeFileRequest.GetResponse(); //Trying swapping this so we get a stream for deserialization //using (StringReader responseReader = new StringReader(new StreamReader(startLargeFileResponse.GetResponseStream()).ReadToEnd())) using (Stream responseStream = startLargeFileResponse.GetResponseStream()) { fileDetails = await JsonSerializer.DeserializeAsync <FileDetails>(responseStream); } startLargeFileResponse.Close(); } catch (WebException e) { using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response) { StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {errorResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) { String text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } }
/// <summary> /// Finishes the file upload by calling b2_finish_large_file to combine all parts. /// </summary> public void FinishLargeFile() { // Create a request object and copy it to the memory stream. B2FinishLargeFileRequest finishLargeFileData = new B2FinishLargeFileRequest { fileId = fileDetails.fileId, partSha1Array = uploadDetails.partSha1Array }; //So instead of using the old json serialized things going to use the new one... string FinishLargeFileInfoJsonString = JsonSerializer.Serialize <B2FinishLargeFileRequest>(finishLargeFileData); //Convert the string to a memory stream byte[] byteArray = Encoding.UTF8.GetBytes(FinishLargeFileInfoJsonString); MemoryStream finishLargeFileMemStream = new MemoryStream(byteArray); HttpWebRequest finishLargeFileRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_finish_large_file"); finishLargeFileRequest.Method = "POST"; finishLargeFileRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken); finishLargeFileRequest.ContentType = "application/json; charset=utf-8"; finishLargeFileRequest.ContentLength = finishLargeFileMemStream.Length; finishLargeFileMemStream.WriteTo(finishLargeFileRequest.GetRequestStream()); HttpWebResponse finishLargeFileResponse; try { finishLargeFileResponse = (HttpWebResponse)finishLargeFileRequest.GetResponse(); } catch (WebException e) { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } }
/// <summary> /// Runs & Manages the upload workers. /// </summary> public void RunUploadWorkers() { //Start the bandwidth monitor bandwidthMonitor.startMonitoring(); //Set WorkFinished to false bool WorkFinished = false; //Put up a blank status summary StaticHelpers.UpdateSummary($"Progress: 0%"); //Start a continious loop until we have finished the work while (WorkFinished == false) { //If we are under max threads and bandwidth monitor doesn't forbid it and we haven't already started all parts if (maxThreads > AllThreads.Count() && bandwidthMonitor.CanIncrease == true && noMoreThreads == false) { //Create thread to run StartUploadWorker Thread thread = new Thread(StartUploadWorker); //Start the thread thread.Start(); //Put the thread in our list of threads AllThreads.Add(thread); } //Recalculate number of active threads by removing inactive AllThreads.RemoveAll(thread => thread.ThreadState == System.Threading.ThreadState.Stopped); //Output a debug message StaticHelpers.DebugLogger($"Current number of threads = {AllThreads.Count}", DebugLevel.FullDebug); //If whole file has been uploaded and all threads are stopped if (uploadDetails.totalBytesSent >= localFileSize && AllThreads.Count() == 0) { //We have reached the end of parts to upload so tell the system not to upload anymore WorkFinished = true; } //Add a wait so we aren't too aggressive in adding threads Thread.Sleep(3000); } bandwidthMonitor.StopMonitoring = true; StaticHelpers.DebugLogger("Upload has finished.", DebugLevel.Verbose); }
/// <summary> /// Gets the uploadPartUrl for each worker function. /// </summary> /// <returns></returns> public async Task <UploadPartsUrlDetails> GetUploadPartUrl() { //Taken from https://www.backblaze.com/b2/docs/b2_get_upload_part_url.html with edits //Create an UploadPartsUrlDetails data object to hold the data UploadPartsUrlDetails uploadPartsUrlDetails = new UploadPartsUrlDetails(); // Get Upload URL String getUploadUrlJsonStr = "{\"fileId\":\"" + fileDetails.fileId + "\"}"; byte[] getUloadUrlJsonData = Encoding.UTF8.GetBytes(getUploadUrlJsonStr); //To allow us to count the number of failed attempts int WebRequestAttempt = 1; //To allow us to retry the webrequest if it fails (there is probably a better option here but I don't know it RetryRequest: HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_url"); //Intentionally generating an error for testing purposes //HttpWebRequest getUploadUrlRequest = (HttpWebRequest)WebRequest.Create(Singletons.authenticationDetails.apiUrl + "/b2api/v2/b2_get_upload_part_ur"); getUploadUrlRequest.Method = "POST"; getUploadUrlRequest.Headers.Add("Authorization", Singletons.authenticationDetails.authorizationToken); getUploadUrlRequest.ContentType = "application/json; charset=utf-8"; getUploadUrlRequest.ContentLength = getUloadUrlJsonData.Length; using (Stream stream = getUploadUrlRequest.GetRequestStream()) { stream.Write(getUloadUrlJsonData, 0, getUloadUrlJsonData.Length); stream.Close(); } // Handle the response and print the json try { HttpWebResponse getUploadUrlResponse = (HttpWebResponse)getUploadUrlRequest.GetResponse(); //I have made heavy changes here so errors are almost certainly due to that! using (Stream responseStream = getUploadUrlResponse.GetResponseStream()) { UploadPartUrlResponse uploadPartUrlResponse = await JsonSerializer.DeserializeAsync <UploadPartUrlResponse>(responseStream); uploadPartsUrlDetails.authorizationToken = uploadPartUrlResponse.authorizationToken; uploadPartsUrlDetails.uploadUrl = uploadPartUrlResponse.uploadUrl; } getUploadUrlResponse.Close(); } catch (WebException e) { //Print error to console before retrying using (HttpWebResponse errorResponse = (HttpWebResponse)e.Response) { StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {errorResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (StreamReader reader = new StreamReader(errorResponse.GetResponseStream())) { String text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("We have failed to get a part upload URL, retrying....", DebugLevel.Verbose); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Go back to retry the request goto RetryRequest; } } return(uploadPartsUrlDetails); }
/// <summary> /// Starts and is the upload worker that does the actual uploads. /// </summary> public async void StartUploadWorker() { StaticHelpers.DebugLogger("Starting an internal upload worker", DebugLevel.Verbose); //Get our object containing our authorisation URLs UploadPartsUrlDetails uploadPartsUrlDetails = await GetUploadPartUrl(); #region Loop where work is done //While there are bytes still be sent while (uploadDetails.totalBytesSent < localFileSize) { #region Check Bandwidth Monitor //If the bandwidth monitor requires a reduction in usage if (bandwidthMonitor.reduceUsage || bandwidthMonitor.urgentReduceUsage) { //Check thread count is greater than 1 if (AllThreads.Count(thread => thread.ThreadState != System.Threading.ThreadState.Stopped) > 1) { //Log to debug StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor. Killing self....", DebugLevel.Verbose); //Set reduceUsage to false bandwidthMonitor.reduceUsage = false; //Kill this thread break; } else { StaticHelpers.DebugLogger("Received Kill Request from Bandwidth Monitor HOWEVER as the only remaining thread I am ignoring.", DebugLevel.Verbose); } } #endregion #region Get details from uploadDetails & Data from file //Create variables outside of the lock so we can set it inside but still access it outside //For a snapshot of uploadDetails UploadDetails uploadDetailsSnapshot = new UploadDetails(); //Create the byte array for the data we are going to use byte[] data = new byte[Singletons.options.PartSize * (1000 * 1000)]; //Lock the uploadDetails lock (uploadDetailsLock) { #region Check if we are on the last part or even if there are no parts left //If there is nothing left to upload if ((localFileSize - uploadDetails.totalBytesSent) == 0) { //Break out of the loop as there is no more work to do break; } //If the remaining bytes are less the minimum part size if ((localFileSize - uploadDetails.totalBytesSent) <= uploadDetails.minimumPartSize) { //Changes the bytes sent for part to the remaining number of bytes uploadDetails.bytesSentForPart = (localFileSize - uploadDetails.totalBytesSent); } #endregion #region Read & hash File // Generate SHA1 Chunk // Open stream of the file FileStream f = File.OpenRead(pathToFile); //Seek to the location in the file we are currently up to f.Seek(uploadDetails.totalBytesSent, SeekOrigin.Begin); //Read the data from the file that we are going to use this time f.Read(data, 0, (int)uploadDetails.bytesSentForPart); //Create a blank SHA1 hash SHA1 sha1 = SHA1.Create(); //Hash the bytes in our current data and keep the hash in hashData byte[] hashData = sha1.ComputeHash(data, 0, (int)uploadDetails.bytesSentForPart); //Dispose of the hash sha1.Dispose(); //Create a string builder to manipulate the hash StringBuilder sb = new StringBuilder(); //Add data to every byte in the range foreach (byte b in hashData) { sb.Append(b.ToString("x2")); } //Close the file read because we now have the data f.Close(); //Add the hash to the hash array uploadDetails.partSha1Array.Add(sb.ToString()); #endregion #region Finalise Operations on uploadDetails so we can release lock //Get all the values we might need to use internally. OR just make a snapshot of Upload Details? (Yes this should work!) uploadDetailsSnapshot = uploadDetails.CloneMe(); //Update the actual uploadDetails with what we intend to do. //Increment the partNo uploadDetails.partNo++; //Increment the totalBytesSent uploadDetails.totalBytesSent = uploadDetails.totalBytesSent + uploadDetails.bytesSentForPart; #endregion } #endregion //To count number of failed attempts int WebRequestAttempt = 1; //To allow retry of the failed request RetryPartUpload: //Output urls for debugging //StaticHelpers.DebugLogger("UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug); //Start a new web request HttpWebRequest uploadPartRequest = (HttpWebRequest)WebRequest.Create(uploadPartsUrlDetails.uploadUrl); //Set to post uploadPartRequest.Method = "POST"; //Set the request timeout to 5 minutes uploadPartRequest.Timeout = 5 * 60 * 1000; //Set authorization token (using the one for the current uploadPartUrl) //Intentionally generating error: //uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken + "r"); uploadPartRequest.Headers.Add("Authorization", uploadPartsUrlDetails.authorizationToken); //Set the part number uploadPartRequest.Headers.Add("X-Bz-Part-Number", uploadDetailsSnapshot.partNo.ToString()); //Set the sha1 hash from the array (minus one on the part number because 0-index array uploadPartRequest.Headers.Add("X-Bz-Content-Sha1", (String)uploadDetailsSnapshot.partSha1Array[(uploadDetailsSnapshot.partNo - 1)]); //Set content type to json uploadPartRequest.ContentType = "application/json; charset=utf-8"; //Set the content length to the bytes sent for the part uploadPartRequest.ContentLength = uploadDetailsSnapshot.bytesSentForPart; //Create a stream to use for the uploadPartRequest (this may be the one to change to a filestream) using (Stream stream = uploadPartRequest.GetRequestStream()) { //Write the data (through the stream?) to the uploadPartRequest stream.Write(data, 0, (int)uploadDetailsSnapshot.bytesSentForPart); //Close the stream stream.Close(); } //Set upload response to null HttpWebResponse uploadPartResponse = null; //Verbose message StaticHelpers.DebugLogger("Starting upload of part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose); //Try the upload try { //Try the upload and set the upload part response to the response uploadPartResponse = (HttpWebResponse)uploadPartRequest.GetResponse(); } //If theres an exception catch and output it catch (WebException e) { if (e.Response == null) { StaticHelpers.DebugLogger("Upload has failed with error: " + e.Message, DebugLevel.Warn); } else { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("Upload has failed, getting fresh uploadparturl and retrying....", DebugLevel.Verbose); //Get our object containing our authorisation URLs uploadPartsUrlDetails = await GetUploadPartUrl(); //Output fresh url for debugging StaticHelpers.DebugLogger("Fresh UploadPartsURL is: " + uploadPartsUrlDetails.uploadUrl + ". Key is: " + uploadPartsUrlDetails.authorizationToken, DebugLevel.FullDebug); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Retry goto RetryPartUpload; } } //Close the upload part response uploadPartResponse.Close(); //Lock so we can work on uploadDetails lock (uploadDetailsLock) { //Update uploadDetails with the fact this part has been completed uploadDetails.BytesConfirmedSent = uploadDetails.BytesConfirmedSent + uploadDetailsSnapshot.bytesSentForPart; //Calculate the decimal amount completed double decimalPercentage = (double)uploadDetails.BytesConfirmedSent / (double)localFileSize; //Calculate the percentage completed int percentage = (int)(decimalPercentage * 100); //Output to the console the percentage completed StaticHelpers.UpdateSummary($"Progress: {percentage}%"); } //Log to the debugger what part we've just done StaticHelpers.DebugLogger("Uploaded Part " + uploadDetailsSnapshot.partNo, DebugLevel.Verbose); } #endregion //Check whether we have finished or if just this thread being killed: if (uploadDetails.totalBytesSent >= localFileSize) { //We have reached the end of parts to upload so tell the system not to upload anymore noMoreThreads = true; } StaticHelpers.DebugLogger("Internal upload worker is dead.", DebugLevel.Verbose); }
/// <summary> /// Confirms a file exists after upload to confirm success. /// </summary> /// <param name="fileId">Id of the file to confirm exists.</param> /// <returns></returns> public bool ConfirmFileExists(string fileId) { //Set variable and goto allowing repeat of process int WebRequestAttempt = 1; RetryFileCheck: HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(authenticationDetails.apiUrl + "/b2api/v2/b2_get_file_info"); string body = "{\"fileId\":\"" + fileId + "\"}"; var data = Encoding.UTF8.GetBytes(body); webRequest.Method = "POST"; webRequest.Headers.Add("Authorization", authenticationDetails.authorizationToken); webRequest.ContentType = "application/json; charset=utf-8"; webRequest.ContentLength = data.Length; using (var stream = webRequest.GetRequestStream()) { stream.Write(data, 0, data.Length); stream.Close(); } try { WebResponse response = (HttpWebResponse)webRequest.GetResponse(); var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd(); response.Close(); //Console.WriteLine(responseString); //If we haven't had an exception by this point we have a code 200 response meaning file does exist so we can return true return(true); //If theres an exception catch and output it } catch (WebException e) { if (e.Response == null) { StaticHelpers.DebugLogger("Upload has failed with error: " + e.Message, DebugLevel.Error); } else { using (WebResponse r = e.Response) { HttpWebResponse httpResponse = (HttpWebResponse)r; StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {httpResponse.StatusCode}. Retrying....", DebugLevel.Warn); using (Stream dataE = r.GetResponseStream()) using (var reader = new StreamReader(dataE)) { string text = reader.ReadToEnd(); StaticHelpers.DebugLogger($"Internal Worker Error with API.Error code: {text}. Retrying....", DebugLevel.Warn); } } } //If we have failed less than 5 times if (WebRequestAttempt < 5) { //Log a message StaticHelpers.DebugLogger("Cannot get upload success confirmation, retrying....", DebugLevel.Verbose); //Wait a while int secToWait = WebRequestAttempt * 2; Thread.Sleep(secToWait * 1000); //Increment counter WebRequestAttempt++; //Retry goto RetryFileCheck; } //If we have got to this point everything has failed for this request and we need to die StaticHelpers.DebugLogger("Sorry, we can't confirm the file has been uploaded successfully, please check manually. fileId: " + fileId, DebugLevel.Error); return(false); } }
/// <summary> /// Monitor function, runs in its own thread doing the monitoring. /// </summary> private void runMonitor() { //So... we will establish a baseline over 5 seconds. Then enter that as our baseline. // an increase from the baseline of 50% or more will be seen as needing us to back off // a failure of a ping will be a full retreat (killing 25% of runnign threads) //Spend 5 seconds looping to get a baseline (done separately to avoid a bunch of extra checks below) for (int i = 0; i < (5000 / pingInterval); i++) { //Add to the list PingReply pingReply = DoPing(); //Add to our ping list Last5Seconds.Add((int)pingReply.RoundtripTime); //Sleep if ping time has been less than the interval time if (pingReply.RoundtripTime < pingInterval) { //Sleep however long is required to have 250ms between pings Thread.Sleep(pingInterval - (int)pingReply.RoundtripTime); } } //Calculate the lowbaseline int LowBaseLine = (int)Last5Seconds.Average() * 5; //Output StaticHelpers.DebugLogger($"Test ping average is: {Last5Seconds.Average()}. Baseline set at: {LowBaseLine}", DebugLevel.Verbose); while (StopMonitoring == false) { //Run the ping PingReply pingReply = DoPing(); //Remove the oldest value and add the new one Last5Seconds.RemoveAt(0); Last5Seconds.Add((int)pingReply.RoundtripTime); //Output the returned ping amount (disabled even for full debug as it generates an insane amount of noise //StaticHelpers.DebugLogger($"Ping roundtrip: {pingReply.RoundtripTime}", DebugLevel.FullDebug); //Check if we need to make a change to our reduceUssage flag if (Last5Seconds.Average() > LowBaseLine) { //Set flag to pull back reduceUsage = true; //Set lastHighAverage to now LastHighAverage = DateTime.Now; //Output thats what we want to log StaticHelpers.DebugLogger($"Current Average Ping: {Last5Seconds.Average()}. Baseline is {LowBaseLine}. Request reduction in threads...", DebugLevel.Verbose); } //If we have a result of zero if (pingReply.RoundtripTime == 0) { //Get the last2Results into a list var Last3Results = Last5Seconds.TakeLast <int>(3); //If the last 2 have been 0 meaning we've had it 3 in a row if (Last3Results.Average() == 0) { //Set urgentReduceUsage to true urgentReduceUsage = true; //Set this as te time we've most recently requested a reduction in threads LastFailedPing = DateTime.Now; //Output thats what we want StaticHelpers.DebugLogger($"Average of last 3 pings: {Last3Results.Average()}. Request urgent reduction in threads...", DebugLevel.Verbose); } } else { //Make sure we aren't requesting an urgent reduction urgentReduceUsage = false; } //Check if the last failed ping was within the last 30 second and prevent an increase in that time int LastFailedPingInSeconds = (int)(DateTime.Now - LastFailedPing).TotalSeconds; int LastHighAverageInSeconds = (int)(DateTime.Now - LastHighAverage).TotalSeconds; if (LastFailedPingInSeconds < 30 || LastHighAverageInSeconds < 30) { CanIncrease = false; } else { CanIncrease = true; } //Sleep if ping time has been less than the interval time if (pingReply.RoundtripTime < pingInterval) { //Sleep however long is required to have 250ms between pings Thread.Sleep(pingInterval - (int)pingReply.RoundtripTime); } } StaticHelpers.DebugLogger($"Bandwidth Monitor has shutdown.", DebugLevel.Verbose); }