public async Task UploadAsync() { try { switch (Status) { case UploadStatus.ErrorNeedResume: case UploadStatus.Paused: { MyLogger.Assert(resumableUri != null); //indexRetry:; MessageAppended?.Invoke($"Resuming... Uri: {resumableUri}"); var request = WebRequest.CreateHttp(resumableUri); request.Headers["Content-Length"] = "0"; request.Headers["Content-Range"] = $"bytes */{fileStream.Length}"; request.Method = "PUT"; using (var response = await GetHttpResponseAsync(request)) { if (response == null) { MyLogger.Log("c"); MessageAppended?.Invoke("Null response"); Status = UploadStatus.ErrorNeedResume; return; } MessageAppended?.Invoke($"Http response: {response.StatusCode} ({response.StatusCode})"); if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created) { MessageAppended?.Invoke("The upload was already completed"); MessageAppended?.Invoke(await LogHttpWebResponse(response, false)); using (var reader = new System.IO.StreamReader(response.GetResponseStream())) { var data = await reader.ReadToEndAsync(); var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data); MyLogger.Assert(fileName == (string)jsonObject["name"]); CloudFileId = (string)jsonObject["id"]; } response.Dispose(); Status = UploadStatus.Completed; return; } else if ((int)response.StatusCode == 308) { long position; if (Array.IndexOf(response.Headers.AllKeys, "range") == -1) { MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte..."); position = 0; } else { //bytes=0-42 string s = response.Headers["range"]; MessageAppended?.Invoke($"Range received by server: {s}"); string pattern = "bytes=0-"; MyLogger.Assert(s.StartsWith(pattern)); position = long.Parse(s.Substring(pattern.Length)); } response.Dispose(); await StartUploadAsync(position); return; } else if (response.StatusCode == HttpStatusCode.NotFound) { MessageAppended?.Invoke("The upload session has expired and the upload needs to be restarted from the beginning."); MessageAppended?.Invoke(await LogHttpWebResponse(response, true)); response.Dispose(); Status = UploadStatus.ErrorNeedRestart; return; } else { MessageAppended?.Invoke("Http response isn't OK, Created or 308!"); MessageAppended?.Invoke(await LogHttpWebResponse(response, true)); response.Dispose(); Status = UploadStatus.ErrorNeedResume; return; } } } case UploadStatus.NotStarted: { if (fileStream.Length < MinChunkSize) { MessageAppended?.Invoke($"File too small ({fileStream.Length}), using multipart upload instead"); await DoMultipartUploadAsync(); } else { await CreateResumableUploadAsync(); if (Status != UploadStatus.Created) { return; } MyLogger.Assert(resumableUri.StartsWith("\"") && resumableUri.EndsWith("\"")); resumableUri = resumableUri.Substring(1, resumableUri.Length - 2); await StartUploadAsync(0); } return; } case UploadStatus.Completed: case UploadStatus.Created: case UploadStatus.Creating: case UploadStatus.ErrorNeedRestart: case UploadStatus.Uploading: default: throw new Exception($"Status: {Status}"); } } finally { if (pauseRequest) { MyLogger.Assert(Status != UploadStatus.Uploading); lock (semaphoreSlim) semaphoreSlim.Release(); } } }
private static async Task <HttpWebResponse> GetHttpResponseAsync(HttpWebRequest request) { await semaphoreSlimGetHttpResponse.WaitAsync(); try { cnt++; if ((DateTime.Now - time).TotalSeconds > 30) { MyLogger.Log($"{cnt} requests from {DateTime.Now} to {time}"); time = DateTime.Now; cnt = 0; //await MyLogger.Alert($"cnt={a}"); } HttpWebResponse ans; int minisecs = 500; indexRetry :; try { ans = (await request.GetResponseAsync()) as HttpWebResponse; } catch (WebException error) { ans = error.Response as HttpWebResponse; } finally { request.Abort(); } if (ans == null) { MyLogger.Log("Got null response"); return(null); } switch (ans.StatusCode) { case HttpStatusCode.InternalServerError: case HttpStatusCode.BadGateway: case HttpStatusCode.ServiceUnavailable: case HttpStatusCode.GatewayTimeout: case HttpStatusCode.Forbidden: { MyLogger.Log(await LogHttpWebResponse(ans, true)); if (minisecs > Constants.MaxTimeToWait) { MyLogger.Log("Attempted to reconnect but still failed."); return(ans); } else { MyLogger.Log($"Response: {ans.StatusCode} ({(int)ans.StatusCode}), waiting for {minisecs} ms and try again..."); await Task.Delay(minisecs); minisecs *= 2; goto indexRetry; } } case HttpStatusCode.Unauthorized: { MyLogger.Log("Http response: Unauthorized (401). May due to expired access token, refreshing..."); MyLogger.Log(await LogHttpWebResponse(ans, true)); MyLogger.Assert(Array.IndexOf(request.Headers.AllKeys, "Authorization") != -1); request.Headers["Authorization"] = "Bearer " + (await Drive.RefreshAccessTokenAsync()); await Task.Delay(500); goto indexRetry; } default: return(ans); } } finally { lock (semaphoreSlimGetHttpResponse) semaphoreSlimGetHttpResponse.Release(); } }
private async Task DoResumableUploadAsync(long startByte, byte[] dataBytes) { var request = WebRequest.CreateHttp(resumableUri); request.Headers["Content-Length"] = dataBytes.Length.ToString(); request.Headers["Content-Range"] = $"bytes {startByte}-{startByte + dataBytes.Length - 1}/{fileStream.Length}"; request.Method = "PUT"; using (System.IO.Stream requestStream = await request.GetRequestStreamAsync()) { await requestStream.WriteAsync(dataBytes, 0, dataBytes.Length); } using (var response = await GetHttpResponseAsync(request)) { if (response == null) { MyLogger.Log("b"); Status = UploadStatus.ErrorNeedResume; return; } if ((int)response.StatusCode == 308) { if (Array.IndexOf(response.Headers.AllKeys, "range") == -1) { MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte..."); byteReceivedSoFar = 0; } else { //bytes=0-42 string s = response.Headers["range"]; //MyLogger.Log($"Range received by server: {s}"); string pattern = "bytes=0-"; MyLogger.Assert(s.StartsWith(pattern)); byteReceivedSoFar = long.Parse(s.Substring(pattern.Length)) + 1; } return; } else if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created) { using (var reader = new System.IO.StreamReader(response.GetResponseStream())) { var data = await reader.ReadToEndAsync(); //MyLogger.Log($"{data}"); var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data); MyLogger.Assert(fileName == (string)jsonObject["name"]); CloudFileId = (string)jsonObject["id"]; } byteReceivedSoFar = fileStream.Length; Status = UploadStatus.Completed; return; } else { MessageAppended?.Invoke("Http response isn't 308!"); MessageAppended?.Invoke(await LogHttpWebResponse(response, true)); Status = UploadStatus.ErrorNeedResume; return; } } }
private static void Log(string log) { MyLogger.Log(log); }
public SearchUnderSpecificFolderListGetter FilesGetter() { MyLogger.Assert(this.IsFolder); return(FilesGetter(false)); }
public async Task <CloudFile> UploadFolderOnWindowsAsync(Windows.Storage.StorageFolder folder) { MyLogger.Assert(this.IsFolder); if (await this.GetFolderAsync(folder.Name) != null) { if (!await MyLogger.Ask($"Folder, \"{folder.Name}\", already existed in \"{this.FullName}\"!\r\nStill want to upload?")) { return(null); } } var t = new CloudFile.Uploaders.FolderUploader(this, folder); await t.StartAsync(); return(t.UploadedCloudFolder); /*old code * { * MyLogger.Log("Counting total size, files and folders..."); * var statistic = await CountFilesAndFoldersRecursivelyOnWindowsAsync(folder); * MyLogger.Log($"{statistic.Item1} bytes, {statistic.Item2} files, {statistic.Item3} folders to upload"); * CloudFile cloudFolderToUpload; * { * ulong cnt = 0; * MyLogger.SetStatus1("Creating folders"); * MyLogger.SetProgress1(0.0); * var folderCreatedEvent = new FolderCreatedEventHandler((lambda_folder) => * { * cnt++; * double progress = (double)cnt / statistic.Item3; * MyLogger.SetStatus1($"Creating folders...{(progress * 100).ToString("F3")}% ({cnt}/{statistic.Item3})"); * MyLogger.SetProgress1(progress); * }); * FolderCreated += folderCreatedEvent; * try * { * cloudFolderToUpload = await CreateFolderRecursivelyOnWindowsAsync(folder); * } * catch (Exception error) * { * MyLogger.Log($"Error when creating folders:\r\n{error}"); * return null; * } * FolderCreated -= folderCreatedEvent; * } * { * ulong cntWeight = 1024; * var setTotalProgress = new Action<ulong, ulong>((lambda_fileCount, lambda_uploadedSize) => * { * double progress = (double)(lambda_uploadedSize + cntWeight * lambda_fileCount) / (statistic.Item1 + cntWeight * statistic.Item2); * MyLogger.SetStatus1($"Uploading files...{(progress * 100).ToString("F3")}% ({lambda_fileCount}/{statistic.Item2} files) ({lambda_uploadedSize}/{statistic.Item1} bytes)"); * MyLogger.SetProgress1(progress); * }); * ulong fileCount = 0, uploadedSize = 0; * MyLogger.SetStatus1("Uploading files"); * MyLogger.SetProgress1(0.0); * var fileUploadedEvent = new FileUploadedEventHandler((lambda_file, lambda_fileSize) => * { * fileCount++; * uploadedSize += lambda_fileSize; * setTotalProgress(fileCount, uploadedSize); * }); * var fileUploadProgressChangedEvent = new FileUploadProgressChangedEventHandler((lambda_fileName, lambda_bytesSent, lambda_totalLength) => * { * setTotalProgress(fileCount, uploadedSize + (ulong)lambda_bytesSent); * }); * FileUploaded += fileUploadedEvent; * FileUploadProgressChanged += fileUploadProgressChangedEvent; * try * { * await cloudFolderToUpload.UploadFileRecursivelyOnWindowsAsync(folder); * MyLogger.Log($"Folder upload succeeded! {uploadedSize}/{statistic.Item1} bytes, {fileCount}/{statistic.Item2} files, {statistic.Item3} folders"); * } * catch (Exception error) * { * MyLogger.Log($"Error when uploading files:\r\n{error}"); * } * FileUploadProgressChanged -= fileUploadProgressChangedEvent; * FileUploaded -= fileUploadedEvent; * } * return cloudFolderToUpload; * } */ }
private SearchUnderSpecificFolderListGetter FilesGetter(bool isFolder) { MyLogger.Assert(this.IsFolder); return(FilesGetter($"trashed != true and mimeType {(isFolder ? "=" : "!=")} '{Constants.FolderMimeType}'")); }
private async Task <NetworkStatus> VerifyFolder() { Status = NetworkStatus.Networking; long currentProgress = 0, totalProgress = 1; OnProgressChanged(currentProgress, totalProgress); if (cloudFolder.Name != windowsFolder.Name) { OnMessageAppended($"Name not consistent, Cloud: {cloudFolder.Name}, Local: {windowsFolder.Name}"); return(NetworkStatus.ErrorNeedRestart); } var folderVerifiers = new List <Tuple <CloudFile, Windows.Storage.StorageFolder> >(); { if (!await GetCloudSubFolders()) { return(NetworkStatus.ErrorNeedRestart); } Dictionary <string, Windows.Storage.StorageFolder> localSubFolders = new Dictionary <string, Windows.Storage.StorageFolder>(); foreach (var f in await windowsFolder.GetFoldersAsync()) { if (!cloudSubFolders.ContainsKey(f.Name)) { OnMessageAppended($"Cloud Folder doesn't exist: {f.Name}"); return(NetworkStatus.ErrorNeedRestart); } localSubFolders.Add(f.Name, f); } foreach (var p in cloudSubFolders) { if (!localSubFolders.ContainsKey(p.Key)) { OnMessageAppended($"Local Folder doesn't exist: {p.Key}"); return(NetworkStatus.ErrorNeedRestart); } folderVerifiers.Add(new Tuple <CloudFile, Windows.Storage.StorageFolder>(p.Value, localSubFolders[p.Key])); localSubFolders.Remove(p.Key); } MyLogger.Assert(localSubFolders.Count == 0); } { if (!await GetCloudSubFiles()) { return(NetworkStatus.ErrorNeedRestart); } Dictionary <string, Windows.Storage.StorageFile> localSubFiles = new Dictionary <string, Windows.Storage.StorageFile>(); foreach (var f in await windowsFolder.GetFilesAsync()) { if (!cloudSubFiles.ContainsKey(f.Name)) { OnMessageAppended($"Cloud File doesn't exist: {f.Name}"); return(NetworkStatus.ErrorNeedRestart); } localSubFiles.Add(f.Name, f); } foreach (var p in cloudSubFiles) { if (!localSubFiles.ContainsKey(p.Key)) { OnMessageAppended($"Local File doesn't exist: {p.Key}"); return(NetworkStatus.ErrorNeedRestart); } } OnProgressChanged(++currentProgress, totalProgress += cloudSubFiles.Count); CurrentProgressChanged?.Invoke(1); TotalProgressChanged?.Invoke(cloudSubFiles.Count); OnMessageAppended("Verifying subfiles..."); int filesVerified = 0; foreach (var p in cloudSubFiles) { var localFile = localSubFiles[p.Key]; var stream = await localFile.OpenStreamForReadAsync(); var localMd5 = await Libraries.GetSha256ForWindowsStorageFile(stream); stream.Dispose(); if (localMd5 != p.Value || p.Value == null) { OnMessageAppended($"{p.Key} content not consistent, Cloud: {p.Value}, Local: {localMd5}"); CurrentProgressChanged?.Invoke(-filesVerified - 1); TotalProgressChanged?.Invoke(-cloudSubFiles.Count); return(NetworkStatus.ErrorNeedRestart); } OnProgressChanged(++currentProgress, totalProgress); CurrentProgressChanged?.Invoke(1); ++filesVerified; localSubFiles.Remove(p.Key); } OnMessageAppended("Subfiles verified."); MyLogger.Assert(localSubFiles.Count == 0); } OnProgressChanged(currentProgress, totalProgress += folderVerifiers.Count); TotalProgressChanged?.Invoke(folderVerifiers.Count); ReleaseSemaphoreSlim(); OnMessageAppended("Waiting for subfolders to be verified..."); try { await Task.WhenAll(folderVerifiers.Select(async(tuple) => { var totalProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) => { OnProgressChanged(currentProgress, totalProgress += difference); TotalProgressChanged?.Invoke(difference); }); var currentProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) => { //MyLogger.Assert(difference == 1); OnProgressChanged(currentProgress += difference, totalProgress); CurrentProgressChanged?.Invoke(difference); }); var verifier = new Verifiers.FolderVerifier(tuple.Item1, tuple.Item2); verifier.TotalProgressChanged += totalProgressChangedEventHandler; verifier.CurrentProgressChanged += currentProgressChangedEventHandler; await verifier.StartUntilCompletedAsync(); verifier.TotalProgressChanged -= totalProgressChangedEventHandler; verifier.CurrentProgressChanged -= currentProgressChangedEventHandler; })); return(NetworkStatus.Completed); } finally { await WaitSemaphoreSlimAsync(); } }