public async Task UploadAsync()
{
try
{
switch (Status)
{
case UploadStatus.ErrorNeedResume:
case UploadStatus.Paused:
{
MyLogger.Assert(resumableUri != null);
//indexRetry:;
MessageAppended?.Invoke($"Resuming... Uri: {resumableUri}");
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = "0";
request.Headers["Content-Range"] = $"bytes */{fileStream.Length}";
request.Method = "PUT";
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("c");
MessageAppended?.Invoke("Null response");
Status = UploadStatus.ErrorNeedResume;
return;
}
MessageAppended?.Invoke($"Http response: {response.StatusCode} ({response.StatusCode})");
if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
MessageAppended?.Invoke("The upload was already completed");
MessageAppended?.Invoke(await LogHttpWebResponse(response, false));
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
response.Dispose();
Status = UploadStatus.Completed;
return;
}
else if ((int)response.StatusCode == 308)
{
long position;
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
position = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
MessageAppended?.Invoke($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
position = long.Parse(s.Substring(pattern.Length));
}
response.Dispose();
await StartUploadAsync(position);
return;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
MessageAppended?.Invoke("The upload session has expired and the upload needs to be restarted from the beginning.");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedRestart;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't OK, Created or 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
case UploadStatus.NotStarted:
{
if (fileStream.Length < MinChunkSize)
{
MessageAppended?.Invoke($"File too small ({fileStream.Length}), using multipart upload instead");
await DoMultipartUploadAsync();
}
else
{
await CreateResumableUploadAsync();
if (Status != UploadStatus.Created)
{
return;
}
MyLogger.Assert(resumableUri.StartsWith("\"") && resumableUri.EndsWith("\""));
resumableUri = resumableUri.Substring(1, resumableUri.Length - 2);
await StartUploadAsync(0);
}
return;
}
case UploadStatus.Completed:
case UploadStatus.Created:
case UploadStatus.Creating:
case UploadStatus.ErrorNeedRestart:
case UploadStatus.Uploading:
default: throw new Exception($"Status: {Status}");
}
}
finally
{
if (pauseRequest)
{
MyLogger.Assert(Status != UploadStatus.Uploading);
lock (semaphoreSlim) semaphoreSlim.Release();
}
}
}
private static async Task <HttpWebResponse> GetHttpResponseAsync(HttpWebRequest request)
{
await semaphoreSlimGetHttpResponse.WaitAsync();
try
{
cnt++;
if ((DateTime.Now - time).TotalSeconds > 30)
{
MyLogger.Log($"{cnt} requests from {DateTime.Now} to {time}");
time = DateTime.Now;
cnt = 0;
//await MyLogger.Alert($"cnt={a}");
}
HttpWebResponse ans;
int minisecs = 500;
indexRetry :;
try
{
ans = (await request.GetResponseAsync()) as HttpWebResponse;
}
catch (WebException error)
{
ans = error.Response as HttpWebResponse;
}
finally
{
request.Abort();
}
if (ans == null)
{
MyLogger.Log("Got null response");
return(null);
}
switch (ans.StatusCode)
{
case HttpStatusCode.InternalServerError:
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.Forbidden:
{
MyLogger.Log(await LogHttpWebResponse(ans, true));
if (minisecs > Constants.MaxTimeToWait)
{
MyLogger.Log("Attempted to reconnect but still failed.");
return(ans);
}
else
{
MyLogger.Log($"Response: {ans.StatusCode} ({(int)ans.StatusCode}), waiting for {minisecs} ms and try again...");
await Task.Delay(minisecs);
minisecs *= 2;
goto indexRetry;
}
}
case HttpStatusCode.Unauthorized:
{
MyLogger.Log("Http response: Unauthorized (401). May due to expired access token, refreshing...");
MyLogger.Log(await LogHttpWebResponse(ans, true));
MyLogger.Assert(Array.IndexOf(request.Headers.AllKeys, "Authorization") != -1);
request.Headers["Authorization"] = "Bearer " + (await Drive.RefreshAccessTokenAsync());
await Task.Delay(500);
goto indexRetry;
}
default: return(ans);
}
}
finally
{
lock (semaphoreSlimGetHttpResponse) semaphoreSlimGetHttpResponse.Release();
}
}
private async Task DoResumableUploadAsync(long startByte, byte[] dataBytes)
{
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = dataBytes.Length.ToString();
request.Headers["Content-Range"] = $"bytes {startByte}-{startByte + dataBytes.Length - 1}/{fileStream.Length}";
request.Method = "PUT";
using (System.IO.Stream requestStream = await request.GetRequestStreamAsync())
{
await requestStream.WriteAsync(dataBytes, 0, dataBytes.Length);
}
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("b");
Status = UploadStatus.ErrorNeedResume;
return;
}
if ((int)response.StatusCode == 308)
{
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
byteReceivedSoFar = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
//MyLogger.Log($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
byteReceivedSoFar = long.Parse(s.Substring(pattern.Length)) + 1;
}
return;
}
else if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
//MyLogger.Log($"{data}");
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
byteReceivedSoFar = fileStream.Length;
Status = UploadStatus.Completed;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
private static void Log(string log)
{
MyLogger.Log(log);
}
public SearchUnderSpecificFolderListGetter FilesGetter()
{
MyLogger.Assert(this.IsFolder);
return(FilesGetter(false));
}
public async Task <CloudFile> UploadFolderOnWindowsAsync(Windows.Storage.StorageFolder folder)
{
MyLogger.Assert(this.IsFolder);
if (await this.GetFolderAsync(folder.Name) != null)
{
if (!await MyLogger.Ask($"Folder, \"{folder.Name}\", already existed in \"{this.FullName}\"!\r\nStill want to upload?"))
{
return(null);
}
}
var t = new CloudFile.Uploaders.FolderUploader(this, folder);
await t.StartAsync();
return(t.UploadedCloudFolder);
/*old code
* {
* MyLogger.Log("Counting total size, files and folders...");
* var statistic = await CountFilesAndFoldersRecursivelyOnWindowsAsync(folder);
* MyLogger.Log($"{statistic.Item1} bytes, {statistic.Item2} files, {statistic.Item3} folders to upload");
* CloudFile cloudFolderToUpload;
* {
* ulong cnt = 0;
* MyLogger.SetStatus1("Creating folders");
* MyLogger.SetProgress1(0.0);
* var folderCreatedEvent = new FolderCreatedEventHandler((lambda_folder) =>
* {
* cnt++;
* double progress = (double)cnt / statistic.Item3;
* MyLogger.SetStatus1($"Creating folders...{(progress * 100).ToString("F3")}% ({cnt}/{statistic.Item3})");
* MyLogger.SetProgress1(progress);
* });
* FolderCreated += folderCreatedEvent;
* try
* {
* cloudFolderToUpload = await CreateFolderRecursivelyOnWindowsAsync(folder);
* }
* catch (Exception error)
* {
* MyLogger.Log($"Error when creating folders:\r\n{error}");
* return null;
* }
* FolderCreated -= folderCreatedEvent;
* }
* {
* ulong cntWeight = 1024;
* var setTotalProgress = new Action<ulong, ulong>((lambda_fileCount, lambda_uploadedSize) =>
* {
* double progress = (double)(lambda_uploadedSize + cntWeight * lambda_fileCount) / (statistic.Item1 + cntWeight * statistic.Item2);
* MyLogger.SetStatus1($"Uploading files...{(progress * 100).ToString("F3")}% ({lambda_fileCount}/{statistic.Item2} files) ({lambda_uploadedSize}/{statistic.Item1} bytes)");
* MyLogger.SetProgress1(progress);
* });
* ulong fileCount = 0, uploadedSize = 0;
* MyLogger.SetStatus1("Uploading files");
* MyLogger.SetProgress1(0.0);
* var fileUploadedEvent = new FileUploadedEventHandler((lambda_file, lambda_fileSize) =>
* {
* fileCount++;
* uploadedSize += lambda_fileSize;
* setTotalProgress(fileCount, uploadedSize);
* });
* var fileUploadProgressChangedEvent = new FileUploadProgressChangedEventHandler((lambda_fileName, lambda_bytesSent, lambda_totalLength) =>
* {
* setTotalProgress(fileCount, uploadedSize + (ulong)lambda_bytesSent);
* });
* FileUploaded += fileUploadedEvent;
* FileUploadProgressChanged += fileUploadProgressChangedEvent;
* try
* {
* await cloudFolderToUpload.UploadFileRecursivelyOnWindowsAsync(folder);
* MyLogger.Log($"Folder upload succeeded! {uploadedSize}/{statistic.Item1} bytes, {fileCount}/{statistic.Item2} files, {statistic.Item3} folders");
* }
* catch (Exception error)
* {
* MyLogger.Log($"Error when uploading files:\r\n{error}");
* }
* FileUploadProgressChanged -= fileUploadProgressChangedEvent;
* FileUploaded -= fileUploadedEvent;
* }
* return cloudFolderToUpload;
* }
*/
}
private SearchUnderSpecificFolderListGetter FilesGetter(bool isFolder)
{
MyLogger.Assert(this.IsFolder);
return(FilesGetter($"trashed != true and mimeType {(isFolder ? "=" : "!=")} '{Constants.FolderMimeType}'"));
}
private async Task <NetworkStatus> VerifyFolder()
{
Status = NetworkStatus.Networking;
long currentProgress = 0, totalProgress = 1;
OnProgressChanged(currentProgress, totalProgress);
if (cloudFolder.Name != windowsFolder.Name)
{
OnMessageAppended($"Name not consistent, Cloud: {cloudFolder.Name}, Local: {windowsFolder.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
var folderVerifiers = new List <Tuple <CloudFile, Windows.Storage.StorageFolder> >();
{
if (!await GetCloudSubFolders())
{
return(NetworkStatus.ErrorNeedRestart);
}
Dictionary <string, Windows.Storage.StorageFolder> localSubFolders = new Dictionary <string, Windows.Storage.StorageFolder>();
foreach (var f in await windowsFolder.GetFoldersAsync())
{
if (!cloudSubFolders.ContainsKey(f.Name))
{
OnMessageAppended($"Cloud Folder doesn't exist: {f.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
localSubFolders.Add(f.Name, f);
}
foreach (var p in cloudSubFolders)
{
if (!localSubFolders.ContainsKey(p.Key))
{
OnMessageAppended($"Local Folder doesn't exist: {p.Key}");
return(NetworkStatus.ErrorNeedRestart);
}
folderVerifiers.Add(new Tuple <CloudFile, Windows.Storage.StorageFolder>(p.Value, localSubFolders[p.Key]));
localSubFolders.Remove(p.Key);
}
MyLogger.Assert(localSubFolders.Count == 0);
}
{
if (!await GetCloudSubFiles())
{
return(NetworkStatus.ErrorNeedRestart);
}
Dictionary <string, Windows.Storage.StorageFile> localSubFiles = new Dictionary <string, Windows.Storage.StorageFile>();
foreach (var f in await windowsFolder.GetFilesAsync())
{
if (!cloudSubFiles.ContainsKey(f.Name))
{
OnMessageAppended($"Cloud File doesn't exist: {f.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
localSubFiles.Add(f.Name, f);
}
foreach (var p in cloudSubFiles)
{
if (!localSubFiles.ContainsKey(p.Key))
{
OnMessageAppended($"Local File doesn't exist: {p.Key}");
return(NetworkStatus.ErrorNeedRestart);
}
}
OnProgressChanged(++currentProgress, totalProgress += cloudSubFiles.Count);
CurrentProgressChanged?.Invoke(1);
TotalProgressChanged?.Invoke(cloudSubFiles.Count);
OnMessageAppended("Verifying subfiles...");
int filesVerified = 0;
foreach (var p in cloudSubFiles)
{
var localFile = localSubFiles[p.Key];
var stream = await localFile.OpenStreamForReadAsync();
var localMd5 = await Libraries.GetSha256ForWindowsStorageFile(stream);
stream.Dispose();
if (localMd5 != p.Value || p.Value == null)
{
OnMessageAppended($"{p.Key} content not consistent, Cloud: {p.Value}, Local: {localMd5}");
CurrentProgressChanged?.Invoke(-filesVerified - 1);
TotalProgressChanged?.Invoke(-cloudSubFiles.Count);
return(NetworkStatus.ErrorNeedRestart);
}
OnProgressChanged(++currentProgress, totalProgress);
CurrentProgressChanged?.Invoke(1);
++filesVerified;
localSubFiles.Remove(p.Key);
}
OnMessageAppended("Subfiles verified.");
MyLogger.Assert(localSubFiles.Count == 0);
}
OnProgressChanged(currentProgress, totalProgress += folderVerifiers.Count);
TotalProgressChanged?.Invoke(folderVerifiers.Count);
ReleaseSemaphoreSlim();
OnMessageAppended("Waiting for subfolders to be verified...");
try
{
await Task.WhenAll(folderVerifiers.Select(async(tuple) =>
{
var totalProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) =>
{
OnProgressChanged(currentProgress, totalProgress += difference);
TotalProgressChanged?.Invoke(difference);
});
var currentProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) =>
{
//MyLogger.Assert(difference == 1);
OnProgressChanged(currentProgress += difference, totalProgress);
CurrentProgressChanged?.Invoke(difference);
});
var verifier = new Verifiers.FolderVerifier(tuple.Item1, tuple.Item2);
verifier.TotalProgressChanged += totalProgressChangedEventHandler;
verifier.CurrentProgressChanged += currentProgressChangedEventHandler;
await verifier.StartUntilCompletedAsync();
verifier.TotalProgressChanged -= totalProgressChangedEventHandler;
verifier.CurrentProgressChanged -= currentProgressChangedEventHandler;
}));
return(NetworkStatus.Completed);
}
finally
{
await WaitSemaphoreSlimAsync();
}
}