//private async Task<Tuple<ulong, ulong, ulong>> CountFilesAndFoldersRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// ulong contentLength = 0, fileCount = 0, folderCount = 1;
// var storageFiles = await folder.GetFilesAsync();
// fileCount += (ulong)storageFiles.Count;
// foreach (var storageFile in storageFiles) contentLength += (await storageFile.GetBasicPropertiesAsync()).Size;
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// var result = await CountFilesAndFoldersRecursivelyOnWindowsAsync(storageFolder);
// contentLength += result.Item1;
// fileCount += result.Item2;
// folderCount += result.Item3;
// }
// return new Tuple<ulong, ulong, ulong>(contentLength, fileCount, folderCount);
//}
//private async Task<CloudFile> CreateFolderRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// MyLogger.Assert(this.IsFolder);
// MyLogger.Log($"Creating folder: {folder.Name} ({this.FullName})");
// var cloudFolder = await this.CreateFolderAsync(folder.Name);
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// await cloudFolder.CreateFolderRecursivelyOnWindowsAsync(storageFolder);
// }
// return cloudFolder;
//}
//private async Task UploadFileRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// MyLogger.Assert(this.IsFolder);
// MyLogger.Assert(this.Name == folder.Name);
// foreach (var storageFile in await folder.GetFilesAsync())
// {
// MyLogger.Log($"Uploading file: {storageFile.Name} ({this.FullName})");
// await this.UploadFileAsync(storageFile);
// }
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// var cloudFolder = await this.GetFolderAsync(storageFolder.Name);
// MyLogger.Assert(cloudFolder != null);
// await cloudFolder.UploadFileRecursivelyOnWindowsAsync(storageFolder);
// }
//}
private async Task <CloudFile> CreateEmptyFileAsync(string fileName)
{
MyLogger.Assert(this.IsFolder);
var request = (await Drive.GetDriveServiceAsync()).Files.Create(
new Google.Apis.Drive.v3.Data.File
{
Name = fileName,
Parents = new List <string> {
this.Id
},
MimeType = Constants.GetMimeType(System.IO.Path.GetExtension(fileName))
});
MyLogger.Log($"Creating empty file... {fileName} ({this.FullName})");
var result = await new Func <Task <Google.Apis.Drive.v3.Data.File>, Task <Google.Apis.Drive.v3.Data.File> >((Task <Google.Apis.Drive.v3.Data.File> task) =>
{
task.ContinueWith(t =>
{
// NotOnRanToCompletion - this code will be called if the upload fails
MyLogger.Log($"Failed to create file:\r\n{t.Exception}");
}, TaskContinuationOptions.NotOnRanToCompletion);
task.ContinueWith(t =>
{
MyLogger.Log($"File created successfully: {fileName} ({this.FullName})");
});
return(task);
})(request.ExecuteAsync());
MyLogger.Assert(result.Name == fileName);
var ans = new CloudFile(result.Id, result.Name, false, this);
return(ans);
}
public async Task WaitUntilCompletedAsync()
{
SemaphoreSlim semaphoreSlim = new SemaphoreSlim(0, 1);
NetworkStatusChangedEventHandler statusChangedEventHandler = null;
statusChangedEventHandler = new NetworkStatusChangedEventHandler(() =>
{
if (Status == NetworkStatus.Completed)
{
StatusChanged -= statusChangedEventHandler;
lock (semaphoreSlim) semaphoreSlim.Release();
}
});
StatusChanged += statusChangedEventHandler;
MyLogger.Assert(Status != NetworkStatus.Completed);
index_Retry :;
await semaphoreSlim.WaitAsync();
if (Status != NetworkStatus.Completed)
{
string msg = $"Status: {Status}, failed to WaitUntilCompletedAsync";
OnMessageAppended(msg);
MyLogger.Log(msg);
goto index_Retry;
}
}
public async Task DownloadFileOnWindowsAsync(Windows.Storage.StorageFile file)
{
try
{
MyLogger.Assert(!this.IsFolder);
MyLogger.Assert(this.Name == file.Name);
var downloader = new CloudFile.Downloaders.FileDownloader(this, file);
downloader.MessageAppended += new MessageAppendedEventHandler((msg) =>
{
MyLogger.Log(msg);
});
downloader.StatusChanged += new Networker.NetworkStatusChangedEventHandler(async() =>
{
if (downloader.Status == Networker.NetworkStatus.Completed)
{
MyLogger.Log($"File download succeeded!\r\nName: {file.Name}\r\nPath: {file.Path}\r\nID: {this.Id}\r\nSize: {(await file.GetBasicPropertiesAsync()).Size} bytes");
}
});
await downloader.StartAsync();
}
catch (Exception error)
{
MyLogger.Log(error.ToString());
await MyLogger.Alert(error.ToString());
}
}
public FileVerifier(CloudFile _cloudFile, Windows.Storage.StorageFile _windowsFile)
{
this.cloudFile = _cloudFile;
this.windowsFile = _windowsFile;
MyLogger.Assert(!this.cloudFile.IsFolder);
NewFileVerifierCreated?.Invoke(this);
}
public FolderVerifier(CloudFile _cloudFolder, Windows.Storage.StorageFolder _windowsFolder)
{
this.cloudFolder = _cloudFolder;
this.windowsFolder = _windowsFolder;
MyLogger.Assert(this.cloudFolder.IsFolder);
NewFolderVerifierCreated?.Invoke(this);
}
public async Task <CloudFile> CreateFolderAsync(string folderName)
{
MyLogger.Assert(this.IsFolder);
var folderCreator = new RestRequests.FileCreator(this.Id, folderName, true);
//var request = (await Drive.GetDriveServiceAsync()).Files.Create(
// new Google.Apis.Drive.v3.Data.File
// {
// Name = folderName,
// Parents = new List<string> { this.Id },
// MimeType = Constants.FolderMimeType
// });
////MyLogger.Log($"Creating folder... {folderName} ({this.FullName})");
//var result = await request.ExecuteAsync();
////var result = await new Func<Task<Google.Apis.Drive.v3.Data.File>, Task<Google.Apis.Drive.v3.Data.File>>((Task<Google.Apis.Drive.v3.Data.File> task) =>
////{
//// task.ContinueWith(t =>
//// {
//// // NotOnRanToCompletion - this code will be called if the upload fails
//// MyLogger.Log($"Failed to create folder:\r\n{t.Exception}");
//// }, TaskContinuationOptions.NotOnRanToCompletion);
//// task.ContinueWith(t =>
//// {
//// MyLogger.Log($"Folder created successfully: {folderName} ({this.FullName})");
//// });
//// return task;
////})();
//MyLogger.Assert(result.Name == folderName);
await folderCreator.Start();
MyLogger.Assert(folderCreator.Result != null);
var ans = new CloudFile(folderCreator.Result, folderName, true, this);
return(ans);
}
private SearchUnderSpecificFolderListGetter FilesGetter(string pattern)
{
MyLogger.Assert(this.IsFolder);
var getter = new SearchUnderSpecificFolderListGetter(this, pattern);
getter.MessageAppended += (log) => { MyLogger.Log($"SearchUnderSpecificFolderListGetter: {log}"); };
return(getter);
}
public async Task <CloudFile> UploadFileAsync(Windows.Storage.StorageFile file)
{
try
{
var fileSize = (await file.GetBasicPropertiesAsync()).Size;
if (fileSize == 0)
{
var uploadedFile = await CreateEmptyFileAsync(file.Name);
MyLogger.Log($"File upload succeeded!\r\nName: {uploadedFile.Name}\r\nParent: {this.FullName}\r\nID: {uploadedFile.Id}\r\nSize: {fileSize} bytes");
MyLogger.Assert(uploadedFile.Name == file.Name);
return(uploadedFile);
}
MyLogger.Assert(this.IsFolder);
var uploader = new Uploaders.FileUploader(this, file, file.Name);
indexRetry :;
await uploader.StartAsync();
switch (uploader.Status)
{
case Networker.NetworkStatus.Completed:
{
MyLogger.Log($"File upload succeeded!\r\nName: {file.Name}\r\nParent: {this.FullName}\r\nID: {uploader.UploadedCloudFile.Id}\r\nSize: {fileSize} bytes");
var ans = new CloudFile(uploader.UploadedCloudFile.Id, file.Name, false, this);
return(ans);
}
case Networker.NetworkStatus.Paused:
{
MyLogger.Log("Upload paused");
return(null);
}
default:
{
if (await MyLogger.Ask("Upload failed, try again?"))
{
await uploader.StartAsync();
goto indexRetry;
}
else
{
MyLogger.Log("Upload canceled");
return(null);
}
}
}
}
catch (Exception error)
{
MyLogger.Log(error.ToString());
await MyLogger.Alert(error.ToString());
return(null);
}
}
private async Task StartDownloadAsync()
{
Status = DownloadStatus.Downloading;
MyLogger.Assert(fileStream != null && resumableUri != null);
try
{
if (!await CreateResumableDownloadAsync())
{
Status = DownloadStatus.ErrorNeedRestart; MessageAppended?.Invoke("Error create resumable download async"); return;
}
var fileSize = bytesReceivedSoFar + serverStreamLength;
MyLogger.Log($"File size = {fileSize}");
fileStream.Position = bytesReceivedSoFar;
int chunkSize = MinChunkSize;
for (; bytesReceivedSoFar != fileSize;)
{
DateTime startTime = DateTime.Now;
var bufferSize = (int)Math.Min(chunkSize, fileSize - bytesReceivedSoFar);
if (pauseRequest)
{
Status = DownloadStatus.Paused;
return;
}
var bytesRead = await DoResumableDownloadAsync(bufferSize);
if (Status == DownloadStatus.ErrorNeedResume)
{
MessageAppended?.Invoke($"Failed! Chunk range: {bytesReceivedSoFar}-{bytesReceivedSoFar + bufferSize - 1}");
return;
}
MyLogger.Assert(Status == DownloadStatus.Downloading && bytesRead != -1);
bytesReceivedSoFar += bytesRead;
if ((DateTime.Now - startTime).TotalSeconds < 0.2)
{
chunkSize = (int)Math.Min((long)chunkSize + chunkSize / 2, int.MaxValue);
}
else
{
chunkSize = Math.Max(MinChunkSize, chunkSize / 2);
}
OnProgressChanged(bytesReceivedSoFar, fileSize);
}
OnProgressChanged(fileSize, fileSize);
Status = DownloadStatus.Completed;
}
catch (Exception error)
{
Status = DownloadStatus.ErrorNeedResume;
MessageAppended?.Invoke(error.ToString());
}
}
public async Task <CloudFile> GetFolderAsync(string folderName, bool assertSingleFolder = true)
{
MyLogger.Assert(this.IsFolder);
var ans = await FilesGetter($"'{this.Id}' in parents and trashed != true and mimeType = '{Constants.FolderMimeType}' and name = '{folderName.Replace("'","\\'")}'").GetNextPageAsync(2);
if (ans.Count == 0)
{
return(null);
}
if (assertSingleFolder)
{
MyLogger.Assert(ans.Count == 1);
}
return(ans[0]);
}
public static async Task <string> RefreshAccessTokenAsync()
{
Log("Reauthorizing...");
await GoogleWebAuthorizationBroker.ReauthorizeAsync(_credential, CancellationToken.None);
Log("Refreshing token...");
while (!(await _credential.RefreshTokenAsync(CancellationToken.None)))
{
Log("Failed to refresh token, retrying...");
}
Log("Creating service...");
MyLogger.Assert(_driveService != null);
_driveService.Dispose();
_driveService = new DriveService(new BaseClientService.Initializer()
{
HttpClientInitializer = _credential,
ApplicationName = "Google Drive APIs",
});
Log("Service created!");
MyLogger.Log($"Access token: {await GetAccessTokenAsync()}");
return(await GetAccessTokenAsync());
}
protected override async Task StartPrivateAsync()
{
ReleaseSemaphoreSlim();
try
{
switch (Status)
{
case NetworkStatus.NotStarted:
{
Status = NetworkStatus.Networking;
if (UploadedCloudFolder == null)
{
var fc = new Modifiers.FolderCreator(cloudFolder, this.windowsFolder.Name);
var messageAppendedEventHandler = new MessageAppendedEventHandler((msg) => { OnMessageAppended($"[FC]{msg}"); });
OnMessageAppended("Creating folder...");
fc.MessageAppended += messageAppendedEventHandler;
await fc.StartUntilCompletedAsync();
fc.MessageAppended -= messageAppendedEventHandler;
//switch (fc.Status)
//{
// case NetworkStatus.Completed:
// {
OnMessageAppended("Folder created");
UploadedCloudFolder = fc.CreatedCloudFolder;
OnProgressChanged(0, 0);
}
//NetworkingCount--;
var windowsFiles = await this.windowsFolder.GetFilesAsync();
lock (subTasks)
{
foreach (var f in windowsFiles)
{
subTasks.Add(new FileUploader(UploadedCloudFolder, f, f.Name));
}
}
var windowsFolders = await this.windowsFolder.GetFoldersAsync();
lock (subTasks)
{
foreach (var f in windowsFolders)
{
var folderUploader = new FolderUploader(UploadedCloudFolder, f);
subTasks.Add(folderUploader);
}
}
long currentProgress = 1, totalProgress = windowsFiles.Count + 1;
OnProgressChanged(currentProgress, totalProgress);
CurrentProgressChanged?.Invoke(1);
TotalProgressChanged?.Invoke(windowsFiles.Count + 1);
Func <Networker, Task> action;
if (Status == NetworkStatus.Paused)
{
action = new Func <Networker, Task>(async(st) =>
{
await st.WaitUntilCompletedAsync();
});
}
else
{
action = new Func <Networker, Task>(async(st) =>
{
await st.StartUntilCompletedAsync();
});
}
IEnumerable <Task> tasks;
lock (subTasks)
{
var currentProgressChangedEventHandler = new TotalProgressChangedEventHandler((dif) =>
{
MyLogger.Assert(dif == 1);
CurrentProgressChanged?.Invoke(dif);
OnProgressChanged(currentProgress += dif, totalProgress);
});
var totalProgressChangedEventHandler = new TotalProgressChangedEventHandler((dif) =>
{
TotalProgressChanged?.Invoke(dif);
OnProgressChanged(currentProgress, totalProgress += dif);
});
tasks = subTasks.ToList().Select(async(st) =>
{
if (st.GetType() == typeof(CloudFile.Uploaders.FolderUploader))
{
var fu = st as CloudFile.Uploaders.FolderUploader;
fu.CurrentProgressChanged += currentProgressChangedEventHandler;
fu.TotalProgressChanged += totalProgressChangedEventHandler;
await action(st);
fu.CurrentProgressChanged -= currentProgressChangedEventHandler;
fu.TotalProgressChanged -= totalProgressChangedEventHandler;
}
else
{
await action(st);
CurrentProgressChanged?.Invoke(1);
OnProgressChanged(++currentProgress, totalProgress);
}
});
}
await Task.WhenAll(tasks);
//NetworkingCount++;
Status = NetworkStatus.Completed;
return;
// }
// case NetworkStatus.ErrorNeedRestart:
// {
// this.Status = NetworkStatus.ErrorNeedRestart;
// return;
// }
// default: throw new Exception($"Status: {Status}");
//}
}
case NetworkStatus.Paused:
{
Status = NetworkStatus.Networking;
await Task.WhenAll(subTasks.ToList().Select(async(st) => { await st.StartAsync(); }));
//Status = NetworkStatus.Completed;
} break;
default:
{
OnMessageAppended($"Status: {Status}, no way to start");
} break;
}
}
catch (Exception error)
{
throw error;
}
finally
{
await WaitSemaphoreSlimAsync();
}
}
public async Task UploadAsync()
{
try
{
switch (Status)
{
case UploadStatus.ErrorNeedResume:
case UploadStatus.Paused:
{
MyLogger.Assert(resumableUri != null);
//indexRetry:;
MessageAppended?.Invoke($"Resuming... Uri: {resumableUri}");
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = "0";
request.Headers["Content-Range"] = $"bytes */{fileStream.Length}";
request.Method = "PUT";
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("c");
MessageAppended?.Invoke("Null response");
Status = UploadStatus.ErrorNeedResume;
return;
}
MessageAppended?.Invoke($"Http response: {response.StatusCode} ({response.StatusCode})");
if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
MessageAppended?.Invoke("The upload was already completed");
MessageAppended?.Invoke(await LogHttpWebResponse(response, false));
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
response.Dispose();
Status = UploadStatus.Completed;
return;
}
else if ((int)response.StatusCode == 308)
{
long position;
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
position = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
MessageAppended?.Invoke($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
position = long.Parse(s.Substring(pattern.Length));
}
response.Dispose();
await StartUploadAsync(position);
return;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
MessageAppended?.Invoke("The upload session has expired and the upload needs to be restarted from the beginning.");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedRestart;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't OK, Created or 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
case UploadStatus.NotStarted:
{
if (fileStream.Length < MinChunkSize)
{
MessageAppended?.Invoke($"File too small ({fileStream.Length}), using multipart upload instead");
await DoMultipartUploadAsync();
}
else
{
await CreateResumableUploadAsync();
if (Status != UploadStatus.Created)
{
return;
}
MyLogger.Assert(resumableUri.StartsWith("\"") && resumableUri.EndsWith("\""));
resumableUri = resumableUri.Substring(1, resumableUri.Length - 2);
await StartUploadAsync(0);
}
return;
}
case UploadStatus.Completed:
case UploadStatus.Created:
case UploadStatus.Creating:
case UploadStatus.ErrorNeedRestart:
case UploadStatus.Uploading:
default: throw new Exception($"Status: {Status}");
}
}
finally
{
if (pauseRequest)
{
MyLogger.Assert(Status != UploadStatus.Uploading);
lock (semaphoreSlim) semaphoreSlim.Release();
}
}
}
private SearchUnderSpecificFolderListGetter FilesGetter(bool isFolder)
{
MyLogger.Assert(this.IsFolder);
return(FilesGetter($"trashed != true and mimeType {(isFolder ? "=" : "!=")} '{Constants.FolderMimeType}'"));
}
private async Task DoResumableUploadAsync(long startByte, byte[] dataBytes)
{
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = dataBytes.Length.ToString();
request.Headers["Content-Range"] = $"bytes {startByte}-{startByte + dataBytes.Length - 1}/{fileStream.Length}";
request.Method = "PUT";
using (System.IO.Stream requestStream = await request.GetRequestStreamAsync())
{
await requestStream.WriteAsync(dataBytes, 0, dataBytes.Length);
}
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("b");
Status = UploadStatus.ErrorNeedResume;
return;
}
if ((int)response.StatusCode == 308)
{
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
byteReceivedSoFar = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
//MyLogger.Log($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
byteReceivedSoFar = long.Parse(s.Substring(pattern.Length)) + 1;
}
return;
}
else if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
//MyLogger.Log($"{data}");
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
byteReceivedSoFar = fileStream.Length;
Status = UploadStatus.Completed;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
public async Task <CloudFile> UploadFolderOnWindowsAsync(Windows.Storage.StorageFolder folder)
{
MyLogger.Assert(this.IsFolder);
if (await this.GetFolderAsync(folder.Name) != null)
{
if (!await MyLogger.Ask($"Folder, \"{folder.Name}\", already existed in \"{this.FullName}\"!\r\nStill want to upload?"))
{
return(null);
}
}
var t = new CloudFile.Uploaders.FolderUploader(this, folder);
await t.StartAsync();
return(t.UploadedCloudFolder);
/*old code
* {
* MyLogger.Log("Counting total size, files and folders...");
* var statistic = await CountFilesAndFoldersRecursivelyOnWindowsAsync(folder);
* MyLogger.Log($"{statistic.Item1} bytes, {statistic.Item2} files, {statistic.Item3} folders to upload");
* CloudFile cloudFolderToUpload;
* {
* ulong cnt = 0;
* MyLogger.SetStatus1("Creating folders");
* MyLogger.SetProgress1(0.0);
* var folderCreatedEvent = new FolderCreatedEventHandler((lambda_folder) =>
* {
* cnt++;
* double progress = (double)cnt / statistic.Item3;
* MyLogger.SetStatus1($"Creating folders...{(progress * 100).ToString("F3")}% ({cnt}/{statistic.Item3})");
* MyLogger.SetProgress1(progress);
* });
* FolderCreated += folderCreatedEvent;
* try
* {
* cloudFolderToUpload = await CreateFolderRecursivelyOnWindowsAsync(folder);
* }
* catch (Exception error)
* {
* MyLogger.Log($"Error when creating folders:\r\n{error}");
* return null;
* }
* FolderCreated -= folderCreatedEvent;
* }
* {
* ulong cntWeight = 1024;
* var setTotalProgress = new Action<ulong, ulong>((lambda_fileCount, lambda_uploadedSize) =>
* {
* double progress = (double)(lambda_uploadedSize + cntWeight * lambda_fileCount) / (statistic.Item1 + cntWeight * statistic.Item2);
* MyLogger.SetStatus1($"Uploading files...{(progress * 100).ToString("F3")}% ({lambda_fileCount}/{statistic.Item2} files) ({lambda_uploadedSize}/{statistic.Item1} bytes)");
* MyLogger.SetProgress1(progress);
* });
* ulong fileCount = 0, uploadedSize = 0;
* MyLogger.SetStatus1("Uploading files");
* MyLogger.SetProgress1(0.0);
* var fileUploadedEvent = new FileUploadedEventHandler((lambda_file, lambda_fileSize) =>
* {
* fileCount++;
* uploadedSize += lambda_fileSize;
* setTotalProgress(fileCount, uploadedSize);
* });
* var fileUploadProgressChangedEvent = new FileUploadProgressChangedEventHandler((lambda_fileName, lambda_bytesSent, lambda_totalLength) =>
* {
* setTotalProgress(fileCount, uploadedSize + (ulong)lambda_bytesSent);
* });
* FileUploaded += fileUploadedEvent;
* FileUploadProgressChanged += fileUploadProgressChangedEvent;
* try
* {
* await cloudFolderToUpload.UploadFileRecursivelyOnWindowsAsync(folder);
* MyLogger.Log($"Folder upload succeeded! {uploadedSize}/{statistic.Item1} bytes, {fileCount}/{statistic.Item2} files, {statistic.Item3} folders");
* }
* catch (Exception error)
* {
* MyLogger.Log($"Error when uploading files:\r\n{error}");
* }
* FileUploadProgressChanged -= fileUploadProgressChangedEvent;
* FileUploaded -= fileUploadedEvent;
* }
* return cloudFolderToUpload;
* }
*/
}
private static async Task <HttpWebResponse> GetHttpResponseAsync(HttpWebRequest request)
{
await semaphoreSlimGetHttpResponse.WaitAsync();
try
{
cnt++;
if ((DateTime.Now - time).TotalSeconds > 30)
{
MyLogger.Log($"{cnt} requests from {DateTime.Now} to {time}");
time = DateTime.Now;
cnt = 0;
//await MyLogger.Alert($"cnt={a}");
}
HttpWebResponse ans;
int minisecs = 500;
indexRetry :;
try
{
ans = (await request.GetResponseAsync()) as HttpWebResponse;
}
catch (WebException error)
{
ans = error.Response as HttpWebResponse;
}
finally
{
request.Abort();
}
if (ans == null)
{
MyLogger.Log("Got null response");
return(null);
}
switch (ans.StatusCode)
{
case HttpStatusCode.InternalServerError:
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.Forbidden:
{
MyLogger.Log(await LogHttpWebResponse(ans, true));
if (minisecs > Constants.MaxTimeToWait)
{
MyLogger.Log("Attempted to reconnect but still failed.");
return(ans);
}
else
{
MyLogger.Log($"Response: {ans.StatusCode} ({(int)ans.StatusCode}), waiting for {minisecs} ms and try again...");
await Task.Delay(minisecs);
minisecs *= 2;
goto indexRetry;
}
}
case HttpStatusCode.Unauthorized:
{
MyLogger.Log("Http response: Unauthorized (401). May due to expired access token, refreshing...");
MyLogger.Log(await LogHttpWebResponse(ans, true));
MyLogger.Assert(Array.IndexOf(request.Headers.AllKeys, "Authorization") != -1);
request.Headers["Authorization"] = "Bearer " + (await Drive.RefreshAccessTokenAsync());
await Task.Delay(500);
goto indexRetry;
}
default: return(ans);
}
}
finally
{
lock (semaphoreSlimGetHttpResponse) semaphoreSlimGetHttpResponse.Release();
}
}
protected override async Task StartPrivateAsync()
{
switch (Status)
{
case NetworkStatus.ErrorNeedRestart:
case NetworkStatus.NotStarted:
case NetworkStatus.Paused:
{
if (Status != NetworkStatus.Paused)
{
//Status = NetworkStatus.Starting;
//MyLogger.Assert(downloader == null && windowsFile == null && fileStream == null);
windowsFile = await CreateTemporaryFile();
fileStream = await windowsFile.OpenStreamForWriteAsync();
downloader = new RestRequests.Downloader(CloudFile.Id, fileStream);
}
Status = NetworkStatus.Networking;
var progressChangedEventHandler = new RestRequests.ProgressChangedEventHandler((bytesProcessed, totalLength) =>
{
BytesDownloaded = bytesProcessed;
TotalFileLength = totalLength;
MyLogger.Assert(this.GetType() == typeof(Downloaders.FileDownloader));
OnProgressChanged(BytesDownloaded, TotalFileLength);
});
var messageAppendedEventHandler = new MessageAppendedEventHandler((msg) =>
{
OnMessageAppended("Rest: " + msg);
});
downloader.ProgressChanged += progressChangedEventHandler;
downloader.MessageAppended += messageAppendedEventHandler;
await downloader.DownloadAsync();
downloader.ProgressChanged -= progressChangedEventHandler;
downloader.MessageAppended -= messageAppendedEventHandler;
downloadAgain_index :;
switch (downloader.Status)
{
case RestRequests.Downloader.DownloadStatus.Completed :
{
fileStream.Dispose();
Status = NetworkStatus.Completed;
return;
}
case RestRequests.Downloader.DownloadStatus.ErrorNeedRestart:
{
OnMessageAppended("Error need restart");
Status = NetworkStatus.ErrorNeedRestart;
return;
}
case RestRequests.Downloader.DownloadStatus.ErrorNeedResume:
{
OnMessageAppended("Error need resume...");
goto downloadAgain_index;
}
case RestRequests.Downloader.DownloadStatus.Paused:
{
Status = NetworkStatus.Paused;
return;
}
case RestRequests.Downloader.DownloadStatus.Downloading:
case RestRequests.Downloader.DownloadStatus.NotStarted:
default: throw new Exception($"downloader.Status: {downloader.Status}");
}
}
case NetworkStatus.Completed:
case NetworkStatus.Networking:
//case NetworkStatus.Starting:
default: throw new Exception($"Status: {Status}");
}
}
private async Task <NetworkStatus> VerifyFolder()
{
Status = NetworkStatus.Networking;
long currentProgress = 0, totalProgress = 1;
OnProgressChanged(currentProgress, totalProgress);
if (cloudFolder.Name != windowsFolder.Name)
{
OnMessageAppended($"Name not consistent, Cloud: {cloudFolder.Name}, Local: {windowsFolder.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
var folderVerifiers = new List <Tuple <CloudFile, Windows.Storage.StorageFolder> >();
{
if (!await GetCloudSubFolders())
{
return(NetworkStatus.ErrorNeedRestart);
}
Dictionary <string, Windows.Storage.StorageFolder> localSubFolders = new Dictionary <string, Windows.Storage.StorageFolder>();
foreach (var f in await windowsFolder.GetFoldersAsync())
{
if (!cloudSubFolders.ContainsKey(f.Name))
{
OnMessageAppended($"Cloud Folder doesn't exist: {f.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
localSubFolders.Add(f.Name, f);
}
foreach (var p in cloudSubFolders)
{
if (!localSubFolders.ContainsKey(p.Key))
{
OnMessageAppended($"Local Folder doesn't exist: {p.Key}");
return(NetworkStatus.ErrorNeedRestart);
}
folderVerifiers.Add(new Tuple <CloudFile, Windows.Storage.StorageFolder>(p.Value, localSubFolders[p.Key]));
localSubFolders.Remove(p.Key);
}
MyLogger.Assert(localSubFolders.Count == 0);
}
{
if (!await GetCloudSubFiles())
{
return(NetworkStatus.ErrorNeedRestart);
}
Dictionary <string, Windows.Storage.StorageFile> localSubFiles = new Dictionary <string, Windows.Storage.StorageFile>();
foreach (var f in await windowsFolder.GetFilesAsync())
{
if (!cloudSubFiles.ContainsKey(f.Name))
{
OnMessageAppended($"Cloud File doesn't exist: {f.Name}");
return(NetworkStatus.ErrorNeedRestart);
}
localSubFiles.Add(f.Name, f);
}
foreach (var p in cloudSubFiles)
{
if (!localSubFiles.ContainsKey(p.Key))
{
OnMessageAppended($"Local File doesn't exist: {p.Key}");
return(NetworkStatus.ErrorNeedRestart);
}
}
OnProgressChanged(++currentProgress, totalProgress += cloudSubFiles.Count);
CurrentProgressChanged?.Invoke(1);
TotalProgressChanged?.Invoke(cloudSubFiles.Count);
OnMessageAppended("Verifying subfiles...");
int filesVerified = 0;
foreach (var p in cloudSubFiles)
{
var localFile = localSubFiles[p.Key];
var stream = await localFile.OpenStreamForReadAsync();
var localMd5 = await Libraries.GetSha256ForWindowsStorageFile(stream);
stream.Dispose();
if (localMd5 != p.Value || p.Value == null)
{
OnMessageAppended($"{p.Key} content not consistent, Cloud: {p.Value}, Local: {localMd5}");
CurrentProgressChanged?.Invoke(-filesVerified - 1);
TotalProgressChanged?.Invoke(-cloudSubFiles.Count);
return(NetworkStatus.ErrorNeedRestart);
}
OnProgressChanged(++currentProgress, totalProgress);
CurrentProgressChanged?.Invoke(1);
++filesVerified;
localSubFiles.Remove(p.Key);
}
OnMessageAppended("Subfiles verified.");
MyLogger.Assert(localSubFiles.Count == 0);
}
OnProgressChanged(currentProgress, totalProgress += folderVerifiers.Count);
TotalProgressChanged?.Invoke(folderVerifiers.Count);
ReleaseSemaphoreSlim();
OnMessageAppended("Waiting for subfolders to be verified...");
try
{
await Task.WhenAll(folderVerifiers.Select(async(tuple) =>
{
var totalProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) =>
{
OnProgressChanged(currentProgress, totalProgress += difference);
TotalProgressChanged?.Invoke(difference);
});
var currentProgressChangedEventHandler = new TotalProgressChangedEventHandler((difference) =>
{
//MyLogger.Assert(difference == 1);
OnProgressChanged(currentProgress += difference, totalProgress);
CurrentProgressChanged?.Invoke(difference);
});
var verifier = new Verifiers.FolderVerifier(tuple.Item1, tuple.Item2);
verifier.TotalProgressChanged += totalProgressChangedEventHandler;
verifier.CurrentProgressChanged += currentProgressChangedEventHandler;
await verifier.StartUntilCompletedAsync();
verifier.TotalProgressChanged -= totalProgressChangedEventHandler;
verifier.CurrentProgressChanged -= currentProgressChangedEventHandler;
}));
return(NetworkStatus.Completed);
}
finally
{
await WaitSemaphoreSlimAsync();
}
}
protected override async Task StartPrivateAsync()
{
try
{
OnTotalFilesRemainChanged(1);
switch (Status)
{
case NetworkStatus.ErrorNeedRestart:
case NetworkStatus.NotStarted:
case NetworkStatus.Paused:
{
if (Status != NetworkStatus.Paused)
{
//Status = NetworkStatus.Starting;
//MyLogger.Assert(downloader == null && windowsFile == null && fileStream == null);
if (fileStream != null)
{
fileStream.Dispose();
fileStream = null;
}
fileStream = await windowsFile.OpenStreamForReadAsync();
uploader = new RestRequests.Uploader(new List <string> {
CloudFolder.Id
}, fileStream, fileName);
}
Status = NetworkStatus.Networking;
var progressChangedEventHandler = new RestRequests.ProgressChangedEventHandler((bytesProcessed, totalLength) =>
{
BytesUploaded = bytesProcessed;
TotalFileLength = totalLength;
MyLogger.Assert(this.GetType() == typeof(Uploaders.FileUploader));
OnProgressChanged(BytesUploaded, TotalFileLength);
});
var messageAppendedEventHandler = new MessageAppendedEventHandler((msg) =>
{
OnMessageAppended($"[Rest]{msg}");
});
var chunkSentEventHandler = new RestRequests.ChunkSentEventHandler((coda) =>
{
OnChunkSent(coda);
});
int timeToWait = 500;
uploadAgain_index :;
uploader.ProgressChanged += progressChangedEventHandler;
uploader.MessageAppended += messageAppendedEventHandler;
uploader.ChunkSent += chunkSentEventHandler;
await uploader.UploadAsync();
uploader.ProgressChanged -= progressChangedEventHandler;
uploader.MessageAppended -= messageAppendedEventHandler;
uploader.ChunkSent -= chunkSentEventHandler;
switch (uploader.Status)
{
case RestRequests.Uploader.UploadStatus.Completed:
{
UploadedCloudFile = new CloudFile(uploader.CloudFileId, fileName, false, CloudFolder);
Status = NetworkStatus.Completed;
return;
}
case RestRequests.Uploader.UploadStatus.ErrorNeedRestart:
{
OnMessageAppended("Error need restart");
Status = NetworkStatus.ErrorNeedRestart;
return;
}
case RestRequests.Uploader.UploadStatus.ErrorNeedResume:
{
if (timeToWait > Constants.MaxTimeToWait)
{
Status = NetworkStatus.ErrorNeedRestart;
return;
}
OnMessageAppended($"Error need resume... Waiting for {timeToWait} minisecs...");
await Task.Delay(timeToWait);
timeToWait *= 2;
goto uploadAgain_index;
}
case RestRequests.Uploader.UploadStatus.Paused:
{
Status = NetworkStatus.Paused;
return;
}
case RestRequests.Uploader.UploadStatus.Uploading:
case RestRequests.Uploader.UploadStatus.NotStarted:
default: throw new Exception($"uploader.Status: {uploader.Status}");
}
}
case NetworkStatus.Completed:
case NetworkStatus.Networking:
//case NetworkStatus.Starting:
{
OnMessageAppended($"Status: {Status}, no way to start");
return;
}
default: throw new Exception($"Status: {Status}");
}
}
catch (Exception error)
{
OnMessageAppended($"[Unexpected]{error}");
Status = NetworkStatus.ErrorNeedRestart;
}
finally
{
OnTotalFilesRemainChanged(-1);
}
}
public SearchUnderSpecificFolderListGetter FilesGetter()
{
MyLogger.Assert(this.IsFolder);
return(FilesGetter(false));
}
