//private async Task<Tuple<ulong, ulong, ulong>> CountFilesAndFoldersRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// ulong contentLength = 0, fileCount = 0, folderCount = 1;
// var storageFiles = await folder.GetFilesAsync();
// fileCount += (ulong)storageFiles.Count;
// foreach (var storageFile in storageFiles) contentLength += (await storageFile.GetBasicPropertiesAsync()).Size;
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// var result = await CountFilesAndFoldersRecursivelyOnWindowsAsync(storageFolder);
// contentLength += result.Item1;
// fileCount += result.Item2;
// folderCount += result.Item3;
// }
// return new Tuple<ulong, ulong, ulong>(contentLength, fileCount, folderCount);
//}
//private async Task<CloudFile> CreateFolderRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// MyLogger.Assert(this.IsFolder);
// MyLogger.Log($"Creating folder: {folder.Name} ({this.FullName})");
// var cloudFolder = await this.CreateFolderAsync(folder.Name);
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// await cloudFolder.CreateFolderRecursivelyOnWindowsAsync(storageFolder);
// }
// return cloudFolder;
//}
//private async Task UploadFileRecursivelyOnWindowsAsync(Windows.Storage.StorageFolder folder)
//{
// MyLogger.Assert(this.IsFolder);
// MyLogger.Assert(this.Name == folder.Name);
// foreach (var storageFile in await folder.GetFilesAsync())
// {
// MyLogger.Log($"Uploading file: {storageFile.Name} ({this.FullName})");
// await this.UploadFileAsync(storageFile);
// }
// foreach (var storageFolder in await folder.GetFoldersAsync())
// {
// var cloudFolder = await this.GetFolderAsync(storageFolder.Name);
// MyLogger.Assert(cloudFolder != null);
// await cloudFolder.UploadFileRecursivelyOnWindowsAsync(storageFolder);
// }
//}
private async Task <CloudFile> CreateEmptyFileAsync(string fileName)
{
MyLogger.Assert(this.IsFolder);
var request = (await Drive.GetDriveServiceAsync()).Files.Create(
new Google.Apis.Drive.v3.Data.File
{
Name = fileName,
Parents = new List <string> {
this.Id
},
MimeType = Constants.GetMimeType(System.IO.Path.GetExtension(fileName))
});
MyLogger.Log($"Creating empty file... {fileName} ({this.FullName})");
var result = await new Func <Task <Google.Apis.Drive.v3.Data.File>, Task <Google.Apis.Drive.v3.Data.File> >((Task <Google.Apis.Drive.v3.Data.File> task) =>
{
task.ContinueWith(t =>
{
// NotOnRanToCompletion - this code will be called if the upload fails
MyLogger.Log($"Failed to create file:\r\n{t.Exception}");
}, TaskContinuationOptions.NotOnRanToCompletion);
task.ContinueWith(t =>
{
MyLogger.Log($"File created successfully: {fileName} ({this.FullName})");
});
return(task);
})(request.ExecuteAsync());
MyLogger.Assert(result.Name == fileName);
var ans = new CloudFile(result.Id, result.Name, false, this);
return(ans);
}
public async Task <string> GetFileAsync()
{
var request = WebRequest.CreateHttp($"https://www.googleapis.com/drive/v2/files/{fileId}");
request.Headers["Authorization"] = "Bearer " + (await Drive.GetAccessTokenAsync());
request.Method = "GET";
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("Null response");
return(null);
}
//MyLogger.Log($"Http response: {response.StatusCode} ({(int)response.StatusCode})");
if (response.StatusCode == HttpStatusCode.OK)
{
return(await LogHttpWebResponse(response, true));
}
else
{
MyLogger.Log("Http response isn't OK!");
MyLogger.Log(await LogHttpWebResponse(response, true));
return(null);
}
}
}
public async Task DownloadFileOnWindowsAsync(Windows.Storage.StorageFile file)
{
try
{
MyLogger.Assert(!this.IsFolder);
MyLogger.Assert(this.Name == file.Name);
var downloader = new CloudFile.Downloaders.FileDownloader(this, file);
downloader.MessageAppended += new MessageAppendedEventHandler((msg) =>
{
MyLogger.Log(msg);
});
downloader.StatusChanged += new Networker.NetworkStatusChangedEventHandler(async() =>
{
if (downloader.Status == Networker.NetworkStatus.Completed)
{
MyLogger.Log($"File download succeeded!\r\nName: {file.Name}\r\nPath: {file.Path}\r\nID: {this.Id}\r\nSize: {(await file.GetBasicPropertiesAsync()).Size} bytes");
}
});
await downloader.StartAsync();
}
catch (Exception error)
{
MyLogger.Log(error.ToString());
await MyLogger.Alert(error.ToString());
}
}
private SearchUnderSpecificFolderListGetter FilesGetter(string pattern)
{
MyLogger.Assert(this.IsFolder);
var getter = new SearchUnderSpecificFolderListGetter(this, pattern);
getter.MessageAppended += (log) => { MyLogger.Log($"SearchUnderSpecificFolderListGetter: {log}"); };
return(getter);
}
public static void Assert(bool condition)
{
if (!condition)
{
MyLogger.Log("Assertion failed!");
}
System.Diagnostics.Debug.Assert(condition);
}
public async Task <CloudFile> UploadFileAsync(Windows.Storage.StorageFile file)
{
try
{
var fileSize = (await file.GetBasicPropertiesAsync()).Size;
if (fileSize == 0)
{
var uploadedFile = await CreateEmptyFileAsync(file.Name);
MyLogger.Log($"File upload succeeded!\r\nName: {uploadedFile.Name}\r\nParent: {this.FullName}\r\nID: {uploadedFile.Id}\r\nSize: {fileSize} bytes");
MyLogger.Assert(uploadedFile.Name == file.Name);
return(uploadedFile);
}
MyLogger.Assert(this.IsFolder);
var uploader = new Uploaders.FileUploader(this, file, file.Name);
indexRetry :;
await uploader.StartAsync();
switch (uploader.Status)
{
case Networker.NetworkStatus.Completed:
{
MyLogger.Log($"File upload succeeded!\r\nName: {file.Name}\r\nParent: {this.FullName}\r\nID: {uploader.UploadedCloudFile.Id}\r\nSize: {fileSize} bytes");
var ans = new CloudFile(uploader.UploadedCloudFile.Id, file.Name, false, this);
return(ans);
}
case Networker.NetworkStatus.Paused:
{
MyLogger.Log("Upload paused");
return(null);
}
default:
{
if (await MyLogger.Ask("Upload failed, try again?"))
{
await uploader.StartAsync();
goto indexRetry;
}
else
{
MyLogger.Log("Upload canceled");
return(null);
}
}
}
}
catch (Exception error)
{
MyLogger.Log(error.ToString());
await MyLogger.Alert(error.ToString());
return(null);
}
}
public static async Task <DriveService> GetDriveServiceAsync()
{
if (_driveService == null)
{
await AuthorizeAsync();
MyLogger.Log($"Access token: {await GetAccessTokenAsync()}");
}
return(_driveService);
}
private async Task StartDownloadAsync()
{
Status = DownloadStatus.Downloading;
MyLogger.Assert(fileStream != null && resumableUri != null);
try
{
if (!await CreateResumableDownloadAsync())
{
Status = DownloadStatus.ErrorNeedRestart; MessageAppended?.Invoke("Error create resumable download async"); return;
}
var fileSize = bytesReceivedSoFar + serverStreamLength;
MyLogger.Log($"File size = {fileSize}");
fileStream.Position = bytesReceivedSoFar;
int chunkSize = MinChunkSize;
for (; bytesReceivedSoFar != fileSize;)
{
DateTime startTime = DateTime.Now;
var bufferSize = (int)Math.Min(chunkSize, fileSize - bytesReceivedSoFar);
if (pauseRequest)
{
Status = DownloadStatus.Paused;
return;
}
var bytesRead = await DoResumableDownloadAsync(bufferSize);
if (Status == DownloadStatus.ErrorNeedResume)
{
MessageAppended?.Invoke($"Failed! Chunk range: {bytesReceivedSoFar}-{bytesReceivedSoFar + bufferSize - 1}");
return;
}
MyLogger.Assert(Status == DownloadStatus.Downloading && bytesRead != -1);
bytesReceivedSoFar += bytesRead;
if ((DateTime.Now - startTime).TotalSeconds < 0.2)
{
chunkSize = (int)Math.Min((long)chunkSize + chunkSize / 2, int.MaxValue);
}
else
{
chunkSize = Math.Max(MinChunkSize, chunkSize / 2);
}
OnProgressChanged(bytesReceivedSoFar, fileSize);
}
OnProgressChanged(fileSize, fileSize);
Status = DownloadStatus.Completed;
}
catch (Exception error)
{
Status = DownloadStatus.ErrorNeedResume;
MessageAppended?.Invoke(error.ToString());
}
}
private async Task <bool> CreateResumableDownloadAsync()
{
indexRetry :;
if (serverStream != null)
{
serverStream.Dispose();
serverStream = null;
}
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Authorization"] = "Bearer " + (await Drive.GetAccessTokenAsync());
request.Headers["Range"] = $"bytes={bytesReceivedSoFar}-";
request.Method = "GET";
var response = await GetHttpResponseAsync(request);
if (response == null)
{
MyLogger.Log("Null response, trying to create the download again...");
goto indexRetry;
}
else if (response.StatusCode == HttpStatusCode.PartialContent)
{
serverStream = response.GetResponseStream();
if (Array.IndexOf(response.Headers.AllKeys, "content-length") == -1)
{
MyLogger.Log($"Response header doesn't exist: content-length");
MyLogger.Log(await LogHttpWebResponse(response, true));
return(false);
}
if (!long.TryParse(response.Headers["content-length"], out serverStreamLength))
{
MyLogger.Log($"Error parsing serverStreamLength from \"content-length\" header: {response.Headers["content-length"]}");
MyLogger.Log(await LogHttpWebResponse(response, true));
return(false);
}
return(true);
}
else
{
MyLogger.Log("Http response isn't PartialContent!");
MyLogger.Log(await LogHttpWebResponse(response, true));
return(false);
}
}
public static async Task <string> RefreshAccessTokenAsync()
{
Log("Reauthorizing...");
await GoogleWebAuthorizationBroker.ReauthorizeAsync(_credential, CancellationToken.None);
Log("Refreshing token...");
while (!(await _credential.RefreshTokenAsync(CancellationToken.None)))
{
Log("Failed to refresh token, retrying...");
}
Log("Creating service...");
MyLogger.Assert(_driveService != null);
_driveService.Dispose();
_driveService = new DriveService(new BaseClientService.Initializer()
{
HttpClientInitializer = _credential,
ApplicationName = "Google Drive APIs",
});
Log("Service created!");
MyLogger.Log($"Access token: {await GetAccessTokenAsync()}");
return(await GetAccessTokenAsync());
}
public async Task UploadAsync()
{
try
{
switch (Status)
{
case UploadStatus.ErrorNeedResume:
case UploadStatus.Paused:
{
MyLogger.Assert(resumableUri != null);
//indexRetry:;
MessageAppended?.Invoke($"Resuming... Uri: {resumableUri}");
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = "0";
request.Headers["Content-Range"] = $"bytes */{fileStream.Length}";
request.Method = "PUT";
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("c");
MessageAppended?.Invoke("Null response");
Status = UploadStatus.ErrorNeedResume;
return;
}
MessageAppended?.Invoke($"Http response: {response.StatusCode} ({response.StatusCode})");
if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
MessageAppended?.Invoke("The upload was already completed");
MessageAppended?.Invoke(await LogHttpWebResponse(response, false));
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
response.Dispose();
Status = UploadStatus.Completed;
return;
}
else if ((int)response.StatusCode == 308)
{
long position;
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
position = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
MessageAppended?.Invoke($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
position = long.Parse(s.Substring(pattern.Length));
}
response.Dispose();
await StartUploadAsync(position);
return;
}
else if (response.StatusCode == HttpStatusCode.NotFound)
{
MessageAppended?.Invoke("The upload session has expired and the upload needs to be restarted from the beginning.");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedRestart;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't OK, Created or 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
response.Dispose();
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
case UploadStatus.NotStarted:
{
if (fileStream.Length < MinChunkSize)
{
MessageAppended?.Invoke($"File too small ({fileStream.Length}), using multipart upload instead");
await DoMultipartUploadAsync();
}
else
{
await CreateResumableUploadAsync();
if (Status != UploadStatus.Created)
{
return;
}
MyLogger.Assert(resumableUri.StartsWith("\"") && resumableUri.EndsWith("\""));
resumableUri = resumableUri.Substring(1, resumableUri.Length - 2);
await StartUploadAsync(0);
}
return;
}
case UploadStatus.Completed:
case UploadStatus.Created:
case UploadStatus.Creating:
case UploadStatus.ErrorNeedRestart:
case UploadStatus.Uploading:
default: throw new Exception($"Status: {Status}");
}
}
finally
{
if (pauseRequest)
{
MyLogger.Assert(Status != UploadStatus.Uploading);
lock (semaphoreSlim) semaphoreSlim.Release();
}
}
}
private async Task DoResumableUploadAsync(long startByte, byte[] dataBytes)
{
var request = WebRequest.CreateHttp(resumableUri);
request.Headers["Content-Length"] = dataBytes.Length.ToString();
request.Headers["Content-Range"] = $"bytes {startByte}-{startByte + dataBytes.Length - 1}/{fileStream.Length}";
request.Method = "PUT";
using (System.IO.Stream requestStream = await request.GetRequestStreamAsync())
{
await requestStream.WriteAsync(dataBytes, 0, dataBytes.Length);
}
using (var response = await GetHttpResponseAsync(request))
{
if (response == null)
{
MyLogger.Log("b");
Status = UploadStatus.ErrorNeedResume;
return;
}
if ((int)response.StatusCode == 308)
{
if (Array.IndexOf(response.Headers.AllKeys, "range") == -1)
{
MessageAppended?.Invoke("No bytes have been received by the server yet, starting from the first byte...");
byteReceivedSoFar = 0;
}
else
{
//bytes=0-42
string s = response.Headers["range"];
//MyLogger.Log($"Range received by server: {s}");
string pattern = "bytes=0-";
MyLogger.Assert(s.StartsWith(pattern));
byteReceivedSoFar = long.Parse(s.Substring(pattern.Length)) + 1;
}
return;
}
else if (response.StatusCode == HttpStatusCode.OK || response.StatusCode == HttpStatusCode.Created)
{
using (var reader = new System.IO.StreamReader(response.GetResponseStream()))
{
var data = await reader.ReadToEndAsync();
//MyLogger.Log($"{data}");
var jsonObject = JsonConvert.DeserializeObject <Newtonsoft.Json.Linq.JObject>(data);
MyLogger.Assert(fileName == (string)jsonObject["name"]);
CloudFileId = (string)jsonObject["id"];
}
byteReceivedSoFar = fileStream.Length;
Status = UploadStatus.Completed;
return;
}
else
{
MessageAppended?.Invoke("Http response isn't 308!");
MessageAppended?.Invoke(await LogHttpWebResponse(response, true));
Status = UploadStatus.ErrorNeedResume;
return;
}
}
}
private static async Task <HttpWebResponse> GetHttpResponseAsync(HttpWebRequest request)
{
await semaphoreSlimGetHttpResponse.WaitAsync();
try
{
cnt++;
if ((DateTime.Now - time).TotalSeconds > 30)
{
MyLogger.Log($"{cnt} requests from {DateTime.Now} to {time}");
time = DateTime.Now;
cnt = 0;
//await MyLogger.Alert($"cnt={a}");
}
HttpWebResponse ans;
int minisecs = 500;
indexRetry :;
try
{
ans = (await request.GetResponseAsync()) as HttpWebResponse;
}
catch (WebException error)
{
ans = error.Response as HttpWebResponse;
}
finally
{
request.Abort();
}
if (ans == null)
{
MyLogger.Log("Got null response");
return(null);
}
switch (ans.StatusCode)
{
case HttpStatusCode.InternalServerError:
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.GatewayTimeout:
case HttpStatusCode.Forbidden:
{
MyLogger.Log(await LogHttpWebResponse(ans, true));
if (minisecs > Constants.MaxTimeToWait)
{
MyLogger.Log("Attempted to reconnect but still failed.");
return(ans);
}
else
{
MyLogger.Log($"Response: {ans.StatusCode} ({(int)ans.StatusCode}), waiting for {minisecs} ms and try again...");
await Task.Delay(minisecs);
minisecs *= 2;
goto indexRetry;
}
}
case HttpStatusCode.Unauthorized:
{
MyLogger.Log("Http response: Unauthorized (401). May due to expired access token, refreshing...");
MyLogger.Log(await LogHttpWebResponse(ans, true));
MyLogger.Assert(Array.IndexOf(request.Headers.AllKeys, "Authorization") != -1);
request.Headers["Authorization"] = "Bearer " + (await Drive.RefreshAccessTokenAsync());
await Task.Delay(500);
goto indexRetry;
}
default: return(ans);
}
}
finally
{
lock (semaphoreSlimGetHttpResponse) semaphoreSlimGetHttpResponse.Release();
}
}
private static void Log(string log)
{
MyLogger.Log(log);
}