public async Task LoadFromGoogleDrive(string folderId, string folderName, ILogger log)
{
var creds = GoogleCredential.FromJson(Cfg.Creds.ToString()).CreateScoped(DriveService.Scope.DriveReadonly);
var service = new DriveService(new() {
HttpClientInitializer = creds,
ApplicationName = "recfluence"
});
var list = service.Files.List();
list.Q = $"'{folderId}' in parents";
list.PageSize = 1000;
var files = await list.ExecuteAsync();
Dir.EnsureDirectoryExists();
async Task <FPath> Download(File f, FPath localFile)
{
if (localFile.Exists)
{
return(localFile);
}
using var sw = localFile.Open(CreateNew);
var progress = await service.Files.Get(f.Id).DownloadAsync(sw);
while (progress.Status.In(DownloadStatus.NotStarted, DownloadStatus.Downloading))
{
await 1.Seconds().Delay();
}
if (progress.Status == DownloadStatus.Completed)
{
return(localFile);
}
if (progress.Exception != null)
{
log.Error(progress.Exception, "error when downloading file {File}: {Message}", f.Name, progress.Exception.Message);
return(null);
}
log.Error("download did not complete {File}");
return(null);
}
await files.Files.WithIndex().BlockAction(async f => {
var localFile = Dir.Combine(f.item.Name.Trim().Split(".").First().Replace("Copy of ", "") + ".jsonl.gz");
var blobPath = $"parler/{folderName}/{localFile.FileName}";
if (await Db.Exists(blobPath))
{
log.Information("Skipping existing blob {File}", blobPath);
return;
}
var downloadedFile = await Download(f.item, localFile);
if (downloadedFile != null)
{
await Db.Save(blobPath, downloadedFile, Log);
}
log.Information("Moved {File} {Num}/{Total}", localFile.FileName, f.index + 1, files.Files.Count);
localFile.Delete();
}, 2);
Log.Information("parler - completed loading {Name}", folderName);
}
/// <summary>
/// Saves the result for the given query to Storage and a local tmp file
/// </summary>
async Task<FPath> SaveResult(IDbConnection db, FPath tempDir, ResQuery q) {
var sw = Stopwatch.StartNew();
var reader = await ResQuery(db, q);
var fileName = $"{q.Name}.csv.gz";
var tempFile = tempDir.Combine(fileName);
using (var fileWriter = tempFile.Open(FileMode.Create, FileAccess.Write))
await reader.WriteCsvGz(fileWriter, fileName, Log);
// save to both latest and the current date
await SaveToLatestAndDateDirs(fileName, tempFile);
Log.Information("Complete saving result {Name} in {Duration}", q.Name, sw.Elapsed);
return tempFile;
}
