internal async Task DownloadFileAsync(DownloadFile file)
{
string?directoryName = Path.GetDirectoryName(file.Path);
if (!string.IsNullOrEmpty(directoryName))
{
Directory.CreateDirectory(directoryName);
}
using (var wc = new TimeoutWebClient())
{
long lastBytes = 0;
wc.DownloadProgressChanged += (s, e) =>
{
lock (locker)
{
var progressedBytes = e.BytesReceived - lastBytes;
if (progressedBytes < 0)
{
return;
}
lastBytes = e.BytesReceived;
var progress = new FileDownloadProgress(
file, e.TotalBytesToReceive, progressedBytes, e.BytesReceived, e.ProgressPercentage);
FileDownloadProgressChanged?.Invoke(this, progress);
}
};
await wc.DownloadFileTaskAsync(file.Url, file.Path)
.ConfigureAwait(false);
}
}
public async Task Process()
{
var dirs = Directory.GetDirectories(basePath);
for (var i = 0; i < dirs.Length; i++)
{
var dir = dirs[i];
var rawJsonPath = Path.Combine(dir, "raw.json");
if (!File.Exists(rawJsonPath))
{
continue;
}
var info = JsonConvert.DeserializeObject <RawGrantItemInfo>(File.ReadAllText(rawJsonPath));
var ginfo = info.ToGrantInfo();
{
var imagePath = Path.Combine(dir, "image.jpg");
if (!File.Exists(imagePath))
{
this.Ui.AddStatus($"work on {i + 1}/{dirs.Length} image");
var url = imageBaseUrl + info.Image;
using (var wc = new TimeoutWebClient(10000))
{
await wc.DownloadFileTaskAsync(url, imagePath);
}
}
}
foreach (var link in ginfo.Links)
{
var filePath = Path.Combine(dir, $"{ginfo.Id}-{ginfo.Title}-{link.Title}.pdf");
if (File.Exists(filePath))
{
continue;
}
var jsonFilePath = Path.Combine(dir, $"{link.Title}.json");
switch (link)
{
case GrantItemPamLink gipl:
this.Ui.AddStatus($"work on {i + 1}/{dirs.Length} {link.Title} {gipl.Id}");
var pam = await ParsePamAsync(gipl.Type, gipl.Id, gipl.Index);
File.WriteAllText(jsonFilePath, JsonConvert.SerializeObject(pam));
using (var wc = new TimeoutWebClient(10000))
{
await wc.DownloadFileTaskAsync(pam.FileLink, filePath);
}
this.Ui.AddStatus($"Sleep {20}s");
await Task.Delay(20000);
break;
case GrantItemTxLink gitl:
default:
break;
}
}
if (i % 10 == 0)
{
this.Ui.AddStatus($"Sleep {1}s");
await Task.Delay(1000);
}
}
}
