public async Task UpdateAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> fileCrawlerRecordList = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, date);
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
foreach (FileCrawlerRecord record in list)
{
FileCrawlerRecord fileCrawlerRecord = fileCrawlerRecordList.FirstOrDefault(x => x.SourceTime == record.SourceTime && x.TargetTime == record.TargetTime);
if (fileCrawlerRecord == null)
{
fileCrawlerRecord = record;
await FileCrawlerRecordRepository.InsertAsync(fileCrawlerRecord);
}
else
{
fileCrawlerRecord.Url = record.Url;
fileCrawlerRecord.FileName = record.FileName;
fileCrawlerRecord.Stamp = record.Stamp;
await FileCrawlerRecordRepository.UpdateAsync(fileCrawlerRecord);
}
await EnqueueAsync(fileCrawlerRecord.Id, fileCrawlerRecord.SourceTime, fileCrawler.DelaySeconds);
}
}
}
public override async Task ExecuteAsync(FileCrawlerRecordDownloadingArgs args)
{
FileCrawlerRecord record = await FileCrawlerRecordRepository.GetAsync(args.FileCrawlerRecordId);
if (!Directory.Exists(record.DirectoryName))
{
Directory.CreateDirectory(record.DirectoryName);
}
HttpClient httpClient = HttpClientFactory.CreateClient();
httpClient.Timeout = TimeSpan.FromSeconds(20);
byte[] bytes = await httpClient.GetByteArrayAsync(record.Url);
await File.WriteAllBytesAsync(string.Format("{0}\\{1}", record.DirectoryName, record.FileName), bytes);
record.Status = true;
await FileCrawlerRecordRepository.UpdateAsync(record);
}
