public async Task UpdateAsync(FileCrawler fileCrawler, DateTime date) { List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true); foreach (ParameterCombination parameterCombination in parameterCombinations) { List <FileCrawlerRecord> fileCrawlerRecordList = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, date); List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination); foreach (FileCrawlerRecord record in list) { FileCrawlerRecord fileCrawlerRecord = fileCrawlerRecordList.FirstOrDefault(x => x.SourceTime == record.SourceTime && x.TargetTime == record.TargetTime); if (fileCrawlerRecord == null) { fileCrawlerRecord = record; await FileCrawlerRecordRepository.InsertAsync(fileCrawlerRecord); } else { fileCrawlerRecord.Url = record.Url; fileCrawlerRecord.FileName = record.FileName; fileCrawlerRecord.Stamp = record.Stamp; await FileCrawlerRecordRepository.UpdateAsync(fileCrawlerRecord); } await EnqueueAsync(fileCrawlerRecord.Id, fileCrawlerRecord.SourceTime, fileCrawler.DelaySeconds); } } }
public override async Task ExecuteAsync(FileCrawlerRecordDownloadingArgs args) { FileCrawlerRecord record = await FileCrawlerRecordRepository.GetAsync(args.FileCrawlerRecordId); if (!Directory.Exists(record.DirectoryName)) { Directory.CreateDirectory(record.DirectoryName); } HttpClient httpClient = HttpClientFactory.CreateClient(); httpClient.Timeout = TimeSpan.FromSeconds(20); byte[] bytes = await httpClient.GetByteArrayAsync(record.Url); await File.WriteAllBytesAsync(string.Format("{0}\\{1}", record.DirectoryName, record.FileName), bytes); record.Status = true; await FileCrawlerRecordRepository.UpdateAsync(record); }