public async Task UpdateAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> fileCrawlerRecordList = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, date);
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
foreach (FileCrawlerRecord record in list)
{
FileCrawlerRecord fileCrawlerRecord = fileCrawlerRecordList.FirstOrDefault(x => x.SourceTime == record.SourceTime && x.TargetTime == record.TargetTime);
if (fileCrawlerRecord == null)
{
fileCrawlerRecord = record;
await FileCrawlerRecordRepository.InsertAsync(fileCrawlerRecord);
}
else
{
fileCrawlerRecord.Url = record.Url;
fileCrawlerRecord.FileName = record.FileName;
fileCrawlerRecord.Stamp = record.Stamp;
await FileCrawlerRecordRepository.UpdateAsync(fileCrawlerRecord);
}
await EnqueueAsync(fileCrawlerRecord.Id, fileCrawlerRecord.SourceTime, fileCrawler.DelaySeconds);
}
}
}
public async Task InsertAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
await FileCrawlerRecordRepository.InsertManyAsync(list);
list.ForEach(async x => await EnqueueAsync(x.Id, x.SourceTime, fileCrawler.DelaySeconds));
}
}
public override async Task ExecuteAsync(FileCrawlerRecordDownloadingArgs args)
{
FileCrawlerRecord record = await FileCrawlerRecordRepository.GetAsync(args.FileCrawlerRecordId);
if (!Directory.Exists(record.DirectoryName))
{
Directory.CreateDirectory(record.DirectoryName);
}
HttpClient httpClient = HttpClientFactory.CreateClient();
httpClient.Timeout = TimeSpan.FromSeconds(20);
byte[] bytes = await httpClient.GetByteArrayAsync(record.Url);
await File.WriteAllBytesAsync(string.Format("{0}\\{1}", record.DirectoryName, record.FileName), bytes);
record.Status = true;
await FileCrawlerRecordRepository.UpdateAsync(record);
}
public async Task <List <FileCrawlerRecordDto> > GetListAsync(GetFileCrawlerRecordListDto input)
{
FileCrawler fileCrawler = await FileCrawlerRepository.FindAsync(x => x.Code == input.FileCrawlerCode);
if (fileCrawler != null)
{
List <ParameterCombination> parameterCombinationList = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id);
ParameterCombination parameterCombination = parameterCombinationList.FirstOrDefault(x => Equal(x.Parameters, input.Parameters));
if (parameterCombination != null)
{
List <FileCrawlerRecord> list = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, input.Date);
return(ObjectMapper.Map <List <FileCrawlerRecord>, List <FileCrawlerRecordDto> >(list));
}
}
return(new List <FileCrawlerRecordDto>());
}
