public async Task UpdateAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> fileCrawlerRecordList = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, date);
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
foreach (FileCrawlerRecord record in list)
{
FileCrawlerRecord fileCrawlerRecord = fileCrawlerRecordList.FirstOrDefault(x => x.SourceTime == record.SourceTime && x.TargetTime == record.TargetTime);
if (fileCrawlerRecord == null)
{
fileCrawlerRecord = record;
await FileCrawlerRecordRepository.InsertAsync(fileCrawlerRecord);
}
else
{
fileCrawlerRecord.Url = record.Url;
fileCrawlerRecord.FileName = record.FileName;
fileCrawlerRecord.Stamp = record.Stamp;
await FileCrawlerRecordRepository.UpdateAsync(fileCrawlerRecord);
}
await EnqueueAsync(fileCrawlerRecord.Id, fileCrawlerRecord.SourceTime, fileCrawler.DelaySeconds);
}
}
}
public async Task InsertAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
await FileCrawlerRecordRepository.InsertManyAsync(list);
list.ForEach(async x => await EnqueueAsync(x.Id, x.SourceTime, fileCrawler.DelaySeconds));
}
}
public async Task <List <FileCrawlerRecordDto> > GetListAsync(GetFileCrawlerRecordListDto input)
{
FileCrawler fileCrawler = await FileCrawlerRepository.FindAsync(x => x.Code == input.FileCrawlerCode);
if (fileCrawler != null)
{
List <ParameterCombination> parameterCombinationList = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id);
ParameterCombination parameterCombination = parameterCombinationList.FirstOrDefault(x => Equal(x.Parameters, input.Parameters));
if (parameterCombination != null)
{
List <FileCrawlerRecord> list = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, input.Date);
return(ObjectMapper.Map <List <FileCrawlerRecord>, List <FileCrawlerRecordDto> >(list));
}
}
return(new List <FileCrawlerRecordDto>());
}
public async Task UpdateAsync(FileCrawler fileCrawler)
{
List <ParameterCombination> parameterCombinationList = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id);
parameterCombinationList.ForEach(x => x.Enabled = false);
List <Dictionary <string, string> > list = GetParameterCombinations(fileCrawler);
foreach (Dictionary <string, string> dic in list)
{
string periods;
if (dic.ContainsKey("Periods"))
{
periods = dic["Periods"];
dic.Remove("Periods");
}
else
{
periods = fileCrawler.Periods;
}
ParameterCombination parameterCombination = parameterCombinationList.FirstOrDefault(x => Equal(x, dic));
if (parameterCombination == null)
{
parameterCombination = new ParameterCombination(GuidGenerator.Create(), fileCrawler.Id);
parameterCombination.Periods = periods;
parameterCombination.Parameters = JsonSerializer.Serialize(dic);
parameterCombination.Enabled = true;
await ParameterCombinationRepository.InsertAsync(parameterCombination);
}
else
{
parameterCombination.Periods = periods;
parameterCombination.Enabled = true;
}
}
await ParameterCombinationRepository.UpdateManyAsync(parameterCombinationList);
}
