public async Task UpdateAsync(FileCrawler fileCrawler, DateTime date)
{
List <ParameterCombination> parameterCombinations = await ParameterCombinationRepository.GetListAsync(fileCrawler.Id, true);
foreach (ParameterCombination parameterCombination in parameterCombinations)
{
List <FileCrawlerRecord> fileCrawlerRecordList = await FileCrawlerRecordRepository.GetListAsync(parameterCombination.Id, date);
List <FileCrawlerRecord> list = CreateFileCrawlerRecordList(fileCrawler, date, parameterCombination);
foreach (FileCrawlerRecord record in list)
{
FileCrawlerRecord fileCrawlerRecord = fileCrawlerRecordList.FirstOrDefault(x => x.SourceTime == record.SourceTime && x.TargetTime == record.TargetTime);
if (fileCrawlerRecord == null)
{
fileCrawlerRecord = record;
await FileCrawlerRecordRepository.InsertAsync(fileCrawlerRecord);
}
else
{
fileCrawlerRecord.Url = record.Url;
fileCrawlerRecord.FileName = record.FileName;
fileCrawlerRecord.Stamp = record.Stamp;
await FileCrawlerRecordRepository.UpdateAsync(fileCrawlerRecord);
}
await EnqueueAsync(fileCrawlerRecord.Id, fileCrawlerRecord.SourceTime, fileCrawler.DelaySeconds);
}
}
}
public override async Task ExecuteAsync(FileCrawlerRecordDownloadingArgs args)
{
FileCrawlerRecord record = await FileCrawlerRecordRepository.GetAsync(args.FileCrawlerRecordId);
if (!Directory.Exists(record.DirectoryName))
{
Directory.CreateDirectory(record.DirectoryName);
}
HttpClient httpClient = HttpClientFactory.CreateClient();
httpClient.Timeout = TimeSpan.FromSeconds(20);
byte[] bytes = await httpClient.GetByteArrayAsync(record.Url);
await File.WriteAllBytesAsync(string.Format("{0}\\{1}", record.DirectoryName, record.FileName), bytes);
record.Status = true;
await FileCrawlerRecordRepository.UpdateAsync(record);
}
protected List <FileCrawlerRecord> CreateFileCrawlerRecordList(FileCrawler fileCrawler, DateTime date, ParameterCombination parameterCombination)
{
List <FileCrawlerRecord> list = new List <FileCrawlerRecord>();
Dictionary <string, object> dic = JsonSerializer.Deserialize <Dictionary <string, object> >(parameterCombination.Parameters);
List <FileCrawlerPeriod> periods = JsonSerializer.Deserialize <List <FileCrawlerPeriod> >(parameterCombination.Periods);
if (fileCrawler.Type == FileCrawlerType.Archive)
{
foreach (FileCrawlerPeriod period in periods)
{
Func <DateTime, int, DateTime> getTimeFunc = GetGetTimeFunc(period.Type);
Func <TimeSpan, int> getOffsetFunc = GetGetOffsetFunc(period.Type);
for (int i = period.Start; i <= period.End; i += period.Interval)
{
DateTime sourceTime = getTimeFunc(date, i);
dic["SourceTime"] = sourceTime;
dic["SourceTimeSpan"] = sourceTime - date;
dic["SourceTimeOffset"] = i;
object[] args = dic.Values.ToArray();
object[] urlArgs;
if (fileCrawler.UrlDateTimeKind == DateTimeKind.Utc)
{
DateTime sourceTimeUtc = sourceTime.ToUniversalTime();
dic["SourceTime"] = sourceTimeUtc;
dic["SourceTimeSpan"] = sourceTimeUtc - sourceTimeUtc.Date;
dic["SourceTimeOffset"] = getOffsetFunc(sourceTimeUtc - sourceTimeUtc.Date);
urlArgs = dic.Values.ToArray();
}
else
{
urlArgs = args;
}
FileCrawlerRecord record = new FileCrawlerRecord(GuidGenerator.Create(), parameterCombination.Id, sourceTime, sourceTime)
{
Url = string.Format(fileCrawler.UrlFormat, urlArgs),
DirectoryName = string.Format("{0}\\{1:yyyyMMdd}", Configuration["Settings:Crawlers.Files.RootDirectory"], sourceTime),
FileName = string.Format(fileCrawler.FileNameFormat, args),
Stamp = string.Format(fileCrawler.StampFormat, args)
};
list.Add(record);
}
}
}
else
{
int sourceTimeOffset = int.Parse(dic["TimeOffset"] as string);
DateTime sourceTime = date.AddHours(sourceTimeOffset);
dic["SourceTime"] = sourceTime;
dic["SourceTimeSpan"] = sourceTime - date;
dic["SourceTimeOffset"] = sourceTimeOffset;
foreach (FileCrawlerPeriod period in periods)
{
Func <DateTime, int, DateTime> getTimeFunc = GetGetTimeFunc(period.Type);
for (int i = period.Start; i <= period.End; i += period.Interval)
{
DateTime targetTime = getTimeFunc(sourceTime, i);
dic["TargetTime"] = targetTime;
dic["TargetTimeSpan"] = targetTime - sourceTime;
dic["TargetTimeOffset"] = i;
object[] args = dic.Values.ToArray();
object[] urlArgs;
if (fileCrawler.UrlDateTimeKind == DateTimeKind.Utc)
{
DateTime sourceTimeUtc = sourceTime.ToUniversalTime();
DateTime targetTimeUtc = targetTime.ToUniversalTime();
dic["SourceTime"] = sourceTimeUtc;
dic["SourceTimeSpan"] = sourceTimeUtc - sourceTimeUtc.Date;
dic["SourceTimeOffset"] = (sourceTimeUtc - sourceTimeUtc.Date).Hours;
dic["TargetTime"] = targetTimeUtc;
urlArgs = dic.Values.ToArray();
}
else
{
urlArgs = args;
}
FileCrawlerRecord record = new FileCrawlerRecord(GuidGenerator.Create(), parameterCombination.Id, sourceTime, targetTime)
{
Url = string.Format(fileCrawler.UrlFormat, urlArgs),
DirectoryName = string.Format("{0}\\{1:yyyyMMdd}", Configuration["Settings:Crawlers.Files.RootDirectory"], sourceTime),
FileName = string.Format(fileCrawler.FileNameFormat, args),
Stamp = string.Format(fileCrawler.StampFormat, args)
};
list.Add(record);
}
}
}
return(list);
}
