public ParamWithId GetParameter(object templateParameter, string identificationCode)
{
WebSeed seed = JsonConvert.DeserializeObject <WebSeed>(templateParameter.ToString());
HttpSource httpSource = new HttpSource()
{
GenName = GenType,
Accept = seed.Accept,
ContentType = seed.ContentType,
Encoding = seed.Encoding,
Header = seed.Header,
Method = seed.Method,
Referer = seed.Referer,
Url = seed.SeedUrl,
UserAgent = seed.UserAgent,
Layer = seed.Depth
};
string hashid = (seed.SeedUrl + this.GenType + identificationCode).ToMD5Hex();
string recommendLoacation = $"level{seed.Depth}/" + hashid;
return(new ParamWithId()
{
Parameter = httpSource,
Id = hashid,
RecommendLocation = recommendLoacation,
SourceType = "Http"
});
}
public async Task ScheduleBack(ScrapySource source, PlatformModel platformModel, List <string> urls, ScheduleMessage scheduleMessage)
{
HttpSource httpSource = JsonConvert.DeserializeObject <HttpSource>(source.Source.Parameters.ToString());
if (httpSource.Layer > 0)
{
var transforms = scheduleMessage.Transforms
.Where(x => x.MapToSource.Contains(source.Name))
.Select(x => { x.MapToSource = new string[] { source.Name }; return(x); })
.ToArray();
var scheduleSource = scheduleMessage.Sources.Where(x => x.Name == source.Name).First();
var loadMaps = scheduleMessage.LandingTargets.LoadMaps
.Where(x => transforms.Any(y => y.Name == x.FromTransform))
.ToArray();
WebSeed webSeed = JsonConvert.DeserializeObject <WebSeed>(scheduleSource.Parameters.ToString());
foreach (var url in urls)
{
webSeed.SeedUrl = url;
webSeed.Depth = httpSource.Layer - 1;
scheduleSource.Parameters = webSeed;
ScheduleMessage subSchedule = new ScheduleMessage()
{
MessageId = scheduleMessage.MessageId,
Sources = new ScheduleSource[] { scheduleSource },
Transforms = transforms,
LandingTargets = new ScheduleLoad()
{
LoadProviders = scheduleMessage.LandingTargets.LoadProviders,
LoadMaps = loadMaps
},
MessageName = scheduleMessage.MessageName,
Scheduler = scheduleMessage.Scheduler
};
await Task.Delay(sendRate);
await ScheduleNew(subSchedule, platformModel);
}
}
}
