public ParamWithId GetParameter(object templateParameter, string identificationCode) { WebSeed seed = JsonConvert.DeserializeObject <WebSeed>(templateParameter.ToString()); HttpSource httpSource = new HttpSource() { GenName = GenType, Accept = seed.Accept, ContentType = seed.ContentType, Encoding = seed.Encoding, Header = seed.Header, Method = seed.Method, Referer = seed.Referer, Url = seed.SeedUrl, UserAgent = seed.UserAgent, Layer = seed.Depth }; string hashid = (seed.SeedUrl + this.GenType + identificationCode).ToMD5Hex(); string recommendLoacation = $"level{seed.Depth}/" + hashid; return(new ParamWithId() { Parameter = httpSource, Id = hashid, RecommendLocation = recommendLoacation, SourceType = "Http" }); }
public async Task ScheduleBack(ScrapySource source, PlatformModel platformModel, List <string> urls, ScheduleMessage scheduleMessage) { HttpSource httpSource = JsonConvert.DeserializeObject <HttpSource>(source.Source.Parameters.ToString()); if (httpSource.Layer > 0) { var transforms = scheduleMessage.Transforms .Where(x => x.MapToSource.Contains(source.Name)) .Select(x => { x.MapToSource = new string[] { source.Name }; return(x); }) .ToArray(); var scheduleSource = scheduleMessage.Sources.Where(x => x.Name == source.Name).First(); var loadMaps = scheduleMessage.LandingTargets.LoadMaps .Where(x => transforms.Any(y => y.Name == x.FromTransform)) .ToArray(); WebSeed webSeed = JsonConvert.DeserializeObject <WebSeed>(scheduleSource.Parameters.ToString()); foreach (var url in urls) { webSeed.SeedUrl = url; webSeed.Depth = httpSource.Layer - 1; scheduleSource.Parameters = webSeed; ScheduleMessage subSchedule = new ScheduleMessage() { MessageId = scheduleMessage.MessageId, Sources = new ScheduleSource[] { scheduleSource }, Transforms = transforms, LandingTargets = new ScheduleLoad() { LoadProviders = scheduleMessage.LandingTargets.LoadProviders, LoadMaps = loadMaps }, MessageName = scheduleMessage.MessageName, Scheduler = scheduleMessage.Scheduler }; await Task.Delay(sendRate); await ScheduleNew(subSchedule, platformModel); } } }