private void SetDbPrepareStartUrls(ConfigurableDbPrepareStartUrls generalDbPrepareStartUrls, JObject jobject) { generalDbPrepareStartUrls.ConnectString = jobject.SelectToken("$.ConnectString")?.ToString(); generalDbPrepareStartUrls.Filters = jobject.SelectToken("$.Filters")?.ToObject <List <string> >(); generalDbPrepareStartUrls.FormateStrings = jobject.SelectToken("$.FormateStrings")?.ToObject <List <string> >(); var limit = jobject.SelectToken("$.Limit"); generalDbPrepareStartUrls.Limit = limit?.ToObject <int>() ?? int.MaxValue; generalDbPrepareStartUrls.TableName = jobject.SelectToken("$.TableName")?.ToString(); generalDbPrepareStartUrls.Source = jobject.SelectToken("$.Source").ToObject <DataSource>(); foreach (var column in jobject.SelectTokens("$.Columns[*]")) { var c = new ConfigurableDbPrepareStartUrls.Column() { Name = column.SelectToken("$.Name").ToString() }; foreach (var format in column.SelectTokens("$.Formatters[*]")) { var name = format.SelectToken("$.Name").ToString(); var formatterType = FormatterFactory.GetFormatterType(name); c.Formatters.Add((Formatter)format.ToObject(formatterType)); } generalDbPrepareStartUrls.Columns.Add(c); } }
public static List <Formatter.Formatter> GenerateFormatter(IEnumerable <JToken> selectTokens) { if (selectTokens == null) { return(new List <Formatter.Formatter>()); } var results = new List <Formatter.Formatter>(); foreach (var selectToken in selectTokens) { Type type = FormatterFactory.GetFormatterType(selectToken.SelectToken("$.Name")?.ToString()); if (type != null) { results.Add(selectToken.ToObject(type) as Formatter.Formatter); } } return(results); }
private List <TargetUrlExtractor> GetTargetUrlExtractInfos(List <JObject> targetUrlExtractInfos) { List <TargetUrlExtractor> list = new List <TargetUrlExtractor>(); foreach (var obj in targetUrlExtractInfos) { TargetUrlExtractor t = new TargetUrlExtractor(); t.Patterns = obj.SelectTokens("$.Patterns[*]").Select(p => p.ToString()).ToList(); t.Region = obj.SelectToken("$.Region").ToObject <Selector>(); foreach (var format in obj.SelectTokens("$.Formatters[*]")) { var name = format.SelectToken("$.Name").ToString(); var formatterType = FormatterFactory.GetFormatterType(name); t.Formatters.Add((Formatter)format.ToObject(formatterType)); } list.Add(t); } return(list); }
private List <PrepareStartUrls> GetPrepareStartUrls(List <JObject> jobjects) { if (jobjects == null || jobjects.Count == 0) { return(null); } var list = new List <PrepareStartUrls>(); foreach (var jobject in jobjects) { var type = jobject.SelectToken("$.Type")?.ToObject <PrepareStartUrls.Types>(); if (type == null) { throw new SpiderExceptoin("Missing PrepareStartUrls type: " + jobject); } switch (type) { case Configuration.PrepareStartUrls.Types.ConfigDb: { var generalDbPrepareStartUrls = new ConfigurableDbPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.DbList: { var generalDbPrepareStartUrls = new DbListPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.CommonDb: { var generalDbPrepareStartUrls = new DbCommonPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.Cycle: { list.Add(jobject.ToObject <CyclePrepareStartUrls>()); break; } case Configuration.PrepareStartUrls.Types.LinkSpider: { list.Add(jobject.ToObject <LinkSpiderPrepareStartUrls>()); break; } case Configuration.PrepareStartUrls.Types.Base: { var generalDbPrepareStartUrls = new BaseDbPrepareStartUrls(); foreach (var column in jobject.SelectTokens("$.Columns[*]")) { var c = new ConfigurableDbPrepareStartUrls.Column() { Name = column.SelectToken("$.Name").ToString() }; foreach (var format in column.SelectTokens("$.Formatters[*]")) { var name = format.SelectToken("$.Name").ToString(); var formatterType = FormatterFactory.GetFormatterType(name); c.Formatters.Add((Formatter)format.ToObject(formatterType)); } generalDbPrepareStartUrls.Columns.Add(c); } generalDbPrepareStartUrls.ConnectString = jobject.SelectToken("$.ConnectString").ToString(); generalDbPrepareStartUrls.FormateStrings = jobject.SelectToken("$.FormateStrings").ToObject <List <string> >(); generalDbPrepareStartUrls.Method = jobject.SelectToken("$.Method").ToString(); generalDbPrepareStartUrls.Origin = jobject.SelectToken("$.Origin").ToString(); generalDbPrepareStartUrls.PostBody = jobject.SelectToken("$.PostBody").ToString(); generalDbPrepareStartUrls.QueryString = jobject.SelectToken("$.QueryString").ToString(); generalDbPrepareStartUrls.Referer = jobject.SelectToken("$.Referer").ToString(); generalDbPrepareStartUrls.Source = jobject.SelectToken("$.Source").ToObject <DataSource>(); generalDbPrepareStartUrls.Extras = jobject.SelectToken("$.Extras").ToObject <Dictionary <string, object> >(); list.Add(generalDbPrepareStartUrls); break; } } } return(list); }
private List <PrepareStartUrls> GetPrepareStartUrls(List <JObject> jobjects) { if (jobjects == null || jobjects.Count == 0) { return(null); } var list = new List <PrepareStartUrls>(); foreach (var jobject in jobjects) { var type = jobject.SelectToken("$.Type")?.ToObject <PrepareStartUrls.Types>(); if (type == null) { throw new SpiderExceptoin("Missing PrepareStartUrls type: " + jobject); } switch (type) { case Configuration.PrepareStartUrls.Types.ConfigDb: { var generalDbPrepareStartUrls = new ConfigurableDbPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.DbList: { var generalDbPrepareStartUrls = new DbListPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.CommonDb: { var generalDbPrepareStartUrls = new DbCommonPrepareStartUrls(); SetDbPrepareStartUrls(generalDbPrepareStartUrls, jobject); list.Add(generalDbPrepareStartUrls); break; } case Configuration.PrepareStartUrls.Types.Cycle: { list.Add(jobject.ToObject <CyclePrepareStartUrls>()); break; } case Configuration.PrepareStartUrls.Types.LinkSpider: { list.Add(jobject.ToObject <LinkSpiderPrepareStartUrls>()); break; } case Configuration.PrepareStartUrls.Types.Base: { var generalDbPrepareStartUrls = new BaseDbPrepareStartUrls(); foreach (var column in jobject.SelectTokens("$.Columns[*]")) { var c = new ConfigurableDbPrepareStartUrls.Column() { Name = column.SelectToken("$.Name").ToString() }; foreach (var format in column.SelectTokens("$.Formatters[*]")) { var name = format.SelectToken("$.Name").ToString(); var formatterType = FormatterFactory.GetFormatterType(name); c.Formatters.Add((Formatter)format.ToObject(formatterType)); } generalDbPrepareStartUrls.Columns.Add(c); } list.Add(generalDbPrepareStartUrls); break; } } } return(list); }