public override object TransformData(IFreeDocument datas) { var p = datas[Column]; if (p == null) { return(new List <FreeDocument>()); } var url = p.ToString(); WebHeaderCollection responseHeader; HttpStatusCode code; var content = helper.GetHtml(crawler.Http, out responseHeader, out code, url); var keys = responseHeader.AllKeys; if (!string.IsNullOrEmpty(HeaderFilter)) { keys = HeaderFilter.Split(' '); } foreach (var key in keys) { var value = responseHeader.Get(key); if (value != null) { datas.SetValue(key, value); } } if (keys.Contains("Location") && datas.ContainsKey("Location") == false) { datas["Location"] = url; } return(null); }
public override object TransformData(IFreeDocument datas) { var item = datas[Column]; if (item == null) { item = ""; } var strs = new List <object> { item }; if (string.IsNullOrEmpty(Format)) { return(item); } var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); strs.AddRange(columns.Select(key => { if (datas.ContainsKey(key)) { return(datas[key]); } return(key); })); return(string.Format(Format, strs.ToArray())); }
object TransformData(IFreeDocument datas) { var p = datas[Column]; if (p == null) { return(new List <FreeDocument>()); } var url = p.ToString(); var response = helper.GetHtml(Crawler.Http, url).Result; var content = response.Html; var code = response.Code; var responseHeader = response.ResponseHeaders; var keys = responseHeader.AllKeys; if (!string.IsNullOrEmpty(HeaderFilter)) { keys = HeaderFilter.Split(' '); } foreach (var key in keys) { var value = responseHeader.Get(key); if (value != null) { datas.SetValue(key, value); } } if (keys.Contains("Location") && datas.ContainsKey("Location") == false) { datas["Location"] = url; } return(null); }
public override object TransformData(IFreeDocument datas) { var item = datas[Column]; if (item == null) { item = ""; } var strs = new List <object> { item }; if (string.IsNullOrEmpty(Format)) { return(item); } var format = datas.Query(Format); var exps = rgx.Matches(format); foreach (Match exp in exps) { format = format.Replace(exp.Value, datas.Query(exp.Value)); } var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries); strs.AddRange(columns.Select(key => { if (datas.ContainsKey(key)) { return(datas[key]); } return(key); })); return(string.Format(format, strs.ToArray())); }
public override object TransformData(IFreeDocument document) { if (document.ContainsKey(Column)) { document.Remove(Column); } return(null); }
public override object TransformData(IFreeDocument document) { if (document.ContainsKey(Column)) { document.Remove(Column); } return null; }
public override object TransformData(IFreeDocument datas) { object result = null; //获取输出列 var o_columns = new List <string>(); if (string.IsNullOrWhiteSpace(this.NewColumn)) { o_columns.Add(this.Column); } else { o_columns.AddRange(NewColumn.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries) .Select(d => d.Trim()) ); } var indexs = Index.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries) .Select(d => int.Parse(d.Trim())).ToList(); if (datas.ContainsKey(Column)) { if (datas[Column] == null) { return(null); } var data = datas[Column]; var r = data.ToString(); List <string> items = null; if (ShouldSplitChars == false) { items = r.Split(splitstrs.ToArray(), StringSplitOptions.None) .Select(d => d.Trim()) .ToList(); } else { items = r.Select(d => d.ToString()).ToList(); } for (int i = 0; i < Math.Min(indexs.Count, o_columns.Count); i++) { datas[o_columns[i]] = GetValue(items, indexs[i]); } } return(null); }
public override object TransformData(IFreeDocument datas) { object result = null; if (datas.ContainsKey(Column)) { if (datas[Column] == null) { return(null); } var data = datas[Column]; var r = data.ToString(); List <string> items = null; if (ShouldSplitChars == false) { items = r.Split(splitstrs.ToArray(), StringSplitOptions.RemoveEmptyEntries) .Select(d => d.Trim()) .ToList(); } else { items = r.Select(d => d.ToString()).ToList(); } if (items.Count <= Index) { return(result); } if (FromBack == false) { result = items[Index]; } else { var index = items.Count - Index - 1; if (index < 0) { result = items[index]; } } } return(result); }
public override object TransformData(IFreeDocument datas) { object result = null; if (datas.ContainsKey(Column)) { if (datas[Column] == null) return null; var data = datas[Column]; var r = data.ToString(); List<string> items = null; if (ShouldSplitChars == false) { items = r.Split(splitstrs.ToArray(), StringSplitOptions.RemoveEmptyEntries) .Select(d => d.Trim()) .ToList(); } else { items = r.Select(d => d.ToString()).ToList(); } if (items.Count <= Index) return result; if (FromBack == false) result = items[Index]; else { var index = items.Count - Index - 1; if (index < 0) result = items[index]; } } return result; }