public override object TransformData(IFreeDocument datas)
{
var p = datas[Column];
if (p == null)
{
return(new List <FreeDocument>());
}
var url = p.ToString();
WebHeaderCollection responseHeader;
HttpStatusCode code;
var content = helper.GetHtml(crawler.Http, out responseHeader, out code, url);
var keys = responseHeader.AllKeys;
if (!string.IsNullOrEmpty(HeaderFilter))
{
keys = HeaderFilter.Split(' ');
}
foreach (var key in keys)
{
var value = responseHeader.Get(key);
if (value != null)
{
datas.SetValue(key, value);
}
}
if (keys.Contains("Location") && datas.ContainsKey("Location") == false)
{
datas["Location"] = url;
}
return(null);
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null)
{
item = "";
}
var strs = new List <object> {
item
};
if (string.IsNullOrEmpty(Format))
{
return(item);
}
var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
strs.AddRange(columns.Select(key =>
{
if (datas.ContainsKey(key))
{
return(datas[key]);
}
return(key);
}));
return(string.Format(Format, strs.ToArray()));
}
object TransformData(IFreeDocument datas)
{
var p = datas[Column];
if (p == null)
{
return(new List <FreeDocument>());
}
var url = p.ToString();
var response = helper.GetHtml(Crawler.Http, url).Result;
var content = response.Html;
var code = response.Code;
var responseHeader = response.ResponseHeaders;
var keys = responseHeader.AllKeys;
if (!string.IsNullOrEmpty(HeaderFilter))
{
keys = HeaderFilter.Split(' ');
}
foreach (var key in keys)
{
var value = responseHeader.Get(key);
if (value != null)
{
datas.SetValue(key, value);
}
}
if (keys.Contains("Location") && datas.ContainsKey("Location") == false)
{
datas["Location"] = url;
}
return(null);
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null)
{
item = "";
}
var strs = new List <object> {
item
};
if (string.IsNullOrEmpty(Format))
{
return(item);
}
var format = datas.Query(Format);
var exps = rgx.Matches(format);
foreach (Match exp in exps)
{
format = format.Replace(exp.Value, datas.Query(exp.Value));
}
var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
strs.AddRange(columns.Select(key =>
{
if (datas.ContainsKey(key))
{
return(datas[key]);
}
return(key);
}));
return(string.Format(format, strs.ToArray()));
}
public override object TransformData(IFreeDocument document)
{
if (document.ContainsKey(Column))
{
document.Remove(Column);
}
return(null);
}
public override object TransformData(IFreeDocument document)
{
if (document.ContainsKey(Column))
{
document.Remove(Column);
}
return null;
}
public override object TransformData(IFreeDocument datas)
{
object result = null;
//获取输出列
var o_columns = new List <string>();
if (string.IsNullOrWhiteSpace(this.NewColumn))
{
o_columns.Add(this.Column);
}
else
{
o_columns.AddRange(NewColumn.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
.Select(d => d.Trim())
);
}
var indexs = Index.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries)
.Select(d => int.Parse(d.Trim())).ToList();
if (datas.ContainsKey(Column))
{
if (datas[Column] == null)
{
return(null);
}
var data = datas[Column];
var r = data.ToString();
List <string> items = null;
if (ShouldSplitChars == false)
{
items = r.Split(splitstrs.ToArray(), StringSplitOptions.None)
.Select(d => d.Trim())
.ToList();
}
else
{
items = r.Select(d => d.ToString()).ToList();
}
for (int i = 0; i < Math.Min(indexs.Count, o_columns.Count); i++)
{
datas[o_columns[i]] = GetValue(items, indexs[i]);
}
}
return(null);
}
public override object TransformData(IFreeDocument datas)
{
object result = null;
if (datas.ContainsKey(Column))
{
if (datas[Column] == null)
{
return(null);
}
var data = datas[Column];
var r = data.ToString();
List <string> items = null;
if (ShouldSplitChars == false)
{
items = r.Split(splitstrs.ToArray(), StringSplitOptions.RemoveEmptyEntries)
.Select(d => d.Trim())
.ToList();
}
else
{
items = r.Select(d => d.ToString()).ToList();
}
if (items.Count <= Index)
{
return(result);
}
if (FromBack == false)
{
result = items[Index];
}
else
{
var index = items.Count - Index - 1;
if (index < 0)
{
result = items[index];
}
}
}
return(result);
}
public override object TransformData(IFreeDocument datas)
{
object result = null;
if (datas.ContainsKey(Column))
{
if (datas[Column] == null)
return null;
var data = datas[Column];
var r = data.ToString();
List<string> items = null;
if (ShouldSplitChars == false)
{
items = r.Split(splitstrs.ToArray(), StringSplitOptions.RemoveEmptyEntries)
.Select(d => d.Trim())
.ToList();
}
else
{
items = r.Select(d => d.ToString()).ToList();
}
if (items.Count <= Index)
return result;
if (FromBack == false)
result = items[Index];
else
{
var index = items.Count - Index - 1;
if (index < 0)
result = items[index];
}
}
return result;
}