public virtual void SaveOrUpdateEntity(
IFreeDocument updateItem, string tableName, IDictionary <string, object> keys, EntityExecuteType executeType = EntityExecuteType.InsertOrUpdate)
{
FreeDocument data = updateItem.DictSerialize(Scenario.Database);
foreach (var key in data.Keys.ToList())
{
var value = "";
if (data[key] != null)
{
value = data[key].ToString();
}
value = value.Replace("'", "''");
data[key] = value;
}
var str = ",".Join(data.Select(d => $"{d.Key}='{d.Value}'"));
try
{
ExecuteNonQuery($"update {GetTableName(tableName)} set {str} ");
}
catch (Exception e)
{
XLogSys.Print.Debug($"insert database error {e.Message}");
}
}
protected virtual string Insert(IFreeDocument data, string dbTableName)
{
FreeDocument item = data.DictSerialize(Scenario.Database);
var sb = new StringBuilder();
foreach (var o in item)
{
string value;
if (o.Value is DateTime)
{
value = ((DateTime)o.Value).ToString("s");
}
else
{
if (o.Value == null)
{
value = "null";
}
else
{
value = o.Value.ToString();
}
}
value = value.Replace("'", "''");
sb.Append($"'{value}',");
}
sb.Remove(sb.Length - 1, 1);
string sql = $"INSERT INTO {dbTableName} VALUES({sb})";
return(sql);
}
protected override IEnumerable <IFreeDocument> InternalTransformManyData(IFreeDocument document)
{
int itemPerPage = 1;
int total = 1;
int min = 1;
var col = string.IsNullOrEmpty(NewColumn) ? Column : NewColumn;
if (int.TryParse(document.Query(ItemPerPage), out itemPerPage) &&
int.TryParse(document[Column].ToString(), out total) && int.TryParse(document.Query(MinValue), out min))
{
if (itemPerPage == 0)
{
itemPerPage = 1;
}
var remainder = total % itemPerPage;
int totalp = total / itemPerPage;
if (remainder != 0)
{
totalp += 1;
}
for (int i = min; i < min + totalp; i += 1)
{
var doc = document.Clone();
doc[col] = i;
yield return(doc);
}
}
}
public override IEnumerable <IFreeDocument> Generate(IFreeDocument document = null)
{
//TODO
DateTime min, max;
TimeSpan span;
var minvalue = AppHelper.Query(MinValue, document);
var maxvalue = AppHelper.Query(MaxValue, document);
var interval = AppHelper.Query(Interval, document);
if (DateTime.TryParseExact(minvalue,
staticDateFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out min) && DateTime.TryParseExact(maxvalue,
staticDateFormat,
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out max) && TimeSpan.TryParseExact(interval,
staticSpanFormat,
CultureInfo.InvariantCulture,
TimeSpanStyles.None,
out span))
{
for (var i = min; i <= max; i += span)
{
var item = new FreeDocument();
item.Add(Column, i.ToString(Format));
yield return(item);
}
}
}
public IEnumerable <IFreeDocument> TransformManyData(IEnumerable <IFreeDocument> datas)
{
foreach (var data in datas)
{
if (IsCycle)
{
IFreeDocument newdata = data;
while (string.IsNullOrEmpty(newdata[Column].ToString()) == false)
{
var result = etl.Generate(process, IsExecute, new List <IFreeDocument> {
newdata.Clone()
}).FirstOrDefault();
if (result == null)
{
break;
}
yield return(result.Clone());
newdata = result;
}
}
else
{
var result = etl.Generate(process, IsExecute, new List <IFreeDocument> {
data.Clone()
});
foreach (var item in result)
{
yield return(item.MergeQuery(data, NewColumn));
}
}
}
}
public override object TransformData(IFreeDocument data)
{
var path = data[Column].ToString();
if (File.Exists(path))
return "True";
return "False";
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null)
return null;
dynamic d = null;
try
{
d = serialier.DeserializeObject(item.ToString());
}
catch (Exception ex)
{
SetValue(datas,ex.Message);
// XLogSys.Print.Error(ex);
return null;
}
if (ScriptWorkMode == ScriptWorkMode.单文档)
{
var newdoc = ScriptHelper.ToDocument(d) as FreeDocument;
newdoc.DictCopyTo(datas);
}
else
{
SetValue(datas, d);
}
return null;
}
public override IEnumerable <IFreeDocument> Generate(IFreeDocument document = null)
{
var path = FileName;
var result = document?.Query(FileName);
if (result != null)
{
path = result;
}
// var item = datas[Column].ToString();
var res = buffHelper.Get(path);
if (res != null)
{
yield return(res);
yield break;
}
var content = File.ReadAllText(path, AttributeHelper.GetEncoding(EncodingType));
var item = new FreeDocument();
item.Add(Column, content);
buffHelper.Set(path, item);
yield return(item);
}
public static bool IsEqual(this IFreeDocument value, IFreeDocument content)
{
var dic1 = value;
var dic2 = content;
foreach (var o in dic1)
{
object res;
if (dic2.TryGetValue(o.Key, out res))
{
if (res == null && o.Value != null)
{
return(false);
}
if (!res.Equals(o.Value))
{
return(false);
}
}
else
{
return(false);
}
}
return(true);
}
public void AddErrorLog(IFreeDocument item, Exception ex, IColumnProcess process)
{
if (string.IsNullOrEmpty(errorLogName))
{
return;
}
var param = item.Clone() as FreeDocument;
param["__SysObjectID"] = process.ObjectID;
param["__SysETL"] = (process as ToolBase)?.Father.Name;
param["__SysERROR"] = ex.Message;
param["__SysTime"] = DateTime.Now.ToString();
ControlExtended.UIInvoke(() =>
{
if (ConfigFile.GetConfig <DataMiningConfig>().IsAddErrorCollection)
{
if (errorCollection == null)
{
errorCollection = new DataCollection()
{
Name = errorLogName
};
DataManager.AddDataCollection(errorCollection);
}
errorCollection?.ComputeData.Add(param);
errorCollection?.OnPropertyChanged("Count");
}
else
{
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), process.Column, process.TypeName, ex));
}
});
}
public override object TransformData(IFreeDocument document)
{
var item = document[Column];
if (item is IFreeDocument)
{
return (item as IFreeDocument).GetDataFromXPath(XPath);
}
var docu = new HtmlDocument();
docu.LoadHtml(item.ToString());
if (GetText)
{
var path = docu.DocumentNode.GetTextNode();
var textnode = docu.DocumentNode.SelectSingleNode(path);
if (textnode != null)
return textnode.GetNodeText();
}
if (GetCount)
{
var textnode = docu.DocumentNode.SelectNodes(XPath);
return textnode.Count;
}
return docu.DocumentNode.GetDataFromXPath(document.Query(XPath));
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null)
{
return(null);
}
dynamic d = null;
try
{
d = serialier.DeserializeObject(item.ToString());
}
catch (Exception ex)
{
SetValue(datas, ex.Message);
// XLogSys.Print.Error(ex);
return(null);
}
if (ScriptWorkMode == ScriptWorkMode.单文档)
{
var newdoc = ScriptHelper.ToDocument(d) as FreeDocument;
newdoc.DictCopyTo(datas);
}
else
{
SetValue(datas, d);
}
return(null);
}
public virtual void SaveOrUpdateEntity(
IFreeDocument updateItem, string tableName, IDictionary <string, object> keys, EntityExecuteType executeType = EntityExecuteType.InsertOrUpdate)
{
var sb = new StringBuilder();
FreeDocument data = updateItem.DictSerialize(Scenario.Database);
if (data.Count >= 1)
{
foreach (var val in data)
{
sb.Append($" {val.Key} = '{val.Value}',");
}
sb = sb.Remove(sb.Length - 1, 1);
}
try
{
ExecuteNonQuery($"update {GetTableName(tableName)} set {sb} where {ToString()};");
}
catch
{
}
}
public override IEnumerable <IFreeDocument> Generate(IFreeDocument document = null)
{
if (Connector == null)
{
return(new List <IFreeDocument>());
}
var path = FileName;
var result = document?.Query(FileName);
if (result != null)
{
path = result;
}
Connector.FileName = path;
if (!IsExecute)
{
return(Connector.ReadFile());
}
else
{
return(Connector.ReadFile().CacheDo(buffHelper.GetOrCreate(path, new List <FreeDocument>()), this.Father.SampleMount * 2));
}
}
public override object TransformData(IFreeDocument datas)
{
object item = datas[Column];
if (item == null)
{
item = "";
}
List <object> strs = new List <object>();
strs.Add(item);
if (string.IsNullOrEmpty(Format))
{
return(item);
}
var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
strs.AddRange(columns.Select(key =>
{
if (datas.ContainsKey(key))
{
return(datas[key]);
}
return(key);
}));
return(string.Format(Format, strs.ToArray()));
}
public override IEnumerable <FreeDocument> Generate(IFreeDocument document = null)
{
InsertQueue(StartURL);
while (Enabled)
{
string currentURL;
if (queue.TryDequeue(out currentURL))
{
var urlhash = currentURL.GetHashCode();
if (StartURL != currentURL && URLHash.Contains(urlhash))
{
continue;
}
URLHash.Add(urlhash);
var doc = new FreeDocument();
doc.Add(Column, currentURL);
yield return(doc);
}
else
{
if (DelayTime > 0)
{
Thread.Sleep(1000);
XLogSys.Print.Debug("empty queue,wait 1s");
}
}
}
}
object TransformData(IFreeDocument datas)
{
var p = datas[Column];
if (p == null)
{
return(new List <FreeDocument>());
}
var url = p.ToString();
WebHeaderCollection responseHeader;
HttpStatusCode code;
var content = helper.GetHtml(crawler.Http, out responseHeader, out code, url);
var keys = responseHeader.AllKeys;
if (!string.IsNullOrEmpty(HeaderFilter))
{
keys = HeaderFilter.Split(' ');
}
foreach (var key in keys)
{
var value = responseHeader.Get(key);
if (value != null)
{
datas.SetValue(key, value);
}
}
if (keys.Contains("Location") && datas.ContainsKey("Location") == false)
{
datas["Location"] = url;
}
return(null);
}
public override object TransformData(IFreeDocument dict)
{
var item = dict[Column];
if (item == null)
{
return(null);
}
var items = regex.Split(item.ToString());
if (items.Length <= Index)
{
return(null);
}
if (FromBack == false)
{
dict.SetValue(NewColumn, items[Index]);
}
else
{
var index = items.Length - Index - 1;
if (index < 0)
{
return(null);
}
dict.SetValue(NewColumn, items[index]);
}
return(null);
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null)
{
item = "";
}
var strs = new List <object> {
item
};
if (string.IsNullOrEmpty(Format))
{
return(item);
}
var format = datas.Query(Format);
var exps = rgx.Matches(format);
foreach (Match exp in exps)
{
format = format.Replace(exp.Value, datas.Query(exp.Value));
}
var columns = MergeWith.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
strs.AddRange(columns.Select(key =>
{
if (datas.ContainsKey(key))
{
return(datas[key]);
}
return(key);
}));
return(string.Format(format, strs.ToArray()));
}
public override IEnumerable<FreeDocument> Generate(IFreeDocument document = null)
{
InsertQueue(StartURL);
while (Enabled)
{
string currentURL;
if (queue.TryDequeue(out currentURL))
{
var urlhash = currentURL.GetHashCode();
if (StartURL != currentURL && URLHash.Contains(urlhash))
{
continue;
}
URLHash.Add(urlhash);
var doc = new FreeDocument();
doc.Add(Column, currentURL);
yield return doc;
}
else
{
if (DelayTime > 0)
{
Thread.Sleep(1000);
XLogSys.Print.Debug("empty queue,wait 1s");
}
}
}
}
public override object TransformData(IFreeDocument document)
{
var item = document[Column];
if (item is IFreeDocument)
{
return((item as IFreeDocument).GetDataFromXPath(XPath));
}
var docu = new HtmlDocument();
docu.LoadHtml(item.ToString());
if (GetText)
{
var path = docu.DocumentNode.GetTextNode();
var textnode = docu.DocumentNode.SelectSingleNode(path);
if (textnode != null)
{
return(textnode.GetNodeText());
}
}
if (GetCount)
{
var textnode = docu.DocumentNode.SelectNodes(XPath);
return(textnode.Count);
}
return(docu.DocumentNode.GetDataFromXPath(document.Query(XPath)));
}
private object eval(IFreeDocument doc)
{
var value = doc[Column];
var dictionary = new PythonDictionary();
foreach (var data1 in doc)
{
dictionary.Add(data1.Key, data1.Value);
}
scope.SetVariable("data", dictionary);
scope.SetVariable("value", value);
foreach (var data in doc)
{
scope.SetVariable(data.Key, data.Value);
}
dynamic d;
try
{
d = compiledCode.Execute(scope);
}
catch (Exception ex)
{
d = ex.Message;
}
return(d);
}
public IFreeDocument MappingDocument(IFreeDocument doc)
{
if (doc == null)
{
return(null);
}
if (string.IsNullOrEmpty(MappingSet))
{
return(doc);
}
var newdoc = new FreeDocument();
doc.DictCopyTo(newdoc);
foreach (var item in MappingSet.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries))
{
var kv = item.Split(':');
if (kv.Length != 2)
{
continue;
}
if (kv[0] == kv[1])
{
continue;
}
if (newdoc.Keys.Contains(kv[0]))
{
newdoc[kv[1]] = newdoc[kv[0]];
newdoc.Remove(kv[0]);
}
}
return(newdoc);
}
public static IFreeDocument Transform(this IColumnDataTransformer ge,
IFreeDocument item, AnalyzeItem analyzeItem)
{
if (item == null)
{
return(new FreeDocument());
}
var dict = item;
object res = null;
try
{
if (ge.OneOutput && dict[ge.Column] == null)
{
if (analyzeItem != null)
{
analyzeItem.EmptyInput += 1;
}
}
else
{
res = ge.TransformData(dict);
}
}
catch (Exception ex)
{
res = ex.Message;
if (analyzeItem != null)
{
analyzeItem.Error++;
analyzeItem.Analyzer.AddErrorLog(item, ex, ge);
}
XLogSys.Print.Error(string.Format(GlobalHelper.Get("key_208"), ge.Column, ge.TypeName, res));
}
if (ge.OneOutput)
{
if (!string.IsNullOrWhiteSpace(ge.NewColumn))
{
if (res != null)
{
dict.SetValue(ge.NewColumn, res);
}
}
else
{
if (res != null)
{
dict.SetValue(ge.Column, res);
}
}
}
return(dict);
}
public override int Compare(IFreeDocument a, IFreeDocument b)
{
object a1 = a[Column];
object b1 = b[Column];
if (a1 == null || b1 == null)
return 0;
return String.CompareOrdinal(a1.ToString(), b1.ToString());
}
public override bool FilteDataBase(IFreeDocument data)
{
if (index > Skip && index <= Take + Skip)
{
return(true);
}
return(false);
}
public override object TransformData(IFreeDocument datas)
{
object item = datas[Column];
IDataBaseConnector con = ConnectorSelector.SelectItem;
if (con == null)
{
return(null);
}
TableInfo table = TableSelector.SelectItem;
if (table == null)
{
return(null);
}
var keys = KeyName.Split(' ');
var query = keys.ToDictionary(d => d, d => datas[d]);
if (IsMutliDatas)
{
var r = con.TryFindEntities(table.Name, query, null, -1,
SearchStrategy);
if (r.Any() == false)
{
return(null);
}
var dicts = r.Select(d => d.DictSerialize()).ToList();
foreach (string importColumn in ImportColumns)
{
List <object> res = new List <object>();
for (int i = 0; i < dicts.Count; i++)
{
res.Add(dicts[i][importColumn]);
}
if (res.Count != 0)
{
datas.SetValue(importColumn, res);
}
}
}
else
{
var r = con.TryFindEntities(table.Name, query, null, 1, SearchStrategy).FirstOrDefault();
if (r == null)
{
return(null);
}
FreeDocument dict = r.DictSerialize();
foreach (string importColumn in ImportColumns)
{
datas.SetValue(importColumn, dict[importColumn]);
}
}
return(null);
}
public override object TransformData(IFreeDocument document)
{
if (document.ContainsKey(Column))
{
document.Remove(Column);
}
return null;
}
public static IFreeDocument ReadDataFile(string path)
{
IFileConnector json = SmartGetExport(path);
json.FileName = path;
IFreeDocument r = json.ReadFile().LastOrDefault();
return(r);
}
public override object TransformData(IFreeDocument document)
{
object item = document[Column];
DateTime time = (DateTime)item ;
if (time == null)
return null;
return time.ToString(Format);
}
public override object TransformData(IFreeDocument document)
{
if (document.ContainsKey(Column))
{
document.Remove(Column);
}
return(null);
}
public IEnumerable <FreeDocument> Generate(IFreeDocument document = null)
{
var process = GetProcesses();
foreach (var item in etl.Generate(process, IsExecute).Select(d => d as FreeDocument))
{
yield return(item);
}
}
public object TransformData(IFreeDocument data)
{
var result = func(new List <IFreeDocument> {
data.Clone()
}).FirstOrDefault();
data.AddRange(result);
return(null);
}
public override object TransformData(IFreeDocument data)
{
var path = data[Column].ToString();
if (File.Exists(path))
{
return("True");
}
return("False");
}
protected override IEnumerable <IFreeDocument> InternalTransformManyData(IFreeDocument data)
{
var docs = GetDatas(data);
if (docs.Count == 0)
{
return(docs);
}
return(docs.Select(d => d.MergeQuery(data, NewColumn)));
}
public override object TransformData(IFreeDocument dict)
{
object item = dict[Column];
if (item == null)
return null;
string r = regex.Replace(item.ToString(), ReplaceText);
return r;
}
public override object TransformData(IFreeDocument datas)
{
var item = datas[Column];
if (item == null || string.IsNullOrWhiteSpace(item.ToString()))
{
return(null);
}
bool isrealjson;
var itemstr = item.ToString();
if (lastData == null)
{
var html = JavaScriptAnalyzer.Json2XML(itemstr, out isrealjson, true);
if (isrealjson)
{
lastData = itemstr;
}
}
if (crawlerEnabled)
{
var html = JavaScriptAnalyzer.Json2XML(itemstr, out isrealjson, true);
if (isrealjson)
{
HtmlDocument htmldoc = null;
var doc = selector.CrawlHtmlData(html, out htmldoc).FirstOrDefault();
doc.DictCopyTo(datas);
}
return(null);
}
dynamic d = null;
try
{
d = serialier.DeserializeObject(item.ToString());
}
catch (Exception ex)
{
SetValue(datas, ex.Message);
// XLogSys.Print.Error(ex);
return(null);
}
if (ScriptWorkMode == ScriptWorkMode.单文档)
{
var newdoc = ScriptHelper.ToDocument(d) as FreeDocument;
newdoc.DictCopyTo(datas);
}
else
{
SetValue(datas, d);
}
return(null);
}
public override IEnumerable<FreeDocument> Generate(IFreeDocument document = null)
{
DataCollection table = TableSelector.SelectItem;
if(table==null)
yield break;
var me = table.ComputeData.Skip(Position);
foreach (IDictionarySerializable item in me)
{
yield return item.Clone() as FreeDocument;
}
}
public static void SaveDataFile(string filename, IFreeDocument item)
{
IFileConnector json = SmartGetExport(filename);
json.FileName = filename;
var datas = new List <IFreeDocument> {
item
};
json.WriteAll(datas);
}
public override bool FilteDataBase(IFreeDocument data)
{
object item = data[Column];
if (item == null)
return true;
if (string.IsNullOrEmpty(Script)) return true;
MatchCollection r = regex.Matches(item.ToString());
if (r.Count < Count)
return false;
return true;
}
public override int Compare(IFreeDocument a, IFreeDocument b)
{
var a1 = a[Column];
var b1 = b[Column];
var res1 = false;
var n1= (double)AttributeHelper.ConvertTo(a1, SimpleDataType.DOUBLE, ref res1);
if (res1 == false)
return 0;
var n2 = (double) AttributeHelper.ConvertTo(b1, SimpleDataType.DOUBLE, ref res1);
if (res1 == false)
return 0;
return (int)(n1 - n2);
}
public override object TransformData(IFreeDocument datas)
{
object item = datas[Column];
if (item == null)
item = "";
List<object> strs = new List<object>();
strs.Add(item);
if (string.IsNullOrEmpty(Format))
return item;
var Columns = MergeWith.Split(new string[]{" "},StringSplitOptions.RemoveEmptyEntries);
strs.AddRange(Columns.Select(Column => datas[Column]));
return string.Format(Format, strs.ToArray());
}
public override object TransformData(IFreeDocument document)
{
object item = document[Column];
if (item == null)
return "";
switch (ConvertType)
{
case ConvertType.Decode:
return HttpUtility.HtmlDecode(item.ToString());
case ConvertType.Encode:
return HttpUtility.HtmlEncode(item.ToString());
}
return "";
}
public override object TransformData(IFreeDocument datas)
{
string item = datas[Column].ToString();
var res = buffHelper.Get(item);
if (res != null)
return res;
else
{
var content = File.ReadAllText(item, AttributeHelper.GetEncoding(EncodingType));
buffHelper.Set(item, content);
return content;
}
}
public override object TransformData(IFreeDocument doc)
{
var d = eval(doc);
if (ScriptWorkMode == ScriptWorkMode.不进行转换)
{
SetValue(doc, d);
}
else
{
var newdoc = ScriptHelper.ToDocument(d);
newdoc.DictCopyTo(doc);
}
return d;
}
public override IEnumerable<FreeDocument> Generate(IFreeDocument document = null)
{
int interval;
double max, min;
if (int.TryParse(document.Query( Interval), out interval)&&
double.TryParse(document.Query(MinValue), out min) && double.TryParse(document.Query(MaxValue), out max))
{
for (var i = Position * interval + min; i <= max; i += interval)
{
var item = new FreeDocument();
item.Add(Column, Math.Round(i, 5));
yield return item;
}
}
}
public override object TransformData(IFreeDocument datas)
{
//初始化方案信息实体类。
var item = datas[Column];
if (item == null)
return null;
try
{
var lat = item.ToString();
var lng = datas[Lng].ToString();
var bufkey = $"{lat},{lng}";
var newlocation = buffHelper.Get(bufkey);
if (newlocation == null)
{
var apiUrl =
$"http://api.map.baidu.com/place/v2/search?ak={apikey}&output={format}&query={Query}&page_size=10&page_num=0&scope=2&location={lat},{lng}&radius={Radius}";
var result = HttpHelper.GetWebSourceHtml(apiUrl, "utf-8");
if (AllResult)
{
return result;
}
//以 Get 形式请求 Api 地址
// var result = HttpHelper.DoGet(apiUrl, param);
dynamic info = serialier.DeserializeObject(result);
if (info["status"].ToInt32() == 0)
{
var first = info["results"][0];
newlocation = new FreeDocument();
newlocation[Query] = first["name"];
newlocation[Query+"_lat"] = first["location"]["lat"];
newlocation[Query+"_lng"] = first["location"]["lng"];
newlocation[Query+"_distance"] = first["detail_info"]["distance"];
}
buffHelper.Set(bufkey, newlocation);
}
newlocation.DictCopyTo(datas);
}
catch (Exception ex)
{
}
return true;
}
public override object TransformData(IFreeDocument dict)
{
object item = dict[Column];
if (string.IsNullOrEmpty(Script)) return item;
if (item == null) return null;
MatchCollection r = regex.Matches(item.ToString());
if (Index >= 0)
{
if (r.Count > Index)
{
return r[Index].Value; ;
}
}
return null;
}
public override object TransformData(IFreeDocument datas)
{
object item = datas[Column];
if (item == null)
return "";
var v = item.ToString();
v = v.Trim();
if (ReplaceInnerBlank == true)
{
v = v.Replace('\t', ' ');
v = v.Replace(Environment.NewLine, "");
}
if (ReplaceBlank == true)
{
v = v.Replace(" ", "");
}
return v;
}
public override object TransformData(IFreeDocument document)
{
object item = document[Column];
if (item != null)
{
document.Remove(Column);
if (!string.IsNullOrEmpty(NewColumn))
{
document.SetValue(NewColumn, item);
}
else
{
document.SetValue(Column + "1", item);
}
}
return null;
}
public override bool FilteDataBase(IFreeDocument data)
{
object item = data[this.Column];
if (item == null)
{
return false;
}
bool res = false;
var v = (double)AttributeHelper.ConvertTo(item, SimpleDataType.DOUBLE, ref res);
if (res == false)
{
return false;
}
double max=1, min=0;
if (double.TryParse(data.Query(Max), out max) && double.TryParse(data.Query(Min), out min))
return v >= min && v <=max;
return true;
}
public override object TransformData(IFreeDocument datas)
{
object item = datas[Column];
if (item == null)
return null;
var str = item.ToString();
if (Former == "" || End == "")
return item;
var start = str.IndexOf(Former);
if (start == -1)
return "";
if (HaveStartEnd==false)
start += Former.Length;
var end = str.IndexOf(End,start);
if (end == -1)
return "";
if (HaveStartEnd)
end += End.Length;
return str.Substring(start, end - start);
}
public override object TransformData(IFreeDocument dict)
{
var item = dict[Column];
if (item == null)
return null;
var items = regex.Split(item.ToString());
if (items.Length <= Index)
return null;
if (FromBack == false)
dict.SetValue(NewColumn, items[Index]);
else
{
var index = items.Length - Index - 1;
if (index < 0)
return null;
dict.SetValue(NewColumn, items[index]);
}
return null;
}
public override object TransformData(IFreeDocument datas)
{
//初始化方案信息实体类。
var item = datas[Column];
if (item == null)
return null;
try
{
var newlocation = buffHelper.Get(item.ToString());
if (newlocation == null)
{
//以 Get 形式请求 Api 地址
var apiUrl =
$"http://api.map.baidu.com/location/ip?ak={apikey}&ip={item}&coor=bd09ll";
//初始化方案信息实体类。
var result = HttpHelper.GetWebSourceHtml(apiUrl, "utf-8");
//以 Get 形式请求 Api 地址
// var result = HttpHelper.DoGet(apiUrl, param);
dynamic info = serialier.DeserializeObject(result);
if (info["status"].ToInt32() == 0)
{
newlocation = new FreeDocument();
newlocation["ip_content"] = info["address"];
newlocation["pos_lat"] = info["content"]["point"]["x"];
newlocation["pos_lng"] = info["content"]["point"]["y"];
}
buffHelper.Set(item.ToString(), newlocation);
}
newlocation.DictCopyTo(datas);
}
catch (Exception ex)
{
}
return true;
}
public override object TransformData(IFreeDocument datas)
{
//初始化方案信息实体类。
var item = datas[Column];
if (item == null)
return null;
try
{
var newlocation = buffHelper.Get(item.ToString());
if (newlocation == null)
{
//以 Get 形式请求 Api 地址
var r = datas.Query(Region);
var apiUrl =
$"http://api.map.baidu.com/place/v2/search?q={item}®ion={r}&output={format}&ak={apikey}";
//初始化方案信息实体类。
var result = HttpHelper.GetWebSourceHtml(apiUrl, "utf-8");
//以 Get 形式请求 Api 地址
// var result = HttpHelper.DoGet(apiUrl, param);
dynamic info = serialier.DeserializeObject(result);
// if (info[0]["status"].ToInt32() == 0)
{
newlocation = Parse(info);
}
buffHelper.Set(item.ToString(), newlocation);
}
newlocation.DictCopyTo(datas);
}
catch (Exception ex)
{
}
return true;
}
public override object TransformData(IFreeDocument datas)
{
object result = null;
if (datas.ContainsKey(Column))
{
if (datas[Column] == null)
return null;
var data = datas[Column];
var r = data.ToString();
List<string> items = null;
if (ShouldSplitChars == false)
{
items = r.Split(splitstrs.ToArray(), StringSplitOptions.RemoveEmptyEntries)
.Select(d => d.Trim())
.ToList();
}
else
{
items = r.Select(d => d.ToString()).ToList();
}
if (items.Count <= Index)
return result;
if (FromBack == false)
result = items[Index];
else
{
var index = items.Count - Index - 1;
if (index < 0)
result = items[index];
}
}
return result;
}
protected JsonObject GetJsonObject(IFreeDocument data)
{
IEnumerable<KeyValuePair<string, object>> dicts =
data.DictSerialize();
string[] keys = dicts.Select(d => d.Key).ToArray();
object[] value = dicts.Select(d => Node2Item(d.Value)).ToArray();
var rc = new JsonObject(keys, value);
return rc;
}
private int ItemtoNode(object d, IFreeDocument dict)
{
var doc = dict as FreeDocument;
if (doc == null)
{
if (d is JsonObject)
dict.DictDeserialize(d as JsonObject);
}
else
{
if (d is JsonArray)
{
foreach (var item in d as JsonArray)
{
var item2 = new FreeDocument();
ItemtoNode(item, item2);
if (doc.Children == null)
{
doc.Children = new List<FreeDocument>();
}
doc.Children.Add(item2);
}
return 1;
}
else if (d is JsonObject)
{
var jb = d as JsonObject;
foreach (var b in jb)
{
var dict2 = new FreeDocument();
var res = ItemtoNode(b.Value, dict2);
if (res == 0)
{
doc.Add(b.Name, b.Value);
}
else if (res == 1)
{
doc.Add(b.Name, dict2);
}
}
return 2;
}
}
return 0;
}
public override IEnumerable<FreeDocument> Generate(IFreeDocument document = null)
{
foreach (string doc in fileList)
{
var item= new FreeDocument();
item.Add(Column, doc);
yield return item;
}
}
public virtual int Compare(IFreeDocument a, IFreeDocument b)
{
return 0;
}
