Beispiel #1
0
        public override object TransformData(IFreeDocument datas)
        {
            var item = datas[Column];

            if (item == null || string.IsNullOrWhiteSpace(item.ToString()))
            {
                return(null);
            }
            bool isrealjson;
            var  itemstr = item.ToString();

            if (lastData == null)
            {
                var html = JavaScriptAnalyzer.Json2XML(itemstr, out isrealjson, true);
                if (isrealjson)
                {
                    lastData = itemstr;
                }
            }

            if (crawlerEnabled)
            {
                var html = JavaScriptAnalyzer.Json2XML(itemstr, out isrealjson, true);
                if (isrealjson)
                {
                    HtmlDocument htmldoc = null;
                    var          doc     = selector.CrawlHtmlData(html, out htmldoc).FirstOrDefault();
                    doc.DictCopyTo(datas);
                }
                return(null);
            }
            dynamic d = null;

            try
            {
                d = serialier.DeserializeObject(item.ToString());
            }
            catch (Exception ex)
            {
                SetValue(datas, ex.Message);
                // XLogSys.Print.Error(ex);
                return(null);
            }
            if (ScriptWorkMode == ScriptWorkMode.单文档)
            {
                var newdoc = ScriptHelper.ToDocument(d) as FreeDocument;
                newdoc.DictCopyTo(datas);
            }
            else
            {
                SetValue(datas, d);
            }

            return(null);
        }
Beispiel #2
0
        public override IEnumerable <IFreeDocument> TransformManyData(IEnumerable <IFreeDocument> datas, AnalyzeItem analyzer)
        {
            foreach (var data in datas)
            {
                var item = data[Column].ToString();
                if (string.IsNullOrEmpty(item))
                {
                    continue;
                }
                var itemstr = item;
                lastData = itemstr;
                if (crawlerEnabled)
                {
                    bool isrealjson;
                    var  html = JavaScriptAnalyzer.Json2XML(itemstr, out isrealjson, true);
                    if (isrealjson)
                    {
                        HtmlDocument htmldoc = null;
                        var          doc     = selector.CrawlHtmlData(html, out htmldoc);
                        foreach (var item3 in doc)
                        {
                            yield return(item3.MergeQuery(data, NewColumn));
                        }
                    }
                    continue;
                }
                dynamic d = null;
                try
                {
                    d = serialier.DeserializeObject(itemstr);
                }
                catch (Exception ex)
                {
                    //  XLogSys.Print.Error(ex);
                    continue;
                }


                foreach (var item2 in ScriptHelper.ToDocuments(d))
                {
                    var item3 = item2 as FreeDocument;
                    yield return(item3.MergeQuery(data, NewColumn));
                }
            }
        }