public static RawlerBase Tags(this RawlerBase rawler, string tag, string className)
{
return(rawler.Add(new Tags()
{
Tag = tag, ClassName = className
}));
}
public RawlerBase CreateRawlerTree(RawlerBase rawler)
{
RawlerLib.Timer.StopWatch.Write("CreateRawlerTree Start");
var baseUrl = BaseUrl;
foreach (var item in Nodes)
{
urlTfidf.AddDocument(item.OuterHtml.ToHtml(baseUrl).GetLink().Select(n => n.Url));
imageTfidf.AddDocument(item.OuterHtml.ToHtml(baseUrl).GetImageLink().Select(n => n.Url));
}
if (rawler == null)
{
rawler = new RawlerBase();
}
//RawlerTreeを組み立てて、同じものを纏める。
var list = Nodes.GroupBy(n => n.GetHTMLWithoutValue()).Select(n => CreateRawler(n.First())).ToArray().GroupBy(n => n.ToXAML()).Select(n => n.First()).ToArray();
RawlerLib.Timer.StopWatch.Write("CreateRawlerTree AddStart");
foreach (var item in list)
{
item.AddFirst(new GetPageUrl().DataWrite("SourceUrl", DataAttributeType.SourceUrl).GetRoot());
item.Add(new NextDataRow());
rawler.Add(item);
}
RawlerLib.Timer.StopWatch.Write("CreateRawlerTree MargeStart");
rawler.MargeChildren();
RawlerLib.Timer.StopWatch.Write("CreateRawlerTree End");
return(rawler);
}
public static RawlerBase Page(this RawlerBase rawler, string url)
{
return(rawler.Add(new Page()
{
Url = url
}));
}
public static RawlerBase GetTesvValue(this RawlerBase rawler, string column)
{
return(rawler.Add(new GetTsvValue()
{
ColumnName = column
}));
}
public void Load(RawlerBase rawler)
{
var file = FileName.Convert(rawler);
if (string.IsNullOrEmpty(file) == false)
{
if (System.IO.File.Exists(file) == true)
{
LTSV = System.IO.File.ReadAllText(file);
}
}
var ltsv = LTSV.Convert(rawler);
if (string.IsNullOrEmpty(ltsv) == false)
{
foreach (var item in ltsv.ReadLines())
{
TaskParameter tp = new TaskParameter();
foreach (var dic in item.ParseLtsvLine())
{
tp.Add(new KeyValue(dic.Key, dic.Value));
}
if (tp.Count > 0)
{
this.Add(tp);
}
}
}
//ConvertにConvertを適用する。
foreach (var item in this.SelectMany(n => n))
{
item.Value = item.Value.Convert(rawler);
}
}
static void Main(string[] args)
{
Rawler.Tool.ReportManage.ErrReportEvent += ReportManage_ErrReportEvent;
Rawler.Tool.ReportManage.ReportEvnet += ReportManage_ReportEvnet;
// args = new List<string>() { @"C:\Users\kiichi\Documents\TwitterData\hamano_satoshi\tweet.xaml" }.ToArray();
if (args.Length > 0)
{
try
{
Dictionary <string, string> dic = new Dictionary <string, string>();
foreach (var item in args.Skip(1))
{
var d = item.Split('=');
if (d.Length > 1)
{
dic[d[0]] = d[1];
Rawler.Tool.TempVar.SetVar(d[0], d[1]);
}
}
RawlerBase rawler = (RawlerBase)System.Xaml.XamlServices.Load(args[0]);
rawler.SetParent();
rawler.Run();
}
catch (Exception e)
{
System.Console.WriteLine(e.ToString());
}
}
}
public static RawlerBase Tags(this RawlerBase rawler, string tag)
{
return(rawler.Add(new Tags()
{
Tag = tag
}));
}
//public new void Run()
//{
// Run(true);
//}
/// <summary>
/// 実行
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
string containsText = this.ContainsText.Convert(this);
string text = this.GetText();
if (ContainsTextTree != null)
{
containsText = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
}
if (TextTree != null)
{
text = RawlerBase.GetText(this.Parent.Text, TextTree, this);
}
if (text.Contains(containsText) == this.Result)
{
this.RunChildren(runChildren);
}
//if (ContainsTextTree == null)
//{
// if (this.GetText().Contains(this.ContainsText) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
//else
//{
// string t = RawlerBase.GetText(this.Parent.Text, ContainsTextTree, this);
// if (this.GetText().Contains(t) == this.Result)
// {
// this.RunChildren(runChildren);
// }
//}
}
public static RawlerBase DataWrite(this RawlerBase rawler, string attribute, DataAttributeType attributeType)
{
return(rawler.Add(new DataWrite()
{
Attribute = attribute, AttributeType = attributeType
}));
}
public static RawlerBase TagClear(this RawlerBase rawler, ReplaceType replece)
{
return(rawler.Add(new TagClear()
{
ReplaceType = replece
}));
}
protected override void CloneEvent(RawlerBase rawler)
{
base.CloneEvent(rawler);
if (rawler is CustomText)
{
var r = (CustomText)rawler;
r.CreateText = this.CreateText;
}
}
protected override void CloneEvent(RawlerBase rawler)
{
base.CloneEvent(rawler);
if (rawler is CustomFilter)
{
var r = (CustomFilter)rawler;
r.Check = this.Check;
}
}
public void addDocument(string text, RawlerBase rawlerTree)
{
var d = new Document();
d.SetText(text);
d.SetParent(this);
var tree = rawlerTree.Clone();
d.AddChildren(tree);
documents.Add(d);
}
protected string GetSwitchValue()
{
if (SwitchValueTree != null)
{
SwitchValueTree.SetParent();
return(RawlerBase.GetText(this.Parent.Text, SwitchValueTree, this.Parent));
}
else
{
return(GetText());
}
}
/// <summary>
/// クローンを作る
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
var clone = new ReLogin();
RawlerLib.ObjectLib.FildCopy(this, clone);
clone.SetParent(parent);
this.CloneEvent(clone);
clone.children.Clear();
foreach (var item in this.Children)
{
var child = item.Clone(clone);
clone.AddChildren(child);
}
return clone;
}
public override void Run(bool runChildren)
{
if (runChildren)
{
SetText(GetText());
if (TaskLTSV.NullIsEmpty().Length > 0)
{
TaskParameterList.LTSV = TaskLTSV.Convert(this);
}
if (TaskLTSVFile.NullIsEmpty().Length > 0)
{
TaskParameterList.FileName = TaskLTSVFile.Convert(this);
}
TaskParameterList.Load(this);
List <Task> list = new List <Task>();
KeyValueStore key = new KeyValueStore();
foreach (var item in this.Children)
{
key.AddChildren(item);
}
var xaml = key.ToXAML();
string err;
if (TaskParameterList.Count > 0)
{
foreach (var item in TaskParameterList)
{
var r = (KeyValueStore)RawlerBase.Parse(xaml, out err);
foreach (var keyvalue in item)
{
r.SetKeyValue(keyvalue.Key, keyvalue.Value);
}
r.SetParent(this.Parent);
list.Add(Task.Factory.StartNew(() => r.Run()));
}
}
else
{
for (int i = 0; i < TaskNum; i++)
{
var r = (KeyValueStore)RawlerBase.Parse(xaml, out err);
r.SetParent(this.Parent);
list.Add(Task.Factory.StartNew(() => r.Run()));
}
}
Task.WaitAll(list.ToArray());
}
}
public void Dispose()
{
pause = false;
if (rawler != null)
{
rawler.Dispose();
rawler = null;
}
if (tokenSource != null)
{
tokenSource.Cancel();
tokenSource.Dispose();
}
ReportManage.ListClear();
Application.Current.Shutdown();
}
/// <summary>
/// クローンを作る。
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<GetScriptVariable>(parent);
//var clone = new GetScriptVariable();
//RawlerLib.ObjectLib.FildCopy(this, clone);
//clone.SetParent(parent);
//this.CloneEvent(clone);
//clone.children.Clear();
//foreach (var item in this.Children)
//{
// var child = item.Clone(clone);
// clone.AddChildren(child);
//}
//return clone;
}
public void Dispose()
{
pause = false;
if (rawler != null)
{
rawler.Dispose();
rawler = null;
}
if (tokenSource != null)
{
tokenSource.Cancel();
tokenSource.Dispose();
}
ReportManage.ListClear();
Application.Current.Shutdown();
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (Text2Tree == null)
{
ReportManage.ErrReport(this, "Text2Treeが空です。実行にはText2Treeが必要です。");
return;
}
Text2Tree.SetParent(this);
string t = RawlerBase.GetText(this.parent.Text, Text2Tree, this);
var result = Compute(GetText(), t);
if (result <= maxDistance)
{
SetText(result.ToString());
base.Run(runChildren);
}
}
/// <summary>
/// データをゲットする。
/// </summary>
/// <returns></returns>
public ICollection <Data> GetData()
{
List <Data> list = new List <Data>();
Queue <RawlerBase> queueRawler = new Queue <RawlerBase>();
queueRawler.Enqueue(Rawler);
while (queueRawler.Count > 0)
{
RawlerBase tmp = queueRawler.Dequeue();
if (tmp is Data)
{
list.Add(tmp as Data);
}
foreach (var item in tmp.Children)
{
queueRawler.Enqueue(item);
}
}
return(list);
}
/// <summary>
/// クローンを作る。
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
var clone = new Parallel();
RawlerLib.ObjectLib.FildCopy(this, clone);
clone.SetParent(parent);
this.CloneEvent(clone);
clone.children.Clear();
clone.DocumentsClear();
foreach (var item in documents)
{
var child = (Document)item.Clone(clone);
clone.documents.Add(child);
}
foreach (var item in this.Children)
{
var child = item.Clone(clone);
clone.AddChildren(child);
}
return clone;
}
public string Rawler2XAML(RawlerBase rawler)
{
StringBuilder xaml = new StringBuilder(System.Xaml.XamlServices.Save(rawler));
xaml = xaml.Replace("\"{x:Null}\"", "Null").Replace(" Enable=\"True\"", "").Replace(" Comment=\"\"", "");
System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(@"\w*=Null");
List <string> list = new List <string>();
foreach (System.Text.RegularExpressions.Match item in regex.Matches(xaml.ToString()))
{
list.Add(item.Value);
}
foreach (var item in list.Distinct())
{
xaml = xaml.Replace(" " + item, string.Empty);
}
return(xaml.ToString());
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
var login = this.GetAncestorRawler().OfType <TwitterLogin>().First();
bool flag = true;
Dictionary <string, object> dic = new Dictionary <string, object>();
string track;
if (TrackTree != null)
{
TrackTree.SetParent(this);
track = RawlerBase.GetText(GetText(), TrackTree, this);
}
else
{
track = Track;
}
dic.Add("track", track);
var stream = login.Token.Streaming.StartObservableStream(StreamingType.Filter, new StreamingParameters(dic)).Publish();
stream.OfType <StatusMessage>()
.Subscribe(x => {
Document d = new Document()
{
TextValue = Codeplex.Data.DynamicJson.Serialize(x.Status)
};
d.SetParent(this);
foreach (var item in this.Children)
{
d.AddChildren(item);
}
d.Run();
});
stream.OfType <WarningMessage>().Subscribe(x => ReportManage.ErrReport(this, x.Message));
//// stream.OfType<EventMessage>()
// .Subscribe(x => Console.WriteLine("{0} by @{1}", x.Event, x.Source.ScreenName));
disposable = stream.Connect();
}
public void Write(RawlerBase rawler, string text, bool viewTotal)
{
if (DoRun)
{
if (viewParent)
{
text = this.Parent.ToObjectString() + " " + text;
}
var t = sw.ElapsedMilliseconds + "\t" + text;
total += sw.ElapsedMilliseconds;
if (viewTotal)
{
ReportManage.Report(rawler, t,true,true);
ReportManage.Report(rawler,"Total:"+ total, true, true);
}
else
{
ReportManage.Report(rawler, t,true,true);
}
sw.Restart();
}
}
public void SetUp(RawlerBase rawler)
{
if (string.IsNullOrEmpty(this.ConsumerKey) || string.IsNullOrEmpty(this.ConsumerSecret))
{
if( KeyValueStore.ContainsKey(rawler, "ConsumerKey", "ConsumerSecret"))
{
this.ConsumerKey = KeyValueStore.GetValueByKey(rawler, "ConsumerKey");
this.ConsumerSecret = KeyValueStore.GetValueByKey(rawler, "ConsumerSecret");
ReportManage.Report(rawler, "KeyValueStoreからのAPI Keyを使います", true, true);
}
else if (string.IsNullOrEmpty(SetTwitterApiKeys.consumerKey) || string.IsNullOrEmpty(SetTwitterApiKeys.consumerSecret))
{
this.ConsumerKey = "gHVupgapEXlTZdu7rf3oOg";
this.ConsumerSecret = "YOicLtW8utx3NJyy88wtzq8QN3ilXeQoEGCPIJNzo";
ReportManage.Report(rawler, "RawlerのAPI Keyを使います", true, true);
}
else
{
this.ConsumerKey = SetTwitterApiKeys.consumerKey;
this.ConsumerSecret = SetTwitterApiKeys.consumerSecret;
}
}
}
/// <summary>
/// このクラスでの実行すること。
/// </summary>
/// <param name="runChildren"></param>
public override void Run(bool runChildren)
{
if (string.IsNullOrEmpty(Format))
{
ReportManage.ErrReport(this, "Formatが空です");
}
else if (Args == null)
{
ReportManage.ErrReport(this, "Argsが空です");
}
else
{
List <string> list = new List <string>();
foreach (var item in Args)
{
item.SetParent(this.Parent);
item.SetParent();
list.Add(RawlerBase.GetText(GetText(), item, this.Parent));
}
SetText(string.Format(Format, list.ToArray()));
}
base.Run(runChildren);
}
public void SetUp(RawlerBase rawler)
{
if (string.IsNullOrEmpty(this.ConsumerKey) || string.IsNullOrEmpty(this.ConsumerSecret))
{
if (KeyValueStore.ContainsKey(rawler, "ConsumerKey", "ConsumerSecret"))
{
this.ConsumerKey = KeyValueStore.GetValueByKey(rawler, "ConsumerKey");
this.ConsumerSecret = KeyValueStore.GetValueByKey(rawler, "ConsumerSecret");
ReportManage.Report(rawler, "KeyValueStoreからのAPI Keyを使います", true, true);
}
else if (string.IsNullOrEmpty(SetTwitterApiKeys.consumerKey) || string.IsNullOrEmpty(SetTwitterApiKeys.consumerSecret))
{
this.ConsumerKey = "gHVupgapEXlTZdu7rf3oOg";
this.ConsumerSecret = "YOicLtW8utx3NJyy88wtzq8QN3ilXeQoEGCPIJNzo";
ReportManage.Report(rawler, "RawlerのAPI Keyを使います", true, true);
}
else
{
this.ConsumerKey = SetTwitterApiKeys.consumerKey;
this.ConsumerSecret = SetTwitterApiKeys.consumerSecret;
}
}
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<GetWordsInSouce>(parent);
}
/// <summary>
/// クローンを作る
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<LoginClient>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<DataGrouping>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <Keyphrase>(parent));
}
/// <summary>
/// クローンを作る
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<ImageLinks>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<GetFileExtension>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <LevenshteinDistance>(parent));
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<UrlEncode>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <TwitterStreamingFilter>(parent));
}
public static RawlerBase GetPageUrl(this RawlerBase rawler)
{
return(rawler.Add(new GetPageUrl()));
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <StringFormat>(parent));
}
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<NextDataRow>(parent);
}
public RawlerBase CreateRawler(HtmlNode node)
{
var baseUrl = BaseUrl;
RawlerBase rawler = null;
bool flag次のノードを調べる = true;
if (targetTag.Contains(node.Name))
{
Tags tags = new Tags()
{
Tag = node.Name
};
if (node.Attributes.Where(n => n.Name == "class").Any())
{
tags.ClassName = node.Attributes.Where(n => n.Name == "class").First().Value;
}
if (node.Attributes.Where(n => n.Name == "id").Any())
{
tags.IdName = node.Attributes.Where(n => n.Name == "id").First().Value;
}
if (node.ChildNodes.Count() == 1 && node.ChildNodes.Where(n => n.Name == "#text").Any())
{
tags.AddChildren(new DataWrite()
{
Attribute = tags.ClassName
});
flag次のノードを調べる = false;
}
if (node.Attributes.Where(n => n.Name == "style" && n.Value.Contains("background")).Any())
{
tags.TagVisbleType = TagVisbleType.Outer;
rawler = tags.Add(new ImageLinks()
{
ImageType = ImageType.BackgroundImage
}).DataWrite(node.GetClassName() + "_Image", DataAttributeType.Image).GetRoot();
}
rawler = tags;
}
else if (node.Name == "a")
{
var resultUrlTFIDF = urlTfidf.GetResult(node.OuterHtml.ToHtml(baseUrl).GetLink().Select(n => n.Url));
var url = node.OuterHtml.ToHtml(baseUrl).GetLink().FirstDefault <RawlerLib.Web.Link, string>(n => n.Url, null);
if (url != null)
{
//IDF が0以下の時、すべてのドキュメントで存在する。
if (urlTfidf.IDFDic.GetValueOrDefault(url) != null && urlTfidf.IDFDic.GetValueOrDefault(url).Value <= 0)
{
rawler = null;
flag次のノードを調べる = false;
}
else
{
if (resultUrlTFIDF.GetTakeTopValue(n => n.TFIDF).Where(n => n.Word == url).Any())
{
rawler = new Links()
{
VisbleType = LinkVisbleType.Tag
}.AddRange(
new Links()
{
VisbleType = LinkVisbleType.Url
}.DataWrite(node.GetClassName() + "_MainLink", DataAttributeType.Url).GetRoot());
if (node.ChildNodes.Count == 1 && node.ChildNodes.First().Name == "#text")
{
rawler.Add(new Links()
{
VisbleType = LinkVisbleType.Label
}.DataWrite(node.GetClassName() + "_MainLabel").GetRoot());
}
}
else
{
rawler = new Links()
{
VisbleType = LinkVisbleType.Tag
}.AddRange(
new Links()
{
VisbleType = LinkVisbleType.Url, Enable = enableGetSubUrlLink
}.DataWrite(node.GetClassName() + "_SubLink").GetRoot(),
new Links()
{
VisbleType = LinkVisbleType.Label
}.DataWrite(node.GetClassName() + "_SubLabel").GetRoot()
);
}
}
}
else
{
//URLがないAタグの場合。
Tags tags = new Tags()
{
Tag = node.Name
};
if (node.Attributes.Where(n => n.Name == "class").Any())
{
tags.ClassName = node.Attributes.Where(n => n.Name == "class").First().Value;
}
if (node.Attributes.Where(n => n.Name == "id").Any())
{
tags.IdName = node.Attributes.Where(n => n.Name == "id").First().Value;
}
rawler = tags;
}
if (node.ChildNodes.Count == 1 && node.ChildNodes.Where(n => n.Name == "#text").Any())
{
flag次のノードを調べる = false;
}
}
else if (node.Name == "img")
{
var resultImgeTFIDF = imageTfidf.GetResult(node.OuterHtml.ToHtml(baseUrl).GetImageLink().Select(n => n.Url));
var url = node.OuterHtml.ToHtml(baseUrl).GetImageLink().FirstDefault(n => n.Url, null);
if (url != null)
{
if (imageTfidf.IDFDic.Count > 0 && imageTfidf.IDFDic.GetValueOrDefault(url).Value <= 0)
{
rawler = null;
flag次のノードを調べる = false;
}
else
{
if (resultImgeTFIDF.GetTakeTopValue(n => n.TFIDF).Where(n => n.Word == url).Any())
{
rawler = new ImageLinks().DataWrite(node.GetClassName() + "_Image", DataAttributeType.Image).GetRoot();
}
else
{
if (iconImageColumn)
{
rawler = new DataWrite()
{
AttributeTree = new ImageLinks()
{
VisbleType = LinkVisbleType.Label
}, Value = "1"
};
}
else
{
rawler = new ImageLinks().DataWrite(node.GetClassName() + "_Icon", DataAttributeType.Image).GetRoot();
}
}
}
}
}
///背景画像に反応させる。
else if (node.Attributes.Where(n => n.Name == "style" && n.Value.Contains("background")).Any())
{
rawler = new ImageLinks()
{
ImageType = ImageType.BackgroundImage
}.DataWrite(node.GetClassName() + "_Image", DataAttributeType.Image).GetRoot();
}
else if (node.Name == "span")
{
Tags tags = new Tags()
{
Tag = node.Name
};
if (node.Attributes.Where(n => n.Name == "class").Any())
{
tags.ClassName = node.Attributes.Where(n => n.Name == "class").First().Value;
}
if (node.ChildNodes.Count() == 1 && node.ChildNodes.Where(n => n.Name == "#text").Any())
{
tags.AddChildren(new DataWrite()
{
Attribute = tags.ClassName
});
flag次のノードを調べる = false;
}
rawler = tags;
}
else if (node.Name == "#comment")
{
flag次のノードを調べる = false;
}
else
{
var t = node.OuterHtml.Replace("\n", "").Trim();
if (t.Length > 0)
{
rawler = new TagClear().Trim().Add(new DataWrite()
{
Attribute = node.GetClassName() + "_" + node.Name
}).GetRoot();
if (node.ChildNodes.Count == 1 && node.ChildNodes.Where(n => n.Name == "#text").Any())
{
flag次のノードを調べる = false;
}
}
}
if (rawler != null && node.ChildNodes.Count == 1 && node.ChildNodes.Where(n => n.Name == "span").Any())
{
rawler.AddChildren(new DataWrite()
{
Attribute = node.GetClassName()
});
}
foreach (var item in node.ChildNodes)
{
if (flag次のノードを調べる)
{
var r = CreateRawler(item);
if (r != null && rawler != null)
{
rawler.AddChildren(r);
}
else
{
if (r != null && rawler == null)
{
rawler = r;
}
}
}
}
return(rawler);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<MecabPreprocessing>(parent);
}
/// <summary>
/// クローンを作る。
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<GetPageHtml>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<AddAllInputParameter>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<ChangeText>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<IteratorSourceClear>(parent);
}
/// <summary>
/// クローンを作る。
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <Contains>(parent));
}
public static RawlerBase ImageLinks(this RawlerBase rawler)
{
return(rawler.Add(new ImageLinks()));
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<GetCurrentIterator>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<RawlerClass1>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<XPathSelectNodes>(parent);
}
/// <summary>
/// クローンを作る。
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<RemoveHtmlComment>(parent);
}
public static RawlerBase Trim(this RawlerBase rawler)
{
return(rawler.Add(new Trim()));
}
public string Rawler2XAML(RawlerBase rawler)
{
StringBuilder xaml = new StringBuilder(System.Xaml.XamlServices.Save(rawler));
xaml = xaml.Replace("\"{x:Null}\"", "Null").Replace(" Enable=\"True\"","").Replace(" Comment=\"\"","");
System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(@"\w*=Null");
List<string> list = new List<string>();
foreach (System.Text.RegularExpressions.Match item in regex.Matches(xaml.ToString()))
{
list.Add(item.Value);
}
foreach (var item in list.Distinct())
{
xaml = xaml.Replace(" " + item, string.Empty);
}
return xaml.ToString();
}
private void button1_Click(object sender, RoutedEventArgs e)
{
if (isBusy)
{
MessageBox.Show("実行中です");
return;
}
tokenSource = new CancellationTokenSource();
object obj = null;
string xaml = textEditor.Text;
var insertParameter = new Tool.InsertParameterWindow();
if (insertParameter.Analyze(textEditor.Text) == true)
{
xaml = insertParameter.Xaml;
}
else
{
return;
}
try
{
obj = System.Xaml.XamlServices.Parse(xaml);
}
catch (Exception ex)
{
ReportManage.ErrReport(new RawlerBase(), "XAMLの形式がおかしいです" + ex.Message);
}
if (obj == null)
{
return;
}
if ((obj is Rawler.Tool.RawlerBase)==false)
{
ReportManage.ErrReport(new RawlerBase(), "キャストできませんでした。XAMLの形式がおかしいです");
}
try
{
rawler = (obj as Rawler.Tool.RawlerBase);
ReportManage.RowCount = 0;
rawler.SetParent();
startDate = DateTime.Now;
foreach (var item in rawler.GetConectAllRawler())
{
item.BeginRunEvent += (o, arg) =>
{
tokenSource.Token.ThrowIfCancellationRequested();
while (pause)
{
System.Threading.Thread.Sleep(1000);
}
};
}
isBusy = true;
Task.Factory.StartNew(() => rawler.Run(), tokenSource.Token).ContinueWith((t) => { StopWatch(); isBusy = false; });
}
catch (OperationCanceledException oce)
{
ReportManage.ErrReport(new RawlerBase(), "キャンセルされました");
MessageBox.Show("キャンセルされました");
}
catch (Exception ex)
{
ReportManage.ErrReport(new RawlerBase(), ex.Message);
}
}
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<PageOnce>(parent);
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return(base.Clone <CaseDateTime>(parent));
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<DataWriteTweet>(parent);
}
public static RawlerBase NextPage(this RawlerBase rawler)
{
return(rawler.Add(new NextPage()));
}
/// <summary>
/// Clone
/// </summary>
/// <param name="parent"></param>
/// <returns></returns>
public override RawlerBase Clone(RawlerBase parent)
{
return base.Clone<LevenshteinDistance>(parent);
}
