public virtual void Insert_InsertNewAndUpdateOld()
{
using (var conn = CreateDbConnection())
{
try
{
conn.Execute("use test; DROP TABLE multiprimarykey;");
}
catch
{
}
var pipeline = CreatePipeline();
var metadata = new ModelDefinition <MultiPrimaryKey>();
var resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems["AAA"] = new List <MultiPrimaryKey>
{
new MultiPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
new MultiPrimaryKey {
Sku = "111", Category = "3C", Name = "Product 2"
},
new MultiPrimaryKey {
Sku = "112", Category = null, Name = "Product 3"
},
new MultiPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
};
pipeline.Process(new ResultItems[] { resultItems }, null);
var insertNewAndUpdateOldPipeline = CreatePipeline(PipelineMode.InsertNewAndUpdateOld);
resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems["AAA"] = new List <MultiPrimaryKey>
{
new MultiPrimaryKey {
Sku = "110", Name = "Product 1", Category = "4C"
}
};
insertNewAndUpdateOldPipeline.Process(new ResultItems[] { resultItems }, null);
var list = conn.Query("use test; select * from multiprimarykey").Select(r => r as IDictionary <string, dynamic>).ToList();
Assert.Equal(3, list.Count);
Assert.Equal("4C", list[0]["category"]);
try
{
conn.Execute("use test; DROP TABLE multiprimarykey;");
}
catch { }
}
}
public void Process(ResultItems resultItems, ISpider spider)
{
if (_cached.ContainsKey(spider))
{
_cached[spider].Add(resultItems);
}
else
{
while (!_cached.TryAdd(spider, new List <ResultItems>()
{
resultItems
}))
{
}
}
if (_cached[spider].Count >= CachedSize)
{
List <ResultItems> result = new List <ResultItems>();
result.AddRange(_cached[spider]);
_cached.Clear();
// 做成异步
Process(result.ToList(), spider);
}
}
public void Process(ResultItems resultItems, ITask task)
{
foreach (DictionaryEntry entry in resultItems.GetAll())
{
System.Console.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
public virtual void Insert_InsertNewAndUpdateOld()
{
using (var conn = CreateDbConnection())
{
try
{
conn.Execute($"use test; DROP TABLE multiprimarykey;");
}
catch { }
var spider = new DefaultSpider();
var pipeline = CreatePipeline();
var metadata = new ModelDefinition <MultiPrimaryKey>();
var resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems.AddOrUpdateResultItem(metadata.Identity, new Tuple <IModel, IList <dynamic> >(metadata, new dynamic[]
{
new MultiPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
new MultiPrimaryKey {
Sku = "111", Category = "3C", Name = "Product 2"
},
new MultiPrimaryKey {
Sku = "112", Category = null, Name = "Product 3"
},
new MultiPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
}));
pipeline.Process(new ResultItems[] { resultItems }, spider.Logger, spider);
var insertNewAndUpdateOldPipeline = CreatePipeline(PipelineMode.InsertNewAndUpdateOld);
resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems.AddOrUpdateResultItem(metadata.Identity, new Tuple <IModel, IList <dynamic> >(metadata, new dynamic[]
{
new AutoIncrementPrimaryKey {
Sku = "110", Name = "Product 1", Category = "4C"
}
}));
insertNewAndUpdateOldPipeline.Process(new ResultItems[] { resultItems }, spider.Logger, spider);
var list = conn.Query($"use test; select * from multiprimarykey").Select(r => r as IDictionary <string, dynamic>).ToList();
Assert.Equal(3, list.Count);
Assert.Equal("4C", list[0]["category"]);
try
{
conn.Execute($"use test; DROP TABLE multiprimarykey;");
}
catch { }
}
}
public static void Run()
{
//single download
var spider = Spider.Create(new BaiduBaikePageProcessor()).Thread(2);
String urlTemplate = "http://baike.baidu.com/search/word?word=%s&pic=1&sug=1&enc=utf8";
ResultItems resultItems = spider.Get <ResultItems>(string.Format(urlTemplate, "水力发电"));
Console.WriteLine(resultItems);
//multidownload
var list = new List <String>
{
String.Format(urlTemplate, "风力发电"),
String.Format(urlTemplate, "太阳能"),
String.Format(urlTemplate, "地热发电"),
String.Format(urlTemplate, "地热发电")
};
var resultItemses = spider.GetAll <ResultItems>(list);
foreach (var item in resultItemses)
{
Console.WriteLine(item.GetAll().ToKeyValuePairString());
}
spider.Close();
}
public void ScrollPageLast()
{
if (ResultItems is ImageListGrid)
{
ResultItems.PageLast();
}
}
public void Process(ResultItems resultItems, ISpider spider)
{
foreach (var entry in resultItems.Results)
{
System.Console.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
public override void Process(ResultItems resultItems)
{
var result = resultItems.Results["VideoResult"];
if (result != null)
{
foreach (var m in result as List <MeituanModel> )
{
var query = Connection.Query <MeituanModel>("select * from meituan where name=@name",
new { name = m.name });
if (query.Count() > 0)
{
var old = query.First();
m.id = old.id;
m.creation_time = old.creation_time;
m.seen = old.seen.HasValue && old.seen == 1 ? 1 : 0;
Connection.Update(m);
}
else
{
Connection.Insert(m);
}
}
}
}
public override void Process(ResultItems resultItems)
{
foreach (var entry in resultItems.Results)
{
Console.WriteLine($"{entry.Key}:{entry.Value}");
}
}
public void UpdateLikeState(int illustid = -1, bool is_user = false)
{
if (ResultExpander.IsExpanded)
{
ResultItems.UpdateLikeState(illustid, is_user);
}
}
public virtual void Process(ResultItems resultItems, ITask task)
{
string filePath = BasePath + PathSeperator + task.Identify + PathSeperator;
try
{
FileInfo file = GetFile(filePath + Encrypt.Md5Encrypt(resultItems.Request.Url) + ".html");
using (StreamWriter printWriter = new StreamWriter(file.OpenWrite(), Encoding.UTF8))
{
printWriter.WriteLine("url:\t" + resultItems.Request.Url);
foreach (DictionaryEntry entry in resultItems.GetAll())
{
var value = entry.Value as IList;
if (value != null)
{
IList list = value;
printWriter.WriteLine(entry.Key + ":");
foreach (var o in list)
{
printWriter.WriteLine(o);
}
}
else
{
printWriter.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
}
}
catch (Exception e)
{
_logger.Warn("Write file error.", e);
throw;
}
}
public void Process(ResultItems resultItems, ITask task)
{
foreach (DictionaryEntry entry in resultItems.GetAll())
{
System.Console.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
private void ActionRefreshResult_Click(object sender, RoutedEventArgs e)
{
if (sender is MenuItem)
{
var m = sender as MenuItem;
var host = (m.Parent as ContextMenu).PlacementTarget;
if (m.Uid.Equals("ActionRefresh", StringComparison.CurrentCultureIgnoreCase))
{
if (host == ResultExpander || host == ResultItems)
{
UpdateDetail(Contents);
}
}
else if (m.Uid.Equals("ActionRefreshThumb", StringComparison.CurrentCultureIgnoreCase))
{
if (host == ResultExpander || host == ResultItems)
{
ResultItems.UpdateTilesImage(Keyboard.Modifiers == ModifierKeys.Alt);
}
}
}
else if (sender == SearchRefreshThumb)
{
ResultItems.UpdateTilesImage();
}
}
private void ActionFollowAuthor_Click(object sender, RoutedEventArgs e)
{
string uid = (sender as dynamic).Uid;
if (uid.Equals("ActionLikeUser", StringComparison.CurrentCultureIgnoreCase) ||
uid.Equals("ActionLikeUserPrivate", StringComparison.CurrentCultureIgnoreCase) ||
uid.Equals("ActionUnLikeUser", StringComparison.CurrentCultureIgnoreCase))
{
IList <PixivItem> items = new List <PixivItem>();
var host = ((sender as MenuItem).Parent as ContextMenu).PlacementTarget;
if (host == ResultItems || host == ResultExpander)
{
items = ResultItems.GetSelected();
}
try
{
if (uid.Equals("ActionLikeUser", StringComparison.CurrentCultureIgnoreCase))
{
items.LikeUser();
}
else if (uid.Equals("ActionLikeUserPrivate", StringComparison.CurrentCultureIgnoreCase))
{
items.LikeUser(false);
}
else if (uid.Equals("ActionUnLikeUser", StringComparison.CurrentCultureIgnoreCase))
{
items.UnLikeUser();
}
}
catch (Exception ex) { ex.ERROR(); }
}
}
public override void Process(ResultItems resultItems)
{
try
{
string filePath = $"{BasePath}{Common.Environment.PathSeperator}{Spider.Identity}{Common.Environment.PathSeperator}{Guid.NewGuid().ToString("N")}.dsd";
FileInfo file = PrepareFile(filePath);
using (StreamWriter printWriter = new StreamWriter(file.OpenWrite(), Encoding.UTF8))
{
printWriter.WriteLine("url:\t" + resultItems.Request.Url);
foreach (var entry in resultItems.Results)
{
var value = entry.Value as IList;
if (value != null)
{
IList list = value;
printWriter.WriteLine(entry.Key + ":");
foreach (var o in list)
{
printWriter.WriteLine(o);
}
}
else
{
printWriter.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
}
}
catch (Exception e)
{
Spider.Log("Write file error.", LogLevel.Error, e);
throw;
}
}
public void Process(ResultItems resultItems, ISpider spider)
{
foreach (var entry in resultItems.Results)
{
System.Console.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
public override void Process(ResultItems resultItems)
{
try
{
string filePath = $"{BasePath}{PathSeperator}{Spider.Identity}{PathSeperator}{Encrypt.Md5Encrypt(resultItems.Request.Url.ToString())}.fd";
FileInfo file = PrepareFile(filePath);
using (StreamWriter printWriter = new StreamWriter(file.OpenWrite(), Encoding.UTF8))
{
printWriter.WriteLine("url:\t" + resultItems.Request.Url);
foreach (var entry in resultItems.Results)
{
var value = entry.Value as IList;
if (value != null)
{
IList list = value;
printWriter.WriteLine(entry.Key + ":");
foreach (var o in list)
{
printWriter.WriteLine(o);
}
}
else
{
printWriter.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
}
}
catch (Exception e)
{
Spider.Logger.Warn("Write file error.", e);
throw;
}
}
public void Process(ResultItems resultItems, ISpider spider)
{
lock (this)
{
_collector.Add(resultItems);
}
}
public virtual void Update_AutoIncrementPrimaryKey()
{
using (var conn = CreateDbConnection())
{
try
{
conn.Execute($"use test; DROP TABLE autoincrementprimarykey;");
}
catch { }
ISpider spider = new DefaultSpider("test");
// 1. Create pipeline
var pipeline = CreatePipeline();
// 2. Create ModelDefine
var metadata = new ModelDefinition <AutoIncrementPrimaryKey>();
// 3. Create data
var resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems.AddOrUpdateResultItem(metadata.Identity, new Tuple <IModel, IList <dynamic> >(metadata, new dynamic[]
{
new AutoIncrementPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
new AutoIncrementPrimaryKey {
Sku = "111", Category = "3C", Name = "Product 2"
}
}));
var processArgument = new ResultItems[] { resultItems };
// 4. Execute pipline
pipeline.Process(processArgument, spider.Logger, spider);
var updateModePipeline = CreatePipeline(PipelineMode.Update);
resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems.AddOrUpdateResultItem(metadata.Identity, new Tuple <IModel, IList <dynamic> >(metadata, new dynamic[]
{
new AutoIncrementPrimaryKey {
Id = 1, Category = "4C"
}
}));
updateModePipeline.Process(new ResultItems[] { resultItems }, spider.Logger, spider);
var list = conn.Query <AutoIncrementPrimaryKey>($"use test; select * from autoincrementprimarykey").ToList();
Assert.Equal(2, list.Count);
Assert.Equal("110", list[0].Sku);
Assert.Equal("4C", list[0].Category);
try
{
conn.Execute($"use test; DROP TABLE autoincrementprimarykey;");
}
catch { }
}
}
private void Before()
{
_resultItems = new ResultItems();
_resultItems.AddOrUpdateResultItem("content", "爬虫工具");
Request request = new Request("http://www.baidu.com", null);
_resultItems.Request = request;
}
public void Before()
{
_resultItems = new ResultItems();
_resultItems.Put("content", "webmagic 爬虫工具");
Request request = new Request("http://www.baidu.com", null);
_resultItems.Request = request;
_task = new TestTask();
}
private void Before()
{
_resultItems = new ResultItems();
_resultItems["content"] = "爬虫工具";
Request request = new Request("http://www.baidu.com", null);
_resultItems.Request = request;
}
public void Process(ResultItems resultItems, ITask task)
{
Console.WriteLine("get page: " + resultItems.GetRequest().GetUrl());
foreach (var item in resultItems.GetAll())
{
Console.WriteLine(item.Key + ":\t" + item.Value);
}
}
private void ResultExpander_Collapsed(object sender, RoutedEventArgs e)
{
ResultItems.Ready();
if (ResultNextPage is Button)
{
ResultNextPage.Hide();
}
}
public void Process(ResultItems resultItems, ITask task)
{
var resultItemsAll = resultItems.GetAll();
foreach (var item in resultItemsAll)
{
if (item.Value is IMultiPageModel multiPageModel)
{
pageMap.Put(multiPageModel.GetPageKey(),
multiPageModel.GetPage(), false);
lock (this)
{
pageMap.Put(multiPageModel.GetPageKey(),
multiPageModel.GetPage(), true);
var otherPages = multiPageModel.GetOtherPages();
if (otherPages?.Count > 0)
{
foreach (var otherPage in otherPages)
{
if (pageMap.Get(multiPageModel.GetPageKey(),
otherPage) == false)
{
pageMap.Put(multiPageModel.GetPageKey(), otherPage, false);
}
}
}
var booleanMap = pageMap.Get(multiPageModel.GetPageKey());
objectMap.Put(multiPageModel.GetPageKey(),
multiPageModel.GetPage(), multiPageModel);
if (booleanMap == null)
{
return;
}
foreach (var stringBooleanEntry in booleanMap)
{
if (!stringBooleanEntry.Value)
{
booleanMap.Remove(stringBooleanEntry.Key);
return;
}
}
var range = objectMap.Get(multiPageModel.GetPageKey());
var entryList = range.ToList();
if (entryList.Count > 0)
{
//entryList.Sort();
//pai xu sort
var value = entryList[0].Value;
for (int i = 1; i < entryList.Count; i++)
{
value = value.Combine(entryList[i].Value);
}
resultItemsAll[item.Key] = value;
}
}
}
}
}
public void Before()
{
_resultItems = new ResultItems();
_resultItems.Put("content", "webmagic 爬虫工具");
Request request = new Request("http://www.baidu.com", null);
_resultItems.Request = request;
_task = new TestTask();
}
public override void Process(ResultItems resultItems)
{
var result = resultItems.Results["VideoResult"];
if (result != null)
{
Connection.Insert(result as IEnumerable <BoxOfficeModel>);
}
}
public virtual void Update_MutliPrimaryKey()
{
using (var conn = CreateDbConnection())
{
try
{
conn.Execute("use test; DROP TABLE multiprimarykey;");
}
catch { }
ISpider spider = new DefaultSpider("test");
// 1. Create pipeline
var pipeline = CreatePipeline();
// 2. Create ModelDefine
var metadata = new ModelDefinition <MultiPrimaryKey>();
// 3. Create data
var resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems["AAA"] = new List <MultiPrimaryKey>
{
new MultiPrimaryKey {
Sku = "110", Category = "3C", Name = "Product 1"
},
new MultiPrimaryKey {
Sku = "111", Category = "3C", Name = "Product 2"
}
};
var processArgument = new ResultItems[] { resultItems };
// 4. Execute pipline
pipeline.Process(processArgument, spider);
var updateModePipeline = CreatePipeline(PipelineMode.Update);
resultItems = new ResultItems();
resultItems.Request = new Request();
resultItems["AAA"] = new List <MultiPrimaryKey>
{
new MultiPrimaryKey {
Sku = "111", Category = "4C", Name = "Product 2"
}
};
updateModePipeline.Process(new ResultItems[] { resultItems }, spider);
var list = conn.Query <MultiPrimaryKey>("use test; select * from multiprimarykey").ToList();
Assert.Equal(2, list.Count);
Assert.Equal("4C", list[1].Category);
try
{
conn.Execute("use test; DROP TABLE multiprimarykey;");
}
catch { }
}
}
public override void Process(ResultItems resultItems)
{
foreach (YoukuVideo entry in resultItems.Results["VideoResult"])
{
File.AppendAllLines("test.txt", new[] { entry.Name });
}
// 可以自由实现插入数据库或保存到文件
}
public void Before()
{
_resultItems = new ResultItems();
_resultItems.AddOrUpdateResultItem("content", "爬虫工具");
Request request = new Request("http://www.baidu.com", 1, null);
_resultItems.Request = request;
_spider = new DefaultSpider();
}
public bool CompletedSuccessfully()
{
var getAllUncompleted = ResultItems.FirstOrDefault(x => x.CompletedSuccessfully == false);
if (getAllUncompleted == null)
{
return(true);
}
return(false);
}
//[MethodImplAttribute(MethodImplOptions.Synchronized)]
public virtual void Process(ResultItems resultItems, ITask task)
{
dynamic o = resultItems.Get(_type.FullName);
if (o != null)
{
//check
_classPipeline.Process(o, task);
}
}
public override void Process(ResultItems resultItems)
{
foreach (YoukuVideo entry in resultItems.Results["VideoResult"])
{
count++;
Console.WriteLine($"[YoukuVideo {count}] {entry.Name}");
}
// 可以自由实现插入数据库或保存到文件
}
public void Dispose()
{
try
{
ResultItems.Clear(batch: false, force: true);
Contents = null;
}
catch (Exception ex) { ex.ERROR("DisposeSearchResult"); }
finally { }
}
public void Process(ResultItems resultItems, ISpider spider)
{
foreach (var entry in resultItems.Results)
{
#if NET_CORE
Log.WriteLine(entry.Key + ":\t" + entry.Value);
#else
System.Console.WriteLine(entry.Key + ":\t" + entry.Value);
#endif
}
}
internal ResultItems InternalGetWhitespaces
(
ITokenClass leftTokenClass,
int leadingLineBreaks,
int indentLevel,
IEnumerable<IItem> whiteSpaces,
ITokenClass rightTokenClass
)
{
var result = new ResultItems();
var emptyLines = 0;
var isBeginOfLine = leadingLineBreaks > 0;
foreach(var token in whiteSpaces)
{
if(isBeginOfLine && !Lexer.IsLineEnd(token))
{
result.AddLineBreak(leadingLineBreaks - emptyLines);
result.AddSpaces(indentLevel * Configuration.IndentCount);
emptyLines = leadingLineBreaks;
leadingLineBreaks = 0;
isBeginOfLine = false;
}
if(Lexer.IsWhiteSpace(token)
|| (Lexer.IsLineEnd(token) && !IsRelevantLineBreak(emptyLines, rightTokenClass)))
result.AddHidden(token);
else
{
result.Add(token);
if(Lexer.IsLineEnd(token))
emptyLines++;
else
emptyLines = Lexer.IsLineComment(token) ? 1 : 0;
isBeginOfLine = !Lexer.IsComment(token);
}
}
if(isBeginOfLine)
{
result.AddLineBreak(leadingLineBreaks - emptyLines);
result.AddSpaces(indentLevel * Configuration.IndentCount);
leadingLineBreaks = 0;
}
if (result.IsEmpty && SeparatorType.Get(leftTokenClass, rightTokenClass) == SeparatorType.Close)
result.AddSpaces(1);
Tracer.Assert(leadingLineBreaks == 0);
return result;
}
public void Process(ResultItems resultItems, ISpider spider)
{
string path = $"{BasePath}{PathSeperator}{ spider.Identity}{PathSeperator}{Encrypt.Md5Encrypt(resultItems.Request.Url.ToString())}.json";
try
{
FileInfo file = PrepareFile(path);
using (StreamWriter printWriter = new StreamWriter(file.OpenWrite(), Encoding.UTF8))
{
printWriter.WriteLine(JsonConvert.SerializeObject(resultItems.Results));
}
}
catch (IOException e)
{
spider.Logger.Warn("write file error", e);
throw;
}
}
public void Process(ResultItems resultItems, ISpider spider)
{
StringBuilder builer = new StringBuilder(BasePath);
string filePath = $"{BasePath}{PathSeperator}{spider.Identity}{PathSeperator}{Encrypt.Md5Encrypt(resultItems.Request.Url.ToString())}.fd";
try
{
FileInfo file = PrepareFile(filePath);
using (StreamWriter printWriter = new StreamWriter(file.OpenWrite(), Encoding.UTF8))
{
printWriter.WriteLine("url:\t" + resultItems.Request.Url);
foreach (var entry in resultItems.Results)
{
var value = entry.Value as IList;
if (value != null)
{
IList list = value;
printWriter.WriteLine(entry.Key + ":");
foreach (var o in list)
{
printWriter.WriteLine(o);
}
}
else
{
printWriter.WriteLine(entry.Key + ":\t" + entry.Value);
}
}
}
}
catch (Exception e)
{
spider.Logger.Warn("Write file error.", e);
throw;
}
}
public void Process(ResultItems resultItems, ITask task)
{
_collector.Add(resultItems);
}
