/// <summary>
/// 写入文件
/// </summary>
public void Foreach()
{
TygModel.Entities tity = new TygModel.Entities();
var items = tity.文章表.OrderBy(p => p.ID);
int count = items.Count();
tity.Dispose();
for (int k = 0; k < count / 10000; k++)
{
using (TygModel.Entities ent = new TygModel.Entities())
{
var records = ent.文章表.OrderBy(p => p.ID).Skip(k * 10000).Take(10000).ToList();
foreach (var sp in records)
{
if (list.Count >= 10000)
{
CreateMaps(list.ToArray(), SiteIndex++);
list.Clear();
}
//添加到路径
list.Add(url + sp.GetHTMLFilePath());
currentIndex++;
System.Console.WriteLine(string.Format("进度{0}/{1},{2}", currentIndex, count, sp.章节名));
}
records.Clear();
records = null;
}
}
//全部写入
CreateMaps(list.ToArray(), SiteIndex++);
}
protected void Page_Load(object sender, EventArgs e)
{
List <书名表> books = new List <书名表>();
System.Xml.Serialization.XmlSerializer xmlSer = new System.Xml.Serialization.XmlSerializer(books.GetType());
/// <summary>
/// 数据库访问对像
/// </summary>
using (TygModel.Entities Tygdb = new TygModel.Entities())
{
books = Tygdb.书名表.ToList();
System.IO.StringWriter sw = new System.IO.StringWriter();
xmlSer.Serialize(sw, books);
//System.IO.File.WriteAllText(Server.MapPath("./books.html"),sw.ToString());
Response.Write(sw.ToString());
Tygdb.Connection.Close();
}
}
public void Update()
{
System.Threading.Tasks.Task task = System.Threading.Tasks.Task.Factory.StartNew(new Action(delegate
{
using (TygModel.Entities tygdb = new TygModel.Entities())
{
var books = tygdb.书名表.ToLookup(p => p.书名.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (p.作者名称 == null ? "" : p.作者名称.Trim())
);
//要添加的书集合
List <书名表> WillAddBooks = new List <书名表>();
for (int classid = 1; classid <= 8; classid++)
{
System.Diagnostics.UDPGroup.SendStrGB2312("开始采集图书列表" + "http://www.86zw.com/Book/ShowBookList.aspx?tclassid=" + classid + "&page={0}");
var al = new Skybot.Collections.Sites.BookList86zw_com()
{
BaseUrl = "http://www.86zw.com/Book/ShowBookList.aspx?tclassid=" + classid + "&page={0}"
}.DoWork();
//开始写入数据库
var AllBooks = from doc in al select doc;
//添加到Book中
for (int k = 0; k < AllBooks.Count(); k++)
{
try
{
var bookName = "";
var creter = "";
if (AllBooks.ElementAt(k).小说名称 != null)
{
bookName = AllBooks.ElementAt(k).小说名称;
}
if (AllBooks.ElementAt(k).作者 != null)
{
creter = AllBooks.ElementAt(k).作者;
}
//当前记录
//var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim());
TygModel.书名表 book = null;
if (books.Contains(
bookName.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (creter == null ? "" : creter.Trim())
))
{
//得到当前书更新记录
TygModel.书名表 currentBook = books[bookName.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (creter == null ? "" : creter.Trim() )
].ElementAt(0);
// currentBook.分类表 = book.分类表;
//currentBook.分类表ID = book.分类表ID;
//currentBook.GUID = book.GUID;
currentBook.采集用的URL1 = AllBooks.ElementAt(k).小说目录URL;
currentBook.采集用的URL2 = AllBooks.ElementAt(k).小说简介URL;
// currentBook.创建时间 = book.创建时间;
currentBook.最新章节 = AllBooks.ElementAt(k).最新章节;
//currentBook.作者名称 = book.作者名称;
//currentBook.说明 = book.说明;
// currentBook.书名 = book.书名;
//currentBook.最后更新时间 = book.最后更新时间;
currentBook.完本 = AllBooks.ElementAt(k).状态.Trim() == "完结" ? true : false;
//currentBook.配图 = book.配图;
}
else
{
book = AllBooks.ElementAt(k).Convert();
WillAddBooks.Add(book);
}
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
System.Diagnostics.UDPGroup.SendStrGB2312("已经完成书" + k + "/" + AllBooks.Count());
}
}
//开始全部添加
try
{
WillAddBooks.ForEach((book) =>
{
//当前记录
// var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == book.书名.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == book.作者名称.Trim());
if (!books.Contains(
book.书名.Replace("》", "").Replace("《", "").Trim() + "|"
// + (book.作者名称 == null ? "" : book.作者名称.Trim())
))
{
tygdb.AddTo书名表(book);
}
});
tygdb.SaveChanges();
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
finally
{
WillAddBooks.Clear();
}
System.Diagnostics.UDPGroup.SendStrGB2312("86zw 所有图书状态更新完成");
}
}));
Dispatcher.BeginInvoke(new Action(delegate
{
IsNotBusy = false;
}));
Task.Factory.StartNew(delegate
{
try
{
task.Wait();
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
Dispatcher.BeginInvoke(new Action(delegate
{
IsNotBusy = true;
}));
});
}
public void Update()
{
System.Threading.Tasks.Task task = System.Threading.Tasks.Task.Factory.StartNew(new Action(delegate
{
using (TygModel.Entities tygdb = new TygModel.Entities())
{
var books = tygdb.书名表.ToLookup(p => p.书名.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (p.作者名称 == null ? "" : p.作者名称.Trim())
);
//要添加的书集合
List <书名表> WillAddBooks = new List <书名表>();
for (int classid = 1; classid <= 10; classid++)
{
System.Diagnostics.UDPGroup.SendStrGB2312("开始采集图书列表" + "http://www.dawenxue.org/book/sort" + classid + "/0/{0}.html");
IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> alclass = new List <Skybot.Collections.Sites.AbstractBookInfo>();
Task <IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> > taskClass = Task.Factory.StartNew <IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> >(
() =>
{
var al = new Skybot.Collections.Sites.BookListdawenxue_org()
{
BaseUrl = "http://www.dawenxue.org/book/sort" + classid + "/0/{0}.html"
}.DoWork();
return(al);
});
try
{
alclass = taskClass.Result;
}
catch (AggregateException ex)
{
ex.Handle((exx) =>
{
System.Diagnostics.UDPGroup.SendStrGB2312(DateTime.Now + exx.Message + "|||||" + (ex.StackTrace == null ? " " : " " + ex.StackTrace));
return(true);
});
}
//开始写入数据库
var AllBooks = from doc in alclass select doc;
//添加到Book中
for (int k = 0; k < AllBooks.Count(); k++)
{
try
{
var bookName = "";
var creter = "";
if (AllBooks.ElementAt(k).小说名称 != null)
{
bookName = AllBooks.ElementAt(k).小说名称;
}
if (AllBooks.ElementAt(k).作者 != null)
{
creter = AllBooks.ElementAt(k).作者;
}
//当前记录
//var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim());
TygModel.书名表 book = null;
if (books.Contains(
bookName.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (creter == null ? "" : creter.Trim())
))
{
//得到当前书更新记录
TygModel.书名表 currentBook = tygdb.书名表.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim()).FirstOrDefault();
books[bookName.Replace("》", "").Replace("《", "").Trim() + "|"
//+ (creter == null ? "" : creter.Trim() )
].ElementAt(0);
// currentBook.分类表 = book.分类表;
//currentBook.分类表ID = book.分类表ID;
//currentBook.GUID = book.GUID;
currentBook.采集用的URL1 = AllBooks.ElementAt(k).小说目录URL;
currentBook.采集用的URL2 = AllBooks.ElementAt(k).小说简介URL;
// currentBook.创建时间 = book.创建时间;
currentBook.最新章节 = AllBooks.ElementAt(k).最新章节;
//currentBook.作者名称 = book.作者名称;
//currentBook.说明 = book.说明;
// currentBook.书名 = book.书名;
//currentBook.最后更新时间 = book.最后更新时间;
currentBook.完本 = AllBooks.ElementAt(k).状态.Trim() == "完成" ? true : false;
//currentBook.配图 = book.配图;
}
else
{
try
{
book = AllBooks.ElementAt(k).Convert();
WillAddBooks.Add(book);
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312("转换书时出现问题" + ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
}
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
System.Diagnostics.UDPGroup.SendStrGB2312("已经完成书" + k + "/" + AllBooks.Count());
}
}
//开始全部添加
try
{
WillAddBooks.ForEach((book) =>
{
//当前记录
// var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == book.书名.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == book.作者名称.Trim());
if (!books.Contains(
book.书名.Replace("》", "").Replace("《", "").Trim() + "|"
// + (book.作者名称 == null ? "" : book.作者名称.Trim())
))
{
tygdb.AddTo书名表(book);
}
});
tygdb.SaveChanges();
}
catch (Exception ex)
{
System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
}
finally
{
WillAddBooks.Clear();
}
System.Diagnostics.UDPGroup.SendStrGB2312("dawenxue 所有图书状态更新完成");
}
}));
Dispatcher.BeginInvoke(new Action(delegate
{
IsNotBusy = false;
}));
Task.Factory.StartNew(delegate
{
try
{
task.Wait();
}
catch (AggregateException ex)
{
ex.Handle((exx) =>
{
System.Diagnostics.UDPGroup.SendStrGB2312(DateTime.Now + exx.Message + "|||||" + (ex.StackTrace == null ? " " : " " + ex.StackTrace));
return(true);
});
}
Dispatcher.BeginInvoke(new Action(delegate
{
IsNotBusy = true;
}));
});
}