Example #1
0
        /// <summary>
        /// 写入文件
        /// </summary>
        public void Foreach()
        {
            TygModel.Entities tity = new TygModel.Entities();
            var items = tity.文章表.OrderBy(p => p.ID);
            int count = items.Count();

            tity.Dispose();
            for (int k = 0; k < count / 10000; k++)
            {
                using (TygModel.Entities ent = new TygModel.Entities())
                {
                    var records = ent.文章表.OrderBy(p => p.ID).Skip(k * 10000).Take(10000).ToList();

                    foreach (var sp in records)
                    {
                        if (list.Count >= 10000)
                        {
                            CreateMaps(list.ToArray(), SiteIndex++);
                            list.Clear();
                        }
                        //添加到路径
                        list.Add(url + sp.GetHTMLFilePath());
                        currentIndex++;
                        System.Console.WriteLine(string.Format("进度{0}/{1},{2}", currentIndex, count, sp.章节名));
                    }

                    records.Clear();
                    records = null;
                }
            }

            //全部写入
            CreateMaps(list.ToArray(), SiteIndex++);
        }
Example #2
0
    protected void Page_Load(object sender, EventArgs e)
    {
        List <书名表> books = new List <书名表>();

        System.Xml.Serialization.XmlSerializer xmlSer = new System.Xml.Serialization.XmlSerializer(books.GetType());

        /// <summary>
        /// 数据库访问对像
        /// </summary>
        using (TygModel.Entities Tygdb = new TygModel.Entities())
        {
            books = Tygdb.书名表.ToList();
            System.IO.StringWriter sw = new System.IO.StringWriter();
            xmlSer.Serialize(sw, books);

            //System.IO.File.WriteAllText(Server.MapPath("./books.html"),sw.ToString());
            Response.Write(sw.ToString());


            Tygdb.Connection.Close();
        }
    }
        public void Update()
        {
            System.Threading.Tasks.Task task = System.Threading.Tasks.Task.Factory.StartNew(new Action(delegate
            {
                using (TygModel.Entities tygdb = new TygModel.Entities())
                {
                    var books = tygdb.书名表.ToLookup(p => p.书名.Replace("》", "").Replace("《", "").Trim() + "|"
                                                   //+ (p.作者名称 == null ? "" : p.作者名称.Trim())
                                                   );

                    //要添加的书集合
                    List <书名表> WillAddBooks = new List <书名表>();



                    for (int classid = 1; classid <= 8; classid++)
                    {
                        System.Diagnostics.UDPGroup.SendStrGB2312("开始采集图书列表" + "http://www.86zw.com/Book/ShowBookList.aspx?tclassid=" + classid + "&page={0}");
                        var al = new Skybot.Collections.Sites.BookList86zw_com()
                        {
                            BaseUrl = "http://www.86zw.com/Book/ShowBookList.aspx?tclassid=" + classid + "&page={0}"
                        }.DoWork();

                        //开始写入数据库
                        var AllBooks = from doc in al select doc;

                        //添加到Book中
                        for (int k = 0; k < AllBooks.Count(); k++)
                        {
                            try
                            {
                                var bookName = "";
                                var creter   = "";
                                if (AllBooks.ElementAt(k).小说名称 != null)
                                {
                                    bookName = AllBooks.ElementAt(k).小说名称;
                                }

                                if (AllBooks.ElementAt(k).作者 != null)
                                {
                                    creter = AllBooks.ElementAt(k).作者;
                                }

                                //当前记录
                                //var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim());
                                TygModel.书名表 book = null;
                                if (books.Contains(
                                        bookName.Replace("》", "").Replace("《", "").Trim() + "|"
                                        //+ (creter == null ? "" : creter.Trim())
                                        ))
                                {
                                    //得到当前书更新记录
                                    TygModel.书名表 currentBook = books[bookName.Replace("》", "").Replace("《", "").Trim() + "|"
                                                                     //+ (creter == null ? "" : creter.Trim() )
                                                               ].ElementAt(0);
                                    // currentBook.分类表 = book.分类表;
                                    //currentBook.分类表ID = book.分类表ID;
                                    //currentBook.GUID = book.GUID;
                                    currentBook.采集用的URL1 = AllBooks.ElementAt(k).小说目录URL;
                                    currentBook.采集用的URL2 = AllBooks.ElementAt(k).小说简介URL;
                                    // currentBook.创建时间 = book.创建时间;
                                    currentBook.最新章节 = AllBooks.ElementAt(k).最新章节;
                                    //currentBook.作者名称 = book.作者名称;
                                    //currentBook.说明 = book.说明;
                                    // currentBook.书名 = book.书名;
                                    //currentBook.最后更新时间 = book.最后更新时间;
                                    currentBook.完本 = AllBooks.ElementAt(k).状态.Trim() == "完结" ? true : false;
                                    //currentBook.配图 = book.配图;
                                }
                                else
                                {
                                    book = AllBooks.ElementAt(k).Convert();

                                    WillAddBooks.Add(book);
                                }
                            }
                            catch (Exception ex)
                            {
                                System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                            }
                            System.Diagnostics.UDPGroup.SendStrGB2312("已经完成书" + k + "/" + AllBooks.Count());
                        }
                    }
                    //开始全部添加
                    try
                    {
                        WillAddBooks.ForEach((book) =>
                        {
                            //当前记录
                            // var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == book.书名.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == book.作者名称.Trim());

                            if (!books.Contains(
                                    book.书名.Replace("》", "").Replace("《", "").Trim() + "|"
                                    // + (book.作者名称 == null ? "" : book.作者名称.Trim())
                                    ))
                            {
                                tygdb.AddTo书名表(book);
                            }
                        });

                        tygdb.SaveChanges();
                    }
                    catch (Exception ex)
                    {
                        System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                    }
                    finally
                    {
                        WillAddBooks.Clear();
                    }

                    System.Diagnostics.UDPGroup.SendStrGB2312("86zw 所有图书状态更新完成");
                }
            }));
            Dispatcher.BeginInvoke(new Action(delegate
            {
                IsNotBusy = false;
            }));
            Task.Factory.StartNew(delegate
            {
                try
                {
                    task.Wait();
                }
                catch (Exception ex)
                {
                    System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                }
                Dispatcher.BeginInvoke(new Action(delegate
                {
                    IsNotBusy = true;
                }));
            });
        }
        public void Update()
        {
            System.Threading.Tasks.Task task = System.Threading.Tasks.Task.Factory.StartNew(new Action(delegate
            {
                using (TygModel.Entities tygdb = new TygModel.Entities())
                {
                    var books = tygdb.书名表.ToLookup(p => p.书名.Replace("》", "").Replace("《", "").Trim() + "|"
                                                   //+ (p.作者名称 == null ? "" : p.作者名称.Trim())
                                                   );

                    //要添加的书集合
                    List <书名表> WillAddBooks = new List <书名表>();



                    for (int classid = 1; classid <= 10; classid++)
                    {
                        System.Diagnostics.UDPGroup.SendStrGB2312("开始采集图书列表" + "http://www.dawenxue.org/book/sort" + classid + "/0/{0}.html");

                        IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> alclass = new List <Skybot.Collections.Sites.AbstractBookInfo>();

                        Task <IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> > taskClass = Task.Factory.StartNew <IEnumerable <Skybot.Collections.Sites.AbstractBookInfo> >(
                            () =>
                        {
                            var al = new Skybot.Collections.Sites.BookListdawenxue_org()
                            {
                                BaseUrl = "http://www.dawenxue.org/book/sort" + classid + "/0/{0}.html"
                            }.DoWork();
                            return(al);
                        });
                        try
                        {
                            alclass = taskClass.Result;
                        }
                        catch (AggregateException ex)
                        {
                            ex.Handle((exx) =>
                            {
                                System.Diagnostics.UDPGroup.SendStrGB2312(DateTime.Now + exx.Message + "|||||" + (ex.StackTrace == null ? " " : " " + ex.StackTrace));
                                return(true);
                            });
                        }

                        //开始写入数据库
                        var AllBooks = from doc in alclass select doc;

                        //添加到Book中
                        for (int k = 0; k < AllBooks.Count(); k++)
                        {
                            try
                            {
                                var bookName = "";
                                var creter   = "";
                                if (AllBooks.ElementAt(k).小说名称 != null)
                                {
                                    bookName = AllBooks.ElementAt(k).小说名称;
                                }

                                if (AllBooks.ElementAt(k).作者 != null)
                                {
                                    creter = AllBooks.ElementAt(k).作者;
                                }

                                //当前记录
                                //var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim());
                                TygModel.书名表 book = null;
                                if (books.Contains(
                                        bookName.Replace("》", "").Replace("《", "").Trim() + "|"
                                        //+ (creter == null ? "" : creter.Trim())
                                        ))
                                {
                                    //得到当前书更新记录
                                    TygModel.书名表 currentBook = tygdb.书名表.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == bookName.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == creter.Trim()).FirstOrDefault();

                                    books[bookName.Replace("》", "").Replace("《", "").Trim() + "|"
                                          //+ (creter == null ? "" : creter.Trim() )
                                    ].ElementAt(0);
                                    // currentBook.分类表 = book.分类表;
                                    //currentBook.分类表ID = book.分类表ID;
                                    //currentBook.GUID = book.GUID;
                                    currentBook.采集用的URL1 = AllBooks.ElementAt(k).小说目录URL;
                                    currentBook.采集用的URL2 = AllBooks.ElementAt(k).小说简介URL;
                                    // currentBook.创建时间 = book.创建时间;
                                    currentBook.最新章节 = AllBooks.ElementAt(k).最新章节;
                                    //currentBook.作者名称 = book.作者名称;
                                    //currentBook.说明 = book.说明;
                                    // currentBook.书名 = book.书名;
                                    //currentBook.最后更新时间 = book.最后更新时间;
                                    currentBook.完本 = AllBooks.ElementAt(k).状态.Trim() == "完成" ? true : false;
                                    //currentBook.配图 = book.配图;
                                }
                                else
                                {
                                    try
                                    {
                                        book = AllBooks.ElementAt(k).Convert();

                                        WillAddBooks.Add(book);
                                    }
                                    catch (Exception ex)
                                    {
                                        System.Diagnostics.UDPGroup.SendStrGB2312("转换书时出现问题" + ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                                    }
                                }
                            }
                            catch (Exception ex)
                            {
                                System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                            }
                            System.Diagnostics.UDPGroup.SendStrGB2312("已经完成书" + k + "/" + AllBooks.Count());
                        }
                    }
                    //开始全部添加
                    try
                    {
                        WillAddBooks.ForEach((book) =>
                        {
                            //当前记录
                            // var records = books.Where(p => p.书名.Replace("》", "").Replace("《", "").Trim() == book.书名.Replace("》", "").Replace("《", "").Trim() && p.作者名称.Trim() == book.作者名称.Trim());

                            if (!books.Contains(
                                    book.书名.Replace("》", "").Replace("《", "").Trim() + "|"
                                    // + (book.作者名称 == null ? "" : book.作者名称.Trim())
                                    ))
                            {
                                tygdb.AddTo书名表(book);
                            }
                        });

                        tygdb.SaveChanges();
                    }
                    catch (Exception ex)
                    {
                        System.Diagnostics.UDPGroup.SendStrGB2312(ex.Message + (ex.StackTrace != null ? ex.StackTrace : ""));
                    }
                    finally
                    {
                        WillAddBooks.Clear();
                    }

                    System.Diagnostics.UDPGroup.SendStrGB2312("dawenxue 所有图书状态更新完成");
                }
            }));
            Dispatcher.BeginInvoke(new Action(delegate
            {
                IsNotBusy = false;
            }));
            Task.Factory.StartNew(delegate
            {
                try
                {
                    task.Wait();
                }
                catch (AggregateException ex)
                {
                    ex.Handle((exx) =>
                    {
                        System.Diagnostics.UDPGroup.SendStrGB2312(DateTime.Now + exx.Message + "|||||" + (ex.StackTrace == null ? " " : " " + ex.StackTrace));
                        return(true);
                    });
                }
                Dispatcher.BeginInvoke(new Action(delegate
                {
                    IsNotBusy = true;
                }));
            });
        }