private void GetNodes(List <Leaf> nodes) { loger.Error("GetNodes"); if (nodes != null && nodes.Count > 0) { foreach (var node in nodes) { if (stop) { return; } bool loaded = DAL.TreeManager.IsLoaded(node.site_id, node.parent_site_id); //using (var context = new dataEntities()) // loaded = context.data.Any(d => d.site_id == node.site_id && // (d.parent_site_id == node.parent_site_id || (node.parent_site_id == null && d.parent_site_id == null)) && // d.is_loaded == true); if (loaded) { log(node.site_id + " - done"); continue; } string error; string html; html = WebPage.LoadPage(String.Format(urlPattern, node.site_id), Encoding.UTF8, out error); if (String.IsNullOrWhiteSpace(error)) { if (!String.IsNullOrWhiteSpace(html)) { foreach (var item in html.Split('$')) { if (!String.IsNullOrWhiteSpace(item)) { if (stop) { return; } string[] obj = item.Split(':'); long id; if (long.TryParse(obj[0], out id)) { var node_name = System.Web.HttpUtility.HtmlDecode(obj[3]); var n = new Leaf() { site_id = obj[0], Name = node_name, parent_site_id = node.site_id, parent_id = node.id }; node.Nodes.Add(n); using (var context = new dataEntities()) { //var d_indb = context.data.Where(t => t.site_id == n.site_id && t.parent_site_id == n.parent_site_id && t.parent_id == n.parent_id).FirstOrDefault(); var d_indb = DAL.TreeManager.GetId(n.site_id, n.parent_site_id, (long)n.parent_id); if (d_indb == -1) { var d = new data() { id = -1, site_id = n.site_id, name = n.Name, parent_id = n.parent_id, parent_site_id = n.parent_site_id, position = node.Nodes.Count, is_doc = false, is_loaded = false }; context.data.AddObject(d); context.SaveChanges(); n.id = d.id; } else { n.id = d_indb; } } } } } ; GetNodes(node.Nodes); } else //if doc { if (stop) { return; } string urld = "http://online.zakon.kz/Search.aspx"; int max = 30; int maxThread = 5; var taskList = new List <DownloadTask>(); int posiont = 1; for (int i = 1; i <= max; i++) { html = WebPage.LoadPage(urld, CreateNVC22(node.site_id, i.ToString()), Encoding.UTF8, out error); Match m = Regex.Match(html, @"<a[\s]href='http://online\.zakon\.kz/Document/\?doc_id=([\d]*?)'.*?>(.*?)<", RegexOptions.Singleline); if (!m.Success) { break; } while (m.Success && m.Groups.Count > 1) { if (stop) { return; } var dt = (new DownloadTask(m.Groups[1].ToString(), m.Groups[2].ToString(), node.site_id, node.id, posiont, log)); dt.Execute(); GC.Collect(); // Queue.Enqueue(dt); //taskList.Add(dt); posiont++; m = m.NextMatch(); } } //var tList = new List<Thread>(); //foreach (var item in taskList) //{ // var t = new Thread(item.Execute); // tList.Add(t); //} //while (tList.Count > 0) //{ // var qList = new List<Thread>(); // for (int i = 0; i < Math.Min(maxThread, tList.Count); i++) // { // qList.Add(tList[i]); // tList[i].Start(); // } // foreach (var q in qList) // { // q.Join(); // tList.Remove(q); // } //} //All related files dowloaded, let's try to set root like loaded using (var context = new dataEntities()) { var nnn = context.data.Where(d => d.id == node.id).FirstOrDefault(); if (nnn != null) { nnn.is_loaded = true; context.SaveChanges(); } } } } else { log("Node:" + error); } } } }
public void Execute() { var d_indb = DAL.TreeManager.GetId(doc_id, parent_site_id, parent_id); if (d_indb > 0) { log("doc_id =" + doc_id + " redy"); return; } Stopwatch sw = Stopwatch.StartNew(); WebPage.DownloadFile(url, file_name, callback); sw.Stop(); var timeLoad = sw.ElapsedMilliseconds / 1000; var fileSize = (new FileInfo(file_name).Length); if (fileSize > 0) { using (var context = new dataEntities()) { //var d_indb = context.data.Where(t => t.site_id == doc_id && t.parent_site_id == parent_site_id && parent_id == this.parent_id).FirstOrDefault(); string data_rtf, data_text; sw.Reset(); sw.Start(); string doc_info; GetText(Path.GetFullPath(file_name), out data_rtf, out data_text, out doc_info); sw.Stop(); var time_text = sw.ElapsedMilliseconds / 1000; sw.Reset(); sw.Start(); var d = new data() { id = -1, site_id = this.doc_id, name = doc_name, parent_id = parent_id, parent_site_id = parent_site_id, position = position, is_doc = true, is_loaded = true, data_rtf = data_rtf, data_text = data_text }; context.data.AddObject(d); context.SaveChanges(); sw.Stop(); var time_save = sw.ElapsedMilliseconds / 1000; var milliSec = timeLoad + time_text + time_save; var str = String.Empty; if (milliSec > 3) { str += "all:" + milliSec + "\t\tload:" + timeLoad + "\t\ttext:" + time_text + "\t\tLENGTH:" + fileSize + "(" + doc_info + ")"; } log("doc_id =" + doc_id + "\t" + str); } } else { log("Leaf: doc file length is 0"); } }
/// <summary> /// Deprecated Method for adding a new object to the data EntitySet. Consider using the .Add method of the associated ObjectSet<T> property instead. /// </summary> public void AddTodata(data data) { base.AddObject("data", data); }