void worker() { List<queuedata> remove_m_working = new List<queuedata>(); List<int> finishedtime = new List<int>(); while (running) { if (error_emergencystop == true) { Console.WriteLine("threadeddownload: Dumping queues..."); TextWriter tw = new StreamWriter(error_downloadfile); foreach (queuedata q in m_working) { tw.WriteLine(q.url); tw.WriteLine(q.filename); } foreach (queuedata q in m_queue) { tw.WriteLine(q.url); tw.WriteLine(q.filename); } tw.Close(); running = false; break; } if (m_queue.Count == 0 && m_working.Count == 0) { running = false; break; } if (m_working.Count < maxthreads && m_queue.Count > 0) { queuedata q = m_queue[0]; if (m_skipexisting == true) // check existing { if (File.Exists(q.filename)) { //Console.WriteLine("threadeddownload: file already exist: " + q.filename); if (DownloadedEvent != null) DownloadedEvent(q.url, q.filename, q.custom); m_queue.Remove(q); continue; } } if (q.retryitem == true) { q.retryitem = false; // debug } DateTime d = DateTime.Now; //q.timestamp = DateTime.Now; // WRONG WebClient wc = new WebClient(); // unknown state if not properly cleared up..?? wc.DownloadStringCompleted += new DownloadStringCompletedEventHandler(wc_DownloadStringCompleted); if (m_cookie != null || m_cookie != "") wc.Headers.Add(HttpRequestHeader.Cookie, m_cookie); //Console.WriteLine("Downloading: " + q.filename); q.timestamp = new DateTime(d.Year, d.Month, d.Day, d.Hour, d.Minute, d.Second); q.wc = wc; q.wc.DownloadStringAsync(new Uri(q.url), q); m_working.Add(q); m_queue.Remove(q); } foreach (queuedata q in m_working) { if (q.errorcode == 1) { try { File.WriteAllText(q.filename, q.html); // don't call make valid filename.. sometimes filename is dir\file format } catch { if (FailedDownload != null) FailedDownload(q.url, q.custom); Console.WriteLine("threadeddownload: file write failed: " + q.filename); error_download += q.url + "\n"; //\n\r for files, \r\n for windows label error_download += q.filename + "\n"; File.WriteAllText(error_downloadfile, error_download); } bool cookiechanged = MonitorAndChangeCookie(q.wc.ResponseHeaders["Set-Cookie"], q.filename); if (cookiechanged == true) { File.Delete(q.filename); queuedata newq = new queuedata(); newq.url = q.url; newq.filename = q.filename; m_queue.Insert(0, newq); // redownload again RefreshCookie(); } q.wc.Dispose(); // memory leak q.wc = null; q.html = ""; // memory leak if (DownloadedEvent != null && cookiechanged == false) DownloadedEvent(q.url, q.filename, q.custom); double passedseconds = ((TimeSpan)(DateTime.Now - q.timestamp)).TotalSeconds; finishedtime.Add((int)passedseconds); if (finishedtime.Count >= 10) // benchmark { int avg = 0; foreach (int s in finishedtime) { avg += s; } avg /= finishedtime.Count; finishedtime.Remove(0); //howlongtogo = m_queue.Count / 10 * avg; //WRONG algo howlongtogo = m_queue.Count * avg; } remove_m_working.Add(q); // remove this item from itself } else if (q.errorcode == -1) { Console.WriteLine("threadeddownload: retrying: " + q.filename); q.retryitem = true; q.errorcode = 0; q.wc.Dispose(); q.wc = null; if (q.retries < maxretries) { q.retries++; remove_m_working.Add(q); m_queue.Insert(0, q); break; } else { Console.WriteLine("threadeddownload: failed: " + q.filename); error_download += q.url + "\n"; //\n\r for files, \r\n for windows label error_download += q.filename + "\n"; File.WriteAllText(error_downloadfile, error_download); remove_m_working.Add(q); break; } } else if (q.errorcode == 0) { double passedseconds = ((TimeSpan)(DateTime.Now - q.timestamp)).TotalSeconds; if ((int)passedseconds > maxtimeout) { Console.WriteLine("threadeddownload: timeout: " + q.filename); q.wc.CancelAsync(); break; } } } foreach (queuedata q in remove_m_working) m_working.Remove(q); remove_m_working.Clear(); Thread.Sleep(100); } if (FinishedAll != null) // all files done { //Console.WriteLine("threadeddownload: Finished all"); FinishedAll(); } }
public void Queue(string url, string filename, object custom) { queuedata q = new queuedata(); q.url = url; q.filename = filename; q.custom = custom; m_queue.Add(q); }
public void Start() { try { TextReader tr = new StreamReader(error_downloadfile); int counter = 0; for (; ; ) { string url = tr.ReadLine(); string filename = tr.ReadLine(); if (url == null || filename == null) break; if (!File.Exists(filename)) { queuedata q = new queuedata(); q.url = url; q.filename = filename; q.custom = "error casting if passed to user.. check type"; m_queue.Add(q); counter++; } } tr.Close(); Console.WriteLine("threadeddownload: Resuming items added: " + counter); File.Delete(error_downloadfile); } catch { } if (running) return; // dont start another one running = true; Thread t = new Thread(new ThreadStart(worker)); t.Start(); }