void worker()
{
List<queuedata> remove_m_working = new List<queuedata>();
List<int> finishedtime = new List<int>();
while (running)
{
if (error_emergencystop == true)
{
Console.WriteLine("threadeddownload: Dumping queues...");
TextWriter tw = new StreamWriter(error_downloadfile);
foreach (queuedata q in m_working)
{
tw.WriteLine(q.url);
tw.WriteLine(q.filename);
}
foreach (queuedata q in m_queue)
{
tw.WriteLine(q.url);
tw.WriteLine(q.filename);
}
tw.Close();
running = false;
break;
}
if (m_queue.Count == 0 && m_working.Count == 0)
{
running = false;
break;
}
if (m_working.Count < maxthreads && m_queue.Count > 0)
{
queuedata q = m_queue[0];
if (m_skipexisting == true) // check existing
{
if (File.Exists(q.filename))
{
//Console.WriteLine("threadeddownload: file already exist: " + q.filename);
if (DownloadedEvent != null)
DownloadedEvent(q.url, q.filename, q.custom);
m_queue.Remove(q);
continue;
}
}
if (q.retryitem == true)
{
q.retryitem = false; // debug
}
DateTime d = DateTime.Now; //q.timestamp = DateTime.Now; // WRONG
WebClient wc = new WebClient(); // unknown state if not properly cleared up..??
wc.DownloadStringCompleted += new DownloadStringCompletedEventHandler(wc_DownloadStringCompleted);
if (m_cookie != null || m_cookie != "")
wc.Headers.Add(HttpRequestHeader.Cookie, m_cookie);
//Console.WriteLine("Downloading: " + q.filename);
q.timestamp = new DateTime(d.Year, d.Month, d.Day, d.Hour, d.Minute, d.Second);
q.wc = wc;
q.wc.DownloadStringAsync(new Uri(q.url), q);
m_working.Add(q);
m_queue.Remove(q);
}
foreach (queuedata q in m_working)
{
if (q.errorcode == 1)
{
try
{
File.WriteAllText(q.filename, q.html); // don't call make valid filename.. sometimes filename is dir\file format
}
catch
{
if (FailedDownload != null)
FailedDownload(q.url, q.custom);
Console.WriteLine("threadeddownload: file write failed: " + q.filename);
error_download += q.url + "\n"; //\n\r for files, \r\n for windows label
error_download += q.filename + "\n";
File.WriteAllText(error_downloadfile, error_download);
}
bool cookiechanged = MonitorAndChangeCookie(q.wc.ResponseHeaders["Set-Cookie"], q.filename);
if (cookiechanged == true)
{
File.Delete(q.filename);
queuedata newq = new queuedata();
newq.url = q.url;
newq.filename = q.filename;
m_queue.Insert(0, newq); // redownload again
RefreshCookie();
}
q.wc.Dispose(); // memory leak
q.wc = null;
q.html = ""; // memory leak
if (DownloadedEvent != null && cookiechanged == false)
DownloadedEvent(q.url, q.filename, q.custom);
double passedseconds = ((TimeSpan)(DateTime.Now - q.timestamp)).TotalSeconds;
finishedtime.Add((int)passedseconds);
if (finishedtime.Count >= 10) // benchmark
{
int avg = 0;
foreach (int s in finishedtime)
{
avg += s;
}
avg /= finishedtime.Count;
finishedtime.Remove(0);
//howlongtogo = m_queue.Count / 10 * avg; //WRONG algo
howlongtogo = m_queue.Count * avg;
}
remove_m_working.Add(q); // remove this item from itself
}
else if (q.errorcode == -1)
{
Console.WriteLine("threadeddownload: retrying: " + q.filename);
q.retryitem = true;
q.errorcode = 0;
q.wc.Dispose();
q.wc = null;
if (q.retries < maxretries)
{
q.retries++;
remove_m_working.Add(q);
m_queue.Insert(0, q);
break;
}
else
{
Console.WriteLine("threadeddownload: failed: " + q.filename);
error_download += q.url + "\n"; //\n\r for files, \r\n for windows label
error_download += q.filename + "\n";
File.WriteAllText(error_downloadfile, error_download);
remove_m_working.Add(q);
break;
}
}
else if (q.errorcode == 0)
{
double passedseconds = ((TimeSpan)(DateTime.Now - q.timestamp)).TotalSeconds;
if ((int)passedseconds > maxtimeout)
{
Console.WriteLine("threadeddownload: timeout: " + q.filename);
q.wc.CancelAsync();
break;
}
}
}
foreach (queuedata q in remove_m_working)
m_working.Remove(q);
remove_m_working.Clear();
Thread.Sleep(100);
}
if (FinishedAll != null) // all files done
{
//Console.WriteLine("threadeddownload: Finished all");
FinishedAll();
}
}
public void Queue(string url, string filename, object custom)
{
queuedata q = new queuedata();
q.url = url;
q.filename = filename;
q.custom = custom;
m_queue.Add(q);
}
public void Start()
{
try
{
TextReader tr = new StreamReader(error_downloadfile);
int counter = 0;
for (; ; )
{
string url = tr.ReadLine();
string filename = tr.ReadLine();
if (url == null || filename == null)
break;
if (!File.Exists(filename))
{
queuedata q = new queuedata();
q.url = url;
q.filename = filename;
q.custom = "error casting if passed to user.. check type";
m_queue.Add(q);
counter++;
}
}
tr.Close();
Console.WriteLine("threadeddownload: Resuming items added: " + counter);
File.Delete(error_downloadfile);
}
catch { }
if (running) return; // dont start another one
running = true;
Thread t = new Thread(new ThreadStart(worker));
t.Start();
}
