public void Start(Uri baseURI,int threads)
{
n_ThreadCount = 1;
this.AmIBusy = false;
worker = new DocumentWorker[threads];
//Thread[] mythreads = new Thread[threads];
this.mythreads = new Thread[threads];
m_totalthreads = threads;
m_quit = false;
m_base = baseURI;
addURI(m_base);
worker[0] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[0].ThisSpider = this;
worker[0].Number = 0;
worker[0].ProcessFirstPage();
for (int i = 0; i < threads; i++)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
//bool AmIRunning = true;
while (!this.m_quit)
{
// This is a 'orrible 'ack to reduce the CPU overload.
Thread.Sleep(1000);
// We have a few options here...
// 1 - If the number of threads are zero
// 1.1 - There are items on the work queue
// - We restart the thread block...
// 1.2 - There are no items on the work queue
// - We end the process...
// 2 - If the number of threads is = TotalThreads Allowed.
// - We do nothing
// 3 - If the number of threads is > 0 and < Total threads allowed.
// 3.1 - There are items on the work queue
// - We restart all the threads...
// 3.2 - There are no items on the work queue
// - We do nothing. It'll happen on the next loop around...
if (n_ThreadCount <= 1)
{
if (m_workload.Count == 0)
this.m_quit = true;
else
{
for (int i = 0; i < threads; i++)
{
try
{
if (!mythreads[i].IsAlive)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
}
catch { }
}
}
}
else if ((n_ThreadCount > 1) && (n_ThreadCount < threads))
{
if (m_workload.Count > 0)
{
for (int i = 0; i < threads; i++)
{
try
{
if (!mythreads[i].IsAlive)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
}
catch { }
}
}
}
}
}
public void Start(Uri baseURI, int threads)
{
n_ThreadCount = 1;
this.AmIBusy = false;
worker = new DocumentWorker[threads];
//Thread[] mythreads = new Thread[threads];
this.mythreads = new Thread[threads];
m_totalthreads = threads;
m_quit = false;
m_base = baseURI;
addURI(m_base);
worker[0] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[0].ThisSpider = this;
worker[0].Number = 0;
worker[0].ProcessFirstPage();
for (int i = 0; i < threads; i++)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
//bool AmIRunning = true;
while (!this.m_quit)
{
// This is a 'orrible 'ack to reduce the CPU overload.
Thread.Sleep(1000);
// We have a few options here...
// 1 - If the number of threads are zero
// 1.1 - There are items on the work queue
// - We restart the thread block...
// 1.2 - There are no items on the work queue
// - We end the process...
// 2 - If the number of threads is = TotalThreads Allowed.
// - We do nothing
// 3 - If the number of threads is > 0 and < Total threads allowed.
// 3.1 - There are items on the work queue
// - We restart all the threads...
// 3.2 - There are no items on the work queue
// - We do nothing. It'll happen on the next loop around...
if (n_ThreadCount <= 1)
{
if (m_workload.Count == 0)
{
this.m_quit = true;
}
else
{
for (int i = 0; i < threads; i++)
{
try
{
if (!mythreads[i].IsAlive)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
}
catch { }
}
}
}
else if ((n_ThreadCount > 1) && (n_ThreadCount < threads))
{
if (m_workload.Count > 0)
{
for (int i = 0; i < threads; i++)
{
try
{
if (!mythreads[i].IsAlive)
{
worker[i] = new DocumentWorker(this, this.m_includetypes, this.m_contentlength, this.m_excludedirs, this.excludeindexes);
worker[i].ThisSpider = this;
worker[i].Number = i;
mythreads[i] = new Thread(new ThreadStart(worker[i].Process));
mythreads[i].Priority = ThreadPriority.BelowNormal;
mythreads[i].Start();
n_ThreadCount++;
}
}
catch { }
}
}
}
}
}