private void workerSimulator()
{
int errCount = 0, requestNum = 1;
while (true)
{
Thread.Sleep(10);
if (SyncAccessor.queueSize <Url>(serverQueues[0]) > 0)
{
Url url = SyncAccessor.getFromQueue <Url>(serverQueues[0], 100);
if (url != null)
{
Console.WriteLine("******************************************************");
Console.WriteLine("URL DEQUEUED: ");
Console.WriteLine(url.getUrl());
Console.WriteLine("RANK OF URL");
Console.WriteLine(url.getRank());
Console.WriteLine("REQUEST NUM");
Console.WriteLine(requestNum++);
}
else
{
errCount++;
Console.WriteLine("ERROR - " + errCount.ToString());
}
}
}
}
/**
* This method scedules the tasks given to the server queues in order to be processed
* if there is no tasks so the method will wait until there is something to be processed
* NOTE: This method never returns
*/
public override void sceduleTasks()
{
Dictionary <String, String> dictionary = new Dictionary <String, String>();
int serverTurn = 0;
Url request = null;
while (true)
{
try
{
// get new request
SyncAccessor.getSlot(2, 0);
request = SyncAccessor.getFromQueue <Url>(_tasksQueue, _timer);
// handle the request
if (dictionary.ContainsKey(request.getUrl()))
{
// if it already exists need to pick another one
continue;
}
else
{
// if not just mark it as old and continue
dictionary.Add(request.getUrl(), null);
}
// now there's a new request we should put it in the server queues
bool needToPutRequest = true;
int retryCount = 0;
while (needToPutRequest)
{
SyncAccessor.getSlot(2, 0);
if (SyncAccessor.queueSize <Url>(_serversQueues[serverTurn]) < _limit)
{
needToPutRequest = false;
SyncAccessor.getSlot(2, 0);
SyncAccessor.putInQueue <Url>(_serversQueues[serverTurn], request);
}
else
{
retryCount++;
if (retryCount > MAX_RETRY_COUNTER)
{
serverTurn = (serverTurn + 1) % _serversQueues.Count;
}
else
{
Thread.Sleep(_timer * 3);
}
}
}
serverTurn = (serverTurn + 1) % _serversQueues.Count;
}
catch (Exception e)
{
RuntimeStatistics.addToErrors(1);
}
}
}
private void fillQueue(int minRange, int maxRange)
{
for (int i = minRange; i < maxRange; i++)
{
String url = generateString(i);
SyncAccessor.putInQueue <Url>(feedback, new Url(url, 0, i % 100, url, i));
}
}
/**
* it will invoke the worker to start working on the tasks - never returns
*/
public void run()
{
int requestNum = 0, timeoutCounter = 0;
bool needToTerminate = false;
TimeSpan totalProcessTime;
Thread.Sleep(10000);
while (needToTerminate == false)
{
DateTime startTime = DateTime.Now;
try
{
//System.Console.WriteLine("-<>--------------------------------------------------------------------------");
SyncAccessor.getSlot(2, 1);
Url task = SyncAccessor.getFromQueue <Url>(_tasks, _timer);
//System.Console.WriteLine(" Start Working on : " + task.getUrl() + " ...");
ResourceContent content = _fetchers.fetchResource(task.getUrl());
if (content.isValid() != true)
{
timeoutCounter++;
//System.Console.WriteLine(" Fetch Failed Ignoring ... ");
continue;
}
//System.Console.WriteLine(" Fetched Successfully ... ");
ResourceContent modifiedContent = new ResourceContent(content.getResourceUrl(), content.getResourceType()
, content.getResourceContent(), content.getReturnCode(), task.getRank());
DateTime startProcess = DateTime.Now;
_processors.processResource(modifiedContent);
DateTime endProcess = DateTime.Now;
totalProcessTime = endProcess - startProcess;
//System.Console.WriteLine(" URL Processed Successfully ... ");
System.Console.WriteLine(" URL Processed Successfully ... ");
}
catch (Exception e)
{
//System.Console.WriteLine("[Exception Happened] " + e);
RuntimeStatistics.addToErrors(1);
continue;
}
DateTime endTime = DateTime.Now;
TimeSpan totalRequestTime = endTime - startTime;
if (LogDebuggerControl.getInstance().enableTiming)
{
// write request time to timing log file
StreamWriter sw = new
StreamWriter("_DEBUG_INFO_TIMING@" + System.Threading.Thread.CurrentThread.ManagedThreadId + ".txt", true);
sw.WriteLine(" TIMING FOR REQ - " + requestNum++ + " takes about " + totalRequestTime.TotalSeconds + " s, Processed At " + totalProcessTime.TotalSeconds + " s");
sw.Close();
}
}
}
public void deployLinksToFrontier(Url urlProcessed)
{
/*
* StreamWriter sw = new StreamWriter("Frontier.txt", true);
* string temp = urlProcessed.ToString().Replace("[-]", " " + System.Environment.NewLine);
* sw.WriteLine(temp);
* sw.Close();
*/
SyncAccessor.putInQueue <Url>(queueFronier, urlProcessed);
}
/**
* This method scedules the tasks given to the server queues in order to be processed
* if there is no tasks so the method will wait until there is something to be processed
* NOTE: This method never returns
*/
public override void sceduleTasks()
{
int serverTurn = 0;
bool getNewRequest = true;
Url request = null;
Url request2 = null;
while (true)
{
try
{
int inserts = 0;
SyncAccessor.getSlot(ThreadsDim + 1, 0);
while (SyncAccessor.queueSize <Url>(_tasksQueue) != 0 && inserts < MAX_INSERTS_IN_TIME)
{
request2 = SyncAccessor.getFromQueue <Url>(_tasksQueue, _timer);
_rankingTrie.add(request2);
inserts++;
}
if (getNewRequest)
{
RuntimeStatistics.setFrontierUrls(_rankingTrie.count());
if (_rankingTrie.count() == 0)
{
Thread.Sleep(_timer);
continue;
}
request = _rankingTrie.pop();
getNewRequest = false;
}
SyncAccessor.getSlot(2, 0);
if (SyncAccessor.queueSize <Url>(_serversQueues[serverTurn]) < _limit)
{
SyncAccessor.getSlot(2, 0);
SyncAccessor.putInQueue <Url>(_serversQueues[serverTurn], request);
getNewRequest = true;
}
else
{
getNewRequest = false;
}
serverTurn = (serverTurn + 1) % _serversQueues.Count;
}
catch (Exception e)
{
RuntimeStatistics.addToErrors(1);
}
}
}
/**
* This method scedules the tasks given to the server queues in order to be processed
* if there is no tasks so the method will wait until there is something to be processed
* NOTE: This method never returns
*/
public void sceduleTasks()
{
Dictionary <String, String> dictionary = new Dictionary <String, String>();
int serverTurn = 0, iterations = 0;
bool getNewRequest = true, needToTerminate = false;;
Url request = null;
while (needToTerminate == false)
{
try
{
if (getNewRequest)
{
request = SyncAccessor.getFromQueue <Url>(_tasksQueue, _timer);
getNewRequest = false;
}
getNewRequest = true;
if (dictionary.ContainsKey(request.getUrl()))
{
continue;
}
dictionary.Add(request.getUrl(), null);
if (SyncAccessor.queueSize <Url>(_serversQueues[serverTurn]) <= _limit)
{
SyncAccessor.putInQueue <Url>(_serversQueues[serverTurn], request);
}
else
{
getNewRequest = false;
}
serverTurn = (serverTurn + 1) % _serversQueues.Count;
iterations++;
if (iterations >= _checkStatusLimit)
{
iterations = 0;
if (_shouldStop)
{
//System.Console.WriteLine("Frontier Thread recieved should stop");
needToTerminate = true;
}
}
}
catch (Exception e)
{
RuntimeStatistics.addToErrors(1);
}
}
}
private void workerSimulator5()
{
int errCount = 0, urlsNum = 1, emptyQueueCount = 0;
bool queueFirstNotEmpty = false;
while (true)
{
Thread.Sleep(10);
if (SyncAccessor.queueSize <Url>(serverQueues[1]) > 0)
{
queueFirstNotEmpty = true;
emptyQueueCount = 0;
Url url = SyncAccessor.getFromQueue <Url>(serverQueues[1], 100);
if (url != null)
{
RuntimeStatistics.addToCrawledUrls(1);
StreamWriter sw = new
StreamWriter("FrontierDebugger2.txt", true);
sw.WriteLine("******************************************************");
sw.WriteLine("URL DEQUEUED: ");
sw.WriteLine(url.getUrl());
sw.WriteLine("RANK OF URL");
sw.WriteLine(url.getRank());
sw.WriteLine("NumberOfUrls");
sw.WriteLine(urlsNum++);
sw.Close();
}
else
{
errCount++;
Console.WriteLine("ERROR - " + errCount.ToString());
}
}
else
{
Thread.Sleep(1000);
if (emptyQueueCount > 2)
{
break;
}
if (queueFirstNotEmpty)
{
emptyQueueCount++;
}
}
}
}
private void workerSimulator2()
{
int errCount = 0, emptyQueueCount = 0, urlsNum = 1;
bool queueFirstNotEmpty = false;
while (true)
{
//Thread.Sleep(10);
if (SyncAccessor.queueSize <Url>(serverQueues[0]) > 0)
{
queueFirstNotEmpty = true;
emptyQueueCount = 0;
Url url = SyncAccessor.getFromQueue <Url>(serverQueues[0], 100);
if (url != null)
{
Console.WriteLine("******************************************************");
Console.WriteLine("URL DEQUEUED: ");
Console.WriteLine(url.getUrl());
Console.WriteLine("RANK OF URL");
Console.WriteLine(url.getRank());
Console.WriteLine("NumberOfUrls");
Console.WriteLine(urlsNum++);
}
else
{
errCount++;
Console.WriteLine("ERROR - " + errCount.ToString());
}
}
else
{
Thread.Sleep(1000);
if (emptyQueueCount > 2)
{
break;
}
if (queueFirstNotEmpty)
{
emptyQueueCount++;
}
}
}
}
private void workerSimulator3()
{
int errCount = 0, emptyQueueCount = 0;
bool queueFirstNotEmpty = false;
while (true)
{
Thread.Sleep(10);
if (SyncAccessor.queueSize <Url>(serverQueues[0]) > 0)
{
queueFirstNotEmpty = true;
emptyQueueCount = 0;
Url url = SyncAccessor.getFromQueue <Url>(serverQueues[0], 100);
if (url != null)
{
RuntimeStatistics.addToCrawledUrls(1);
}
else
{
RuntimeStatistics.addToErrors(1);
errCount++;
Console.WriteLine("ERROR - " + errCount.ToString());
}
}
else
{
Thread.Sleep(1000);
if (emptyQueueCount > 2)
{
break;
}
if (queueFirstNotEmpty)
{
emptyQueueCount++;
}
}
}
}
