public AddOrUpdateStatus AddOrUpdate(string key, QueueEntry value)
{
cacheLock.EnterUpgradeableReadLock();
try
{
QueueEntry result = null;
if (innerCache.TryGetValue(key, out result))
{
if (result == value)
{
return(AddOrUpdateStatus.Unchanged);
}
else
{
cacheLock.EnterWriteLock();
try
{
innerCache[key] = value;
}
finally
{
cacheLock.ExitWriteLock();
}
return(AddOrUpdateStatus.Updated);
}
}
else
{
cacheLock.EnterWriteLock();
try
{
innerCache.Add(key, value);
}
finally
{
cacheLock.ExitWriteLock();
}
return(AddOrUpdateStatus.Added);
}
}
finally
{
cacheLock.ExitUpgradeableReadLock();
}
}
public bool AddWithTimeout(string key, QueueEntry value, int timeout)
{
if (cacheLock.TryEnterWriteLock(timeout))
{
try
{
innerCache.Add(key, value);
}
finally
{
cacheLock.ExitWriteLock();
}
return(true);
}
else
{
return(false);
}
}
private void HandleQueue()
{
int pageCount = 0;
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
List <Task> tasklist = new List <Task>();
for (int i = 0; i < NUMBEROFTHREADS; i++)
{
tasklist.Add(Task.Run(() =>
{
do
{
string[] threadKeys;
try
{
threadKeys = UrlQueue.Keys().ToArray();
}
catch (ArgumentException)
{
continue;
}
foreach (string entry in threadKeys)
{
if (pageCount >= NUMBEROFPAGES)
{
break;
}
QueueEntry qEntry = UrlQueue.Read(entry);
if (!(DateTime.Now > qEntry.LastVisited.AddSeconds(qEntry.CrawlDelay)))
{
//Console.WriteLine("Delaying the access to:" + entry);
continue;
}
int idx = FindNextIndex(entry, qEntry.SubURLs);
if (idx == -1)
{
continue;
}
Console.WriteLine("Fetching from: " + qEntry.SubURLs[idx].ToString());
qEntry.LastVisited = DateTime.Now;
FetchData(qEntry.SubURLs[idx].ToString());
pageCount++;
Console.WriteLine("PageCount: " + pageCount + " | " + "Pr.sec: " + (pageCount / (stopwatch.ElapsedMilliseconds / 1000f)));
}
} while (pageCount < NUMBEROFPAGES);
}));
}
Task.WaitAll(tasklist.ToArray());
stopwatch.Stop();
Console.WriteLine("Total time elapsed: " + (stopwatch.ElapsedMilliseconds / 1000f));
Console.WriteLine("Queue Handler Complete...");
}
