public void TrackJob(HtmlRecord record)
{
lock (jobSet)
{
jobSet.Add(record.recordid, record);
}
}
public void Enqueue(HtmlRecord page)
{
lock (workQueue)
{
workQueue.Enqueue(page);
}
}
public void EnqueueWork(HtmlRecord record)
{
lock (domainDictionary)
{
string key = record.domain.Host;
if (domainDictionary.ContainsKey(key))
{
domainDictionary[key].Enqueue(record);
}
else
{
domainDictionary[key] = new Domain(key);
domainDictionary[key].Enqueue(record);
domainDictionary[key].InitTimer();
}
}
}
public JobStatus GetJobStatus(ObjectId recordid)
{
lock (jobSet)
{
JobStatus status = JobStatus.None;
if (jobSet.ContainsKey(recordid))
{
HtmlRecord record = jobSet[recordid];
status = record.jobStatus;
if (status == JobStatus.ErrorLoading ||
status == JobStatus.ErrorRequesting ||
status == JobStatus.LoadingPages)
{
record.KillProcess();
}
}
return(status);
}
}
public ServerResponse EnqueueJob(byte[] zippedBytes)
{
ServerResponse response = ServerResponse.ServerError;
try
{
byte[] recordBytes = RestAPI.Unzip(zippedBytes);
HtmlRecord record = BSON.Deserialize <HtmlRecord>(recordBytes);
record.Initialize();
WebCrawler.Instance.EnqueueWork(record);
response = ServerResponse.Success;
System.Diagnostics.Debug.Print("[" + DateTime.Now.ToString() + "]" + " Received: " +
record.domain.AbsoluteUri);
}
catch (Exception ex)
{
System.Diagnostics.Debug.WriteLine(ex.ToString());
response = ServerResponse.ServerError;
}
return(response);
}
public void Update(object stateInfo)
{
if (workQueue.Count != 0)
{
HtmlRecord record = Dequeue();
if (record.waitTime != 0)
{
ThreadPool.QueueUserWorkItem(new WaitCallback(record.SleepThenUpdate));
}
else
{
record.BeginUpdate();
}
}
else
{
timer.Dispose();
WebCrawler.Instance.RemoveDomain(this.domainName);
}
}
public void Send()
{
RestAPI api = new RestAPI();
while (true)
{
HtmlRecord record = messageQueue.Take();
byte[] recordBytes = BSON.Serialize(record);
JObject obj = null;
try
{
obj = api.ReturnFinishedJob(recordBytes);
bool successful = obj.GetValue("Successful").Value <bool>();
if (successful)
{
ServerResponse response = api.ParseResponse(obj);
if (response != ServerResponse.Success)
{
messageQueue.Add(record);
}
else
{
System.Diagnostics.Debug.Print("[" + DateTime.Now.ToString() + "]" +
" Sent: " + record.domain.AbsoluteUri);
}
}
else
{
messageQueue.Add(record);
}
}
catch (WebException ex)
{
System.Diagnostics.Debug.Print(ex.Message);
messageQueue.Add(record);
}
}
}
public void EnqueueResult(HtmlRecord record)
{
messageQueue.Add(record);
}