public override bool ShouldProcess(Task task)
{
AmazonTaskType att = (AmazonTaskType)task.Type;
if (att == AmazonTaskType.IMAGE
//|| att == AmazonTaskType.IMAGE_ORI
)
{
return(false);
}
return(true);
}
public void GetLocalPath(Task task, out string filePath)
{
AmazonTaskType type = (AmazonTaskType)task.Type;
string f = null;
int t1 = System.Environment.TickCount;
semaphore.WaitOne();
int t2 = System.Environment.TickCount;
try
{
if (type == AmazonTaskType.IMAGE
//|| type == AmazonTaskType.IMAGE_ORI
)
{
string filename;
f = "I";
ImageToFilePath(task, out filePath, out filename);
}
else
{
f = "W";
int timea, timeb;
WebPageToFilePath(task, out filePath, out timea, out timeb);
if (timea > 5000)
{
Console.WriteLine("TimeA {0}", timea);
}
if (timeb > 5000)
{
Console.WriteLine("TimeB {0}", timeb);
}
}
}
catch (Exception e)
{
Console.WriteLine(e);
filePath = null;
}
finally
{
semaphore.Release();
}
//Monitor.Exit(locker);
int t3 = System.Environment.TickCount;
if (t3 - t2 > 10000)
{
Console.WriteLine("{0} {1} {2}", f, t2 - t1, t3 - t2);
Console.WriteLine("{0} {1} {2} {3}", task.TaskId, task.Type, task.Url.GetHashCode(), task.Url);
}
}
protected override ContentProcessResult Process(TaskData td)
{
Task task = td.Task;
byte[] bytes = td.Bytes;
AmazonTaskType att = (AmazonTaskType)task.Type;
ContentProcessResult result = new ContentProcessResult();
//set to true, but could fail
result.Success = true;
if (att == AmazonTaskType.IMAGE
//|| att == AmazonTaskType.IMAGE_ORI
)
{
//DoImage();
}
else
{
string html = GetString(bytes, null);
if (!Validate(html))
{
result.Success = false;
return(result);
}
switch (att)
{
case AmazonTaskType.INDEX:
DoIndex(task, html, result);
break;
case AmazonTaskType.CATEGORY:
// DoCategory(task, html, result);
break;
case AmazonTaskType.MORE_CATEGORY:
DoMoreCategory(task, html, result);
break;
case AmazonTaskType.PAGES:
DoPages(task, html, result, null);
break;
case AmazonTaskType.PAGE:
DoPage(task, html, result);
break;
}
}
return(result);
}
private void button4_Click(object sender, EventArgs e)
{
string s = comboBox1.SelectedItem as string;
AmazonTaskType att = (AmazonTaskType)Enum.Parse(typeof(AmazonTaskType), s);
if (!string.IsNullOrEmpty(txtNewTask.Text) && txtNewTask.Text.StartsWith("http://"))
{
ins.sqlServerTaskPool.ProduceTask(new Task
{
Url = txtNewTask.Text,
Type = (int)att,
});
}
UpdateStatistics();
}
private Task MakeTask(string path, AmazonTaskType taskType, string context)
{
Task t = new Task();
if (!path.StartsWith("http://"))
{
//relative path
t.Url = URL_PREFIX + path;
}
else
{
//absolute path
t.Url = path;
}
t.Url = RemoveQid.Replace(t.Url, "");
t.Type = (int)taskType;
t.Context = context;
return(t);
}
