public void Request(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
{
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
}
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
int failed = 0;
int processed = 0;
try
{
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
string id = document.Value.GetPolarity("MechanicalTurk");
double count = document.Value.GetScore("MechanicalTurk");
count = (double.IsNaN(count)) ? 0 : count;
if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
{
continue;
}
IList <Assignment> assignments = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(client.GetAssignmentsForHIT(id, 1, true));
}), null, out time) as IList <Assignment>;
Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
}
else
{
assignments = client.GetAssignmentsForHIT(id, 1, true);
}
if (assignments.Count < count)
{
processed++;
continue;
}
double confidence = 0;
IList <string> answers = DeserializeAnswers(assignments);
string polarity = MergeSentimentPolarity(answers, out confidence);
processed++;
document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
{
break;
}
}
}
catch (Exception ex)
{
failed++;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
return;
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
{
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
}
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
List <QualificationRequirement> requirements = new List <QualificationRequirement>();
QualificationRequirement sucRequirement = new QualificationRequirement();
sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
sucRequirement.IntegerValueSpecified = true;
sucRequirement.IntegerValue = settings.PercentOfSuccess;
sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
requirements.Add(sucRequirement);
if (settings.Locale != null)
{
QualificationRequirement qualReq = new QualificationRequirement();
qualReq.LocaleValue = new Locale()
{
Country = settings.Locale
};
qualReq.Comparator = Comparator.EqualTo;
qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
requirements.Add(qualReq);
}
string hitType = string.Empty;
try
{
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements));
}), null, out time) as string;
Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
}
else
{
hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);
}
NotificationSpecification notification = new NotificationSpecification();
notification.Transport = NotificationTransport.Email;
notification.EventType = new EventType[] { EventType.AssignmentReturned };
notification.Destination = settings.Email;
notification.Version = "2006-05-05";
if (settings.Email != null)
{
client.SetHITTypeNotification(hitType, notification, true);
}
else
{
notification.Destination = "*****@*****.**";
client.SetHITTypeNotification(hitType, notification, false);
}
}
catch (Exception ex)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
return;
}
}
string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
string template = File.ReadAllText(questionFile);
QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
formTemplate.Question[0].QuestionIdentifier = document.Key;
formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
string question = QuestionUtil.SerializeQuestionForm(formTemplate);
HIT hit = new HIT();
hit.Expiration = DateTime.Now.AddDays(1);
hit.ExpirationSpecified = true;
hit.HITGroupId = "SentimentAnalysis";
hit.HITTypeId = hitType;
hit.MaxAssignments = settings.Assignments;
hit.MaxAssignmentsSpecified = true;
hit.Question = question;
HIT serverHit = null;
try
{
processed++;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(client.CreateHIT(hit));
}), null, out time) as HIT;
Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
}
else
{
serverHit = client.CreateHIT(hit);
}
document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
{
break;
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("MechanicalTurk", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
List<QualificationRequirement> requirements = new List<QualificationRequirement>();
QualificationRequirement sucRequirement = new QualificationRequirement();
sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
sucRequirement.IntegerValueSpecified = true;
sucRequirement.IntegerValue = settings.PercentOfSuccess;
sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
requirements.Add(sucRequirement);
if (settings.Locale != null)
{
QualificationRequirement qualReq = new QualificationRequirement();
qualReq.LocaleValue = new Locale() { Country = settings.Locale };
qualReq.Comparator = Comparator.EqualTo;
qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
requirements.Add(qualReq);
}
string hitType = string.Empty;
try
{
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements);
}), null, out time) as string;
Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
}
else
hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);
NotificationSpecification notification = new NotificationSpecification();
notification.Transport = NotificationTransport.Email;
notification.EventType = new EventType[] { EventType.AssignmentReturned };
notification.Destination = settings.Email;
notification.Version = "2006-05-05";
if (settings.Email != null)
client.SetHITTypeNotification(hitType, notification, true);
else
{
notification.Destination = "*****@*****.**";
client.SetHITTypeNotification(hitType, notification, false);
}
}
catch (Exception ex)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
return;
}
string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
string template = File.ReadAllText(questionFile);
QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
formTemplate.Question[0].QuestionIdentifier = document.Key;
formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
string question = QuestionUtil.SerializeQuestionForm(formTemplate);
HIT hit = new HIT();
hit.Expiration = DateTime.Now.AddDays(1);
hit.ExpirationSpecified = true;
hit.HITGroupId = "SentimentAnalysis";
hit.HITTypeId = hitType;
hit.MaxAssignments = settings.Assignments;
hit.MaxAssignmentsSpecified = true;
hit.Question = question;
HIT serverHit = null;
try
{
processed++;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return client.CreateHIT(hit);
}), null, out time) as HIT;
Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
}
else
serverHit = client.CreateHIT(hit);
document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
break;
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("MechanicalTurk", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
private void btProcess_Click(object sender, EventArgs e)
{
if (CancelIfNecessary() || !VerifyInput())
return;
try
{
ReadTheSource(tbSource.Text);
}
catch (Exception ex)
{
MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
return;
}
btProcess.Text = "Cancel";
btClose.Enabled = false;
_cancelFlag = false;
AnalysisExecutionContext context = new AnalysisExecutionContext(_documents);
context.ExecutionProgress += ExecutionProgress;
context.Key = tbMechanicalTurkKey.Text;
context.Secret = tbMechanicalTurkSecret.Text;
context.Language = cbLanguage.Text;
MechanicalTurkSettings settings = new MechanicalTurkSettings(null,
int.Parse(cbMechanicalTurkAssignments.Text), decimal.Parse(tbMechanicalTurkReward.Text),
(int.Parse(tbMechanicalTurkTime.Text) * 60), (int.Parse(tbMechanicalTurkApprove.Text) * 60),
int.Parse(tbMechanicalTurkPercent.Text));
if (cbMechanicalTurkNotification.Checked)
settings.Email = tbMechanicalTurkEmail.Text;
if (cbMechanicalTurkLocale.SelectedIndex > -1)
settings.Locale = LocaleHelper.GetCountryAbbreviation(cbMechanicalTurkLocale.SelectedItem as string);
context.CustomField = settings;
if (UseDebugMode && Benchmark)
context.UseDebugMode = true;
WriteDebugInfo("Running the dedicated thread for MechanicalTurk context serving...");
bool isFirstRun = !_documents.First().Value.GetServices().Contains("MechanicalTurk");
MechanicalTurkExecutor executor = new MechanicalTurkExecutor();
ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object obj)
{
if (isFirstRun)
executor.Execute(context);
else
executor.Request(context);
}), null);
}
public void Request(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
int failed = 0;
int processed = 0;
try
{
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
string id = document.Value.GetPolarity("MechanicalTurk");
double count = document.Value.GetScore("MechanicalTurk");
count = (double.IsNaN(count)) ? 0 : count;
if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
continue;
IList<Assignment> assignments = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return client.GetAssignmentsForHIT(id, 1, true);
}), null, out time) as IList<Assignment>;
Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
}
else
assignments = client.GetAssignmentsForHIT(id, 1, true);
if (assignments.Count < count)
{
processed++;
continue;
}
double confidence = 0;
IList<string> answers = DeserializeAnswers(assignments);
string polarity = MergeSentimentPolarity(answers, out confidence);
processed++;
document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
break;
}
}
catch (Exception ex)
{
failed++;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
return;
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}