public void Request(AnalysisExecutionContext context)
        {
            _context = context;

            if (context.Results.Count <= 0)
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;

            if (settings == null)
            {
                settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
            }

            string key    = context.Key;
            string secret = context.Secret;

            MTurkConfig  config = new MTurkConfig(_serviceURL, key, secret);
            SimpleClient client = new SimpleClient(config);

            int failed    = 0;
            int processed = 0;

            try
            {
                foreach (KeyValuePair <string, ResultSet> document in context.Results)
                {
                    string id    = document.Value.GetPolarity("MechanicalTurk");
                    double count = document.Value.GetScore("MechanicalTurk");
                    count = (double.IsNaN(count)) ? 0 : count;
                    if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
                    {
                        continue;
                    }

                    IList <Assignment> assignments = null;
                    if (context.UseDebugMode)
                    {
                        TimeSpan time = TimeSpan.Zero;
                        assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                        {
                            return(client.GetAssignmentsForHIT(id, 1, true));
                        }), null, out time) as IList <Assignment>;
                        Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
                    }
                    else
                    {
                        assignments = client.GetAssignmentsForHIT(id, 1, true);
                    }

                    if (assignments.Count < count)
                    {
                        processed++;
                        continue;
                    }

                    double         confidence = 0;
                    IList <string> answers    = DeserializeAnswers(assignments);
                    string         polarity   = MergeSentimentPolarity(answers, out confidence);

                    processed++;
                    document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
                    document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
                    AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
                    context.OnExecutionProgress("MechanicalTurk", e);

                    if (e.Cancel)
                    {
                        break;
                    }
                }
            }
            catch (Exception ex)
            {
                failed++;
                AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
                ea.Reason = ex.Message;
                context.OnExecutionProgress("MechanicalTurk", ea);

                if (ea.Cancel)
                {
                    return;
                }
            }

            context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
        }
        public void Execute(AnalysisExecutionContext context)
        {
            _context = context;

            if (context.Results.Count <= 0)
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;

            if (settings == null)
            {
                settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
            }

            string key    = context.Key;
            string secret = context.Secret;

            MTurkConfig  config = new MTurkConfig(_serviceURL, key, secret);
            SimpleClient client = new SimpleClient(config);

            List <QualificationRequirement> requirements   = new List <QualificationRequirement>();
            QualificationRequirement        sucRequirement = new QualificationRequirement();

            sucRequirement.QualificationTypeId   = MTurkSystemQualificationTypes.ApprovalRateQualification;
            sucRequirement.IntegerValueSpecified = true;
            sucRequirement.IntegerValue          = settings.PercentOfSuccess;
            sucRequirement.Comparator            = Comparator.GreaterThanOrEqualTo;
            requirements.Add(sucRequirement);

            if (settings.Locale != null)
            {
                QualificationRequirement qualReq = new QualificationRequirement();
                qualReq.LocaleValue = new Locale()
                {
                    Country = settings.Locale
                };
                qualReq.Comparator          = Comparator.EqualTo;
                qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
                requirements.Add(qualReq);
            }

            string hitType = string.Empty;

            try
            {
                if (context.UseDebugMode)
                {
                    TimeSpan time = TimeSpan.Zero;
                    hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                    {
                        return(hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                                                                settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements));
                    }), null, out time) as string;
                    Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
                }
                else
                {
                    hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                                                     settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);
                }

                NotificationSpecification notification = new NotificationSpecification();
                notification.Transport   = NotificationTransport.Email;
                notification.EventType   = new EventType[] { EventType.AssignmentReturned };
                notification.Destination = settings.Email;
                notification.Version     = "2006-05-05";

                if (settings.Email != null)
                {
                    client.SetHITTypeNotification(hitType, notification, true);
                }
                else
                {
                    notification.Destination = "*****@*****.**";
                    client.SetHITTypeNotification(hitType, notification, false);
                }
            }
            catch (Exception ex)
            {
                AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
                ea.Reason = ex.Message;
                context.OnExecutionProgress("MechanicalTurk", ea);

                if (ea.Cancel)
                {
                    return;
                }
            }

            string       questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
            string       template     = File.ReadAllText(questionFile);
            QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);

            int processed = 0;
            int failed    = 0;

            foreach (KeyValuePair <string, ResultSet> document in context.Results)
            {
                formTemplate.Question[0].QuestionIdentifier       = document.Key;
                formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
                string question = QuestionUtil.SerializeQuestionForm(formTemplate);

                HIT hit = new HIT();
                hit.Expiration              = DateTime.Now.AddDays(1);
                hit.ExpirationSpecified     = true;
                hit.HITGroupId              = "SentimentAnalysis";
                hit.HITTypeId               = hitType;
                hit.MaxAssignments          = settings.Assignments;
                hit.MaxAssignmentsSpecified = true;
                hit.Question = question;

                HIT serverHit = null;
                try
                {
                    processed++;
                    if (context.UseDebugMode)
                    {
                        TimeSpan time = TimeSpan.Zero;
                        serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                        {
                            return(client.CreateHIT(hit));
                        }), null, out time) as HIT;
                        Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
                    }
                    else
                    {
                        serverHit = client.CreateHIT(hit);
                    }

                    document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
                    AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
                    context.OnExecutionProgress("MechanicalTurk", e);

                    if (e.Cancel)
                    {
                        break;
                    }
                }
                catch (Exception ex)
                {
                    failed++;
                    document.Value.AddOutput("MechanicalTurk", 0, "failed");
                    AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
                    ea.Reason = ex.Message;
                    context.OnExecutionProgress("MechanicalTurk", ea);

                    if (ea.Cancel)
                    {
                        break;
                    }
                }
            }

            context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
        }
        public void Execute(AnalysisExecutionContext context)
        {
            _context = context;

            if (context.Results.Count <= 0)
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
            if (settings == null)
                settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);

            string key = context.Key;
            string secret = context.Secret;

            MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
            SimpleClient client = new SimpleClient(config);

            List<QualificationRequirement> requirements = new List<QualificationRequirement>();
            QualificationRequirement sucRequirement = new QualificationRequirement();
            sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
            sucRequirement.IntegerValueSpecified = true;
            sucRequirement.IntegerValue = settings.PercentOfSuccess;
            sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
            requirements.Add(sucRequirement);

            if (settings.Locale != null)
            {
                QualificationRequirement qualReq = new QualificationRequirement();
                qualReq.LocaleValue = new Locale() { Country = settings.Locale };
                qualReq.Comparator = Comparator.EqualTo;
                qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
                requirements.Add(qualReq);
            }

            string hitType = string.Empty;
            try
            {
                if (context.UseDebugMode)
                {
                    TimeSpan time = TimeSpan.Zero;
                    hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                    {
                        return hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                            settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements);
                    }), null, out time) as string;
                    Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
                }
                else
                    hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                        settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);

                NotificationSpecification notification = new NotificationSpecification();
                notification.Transport = NotificationTransport.Email;
                notification.EventType = new EventType[] { EventType.AssignmentReturned };
                notification.Destination = settings.Email;
                notification.Version = "2006-05-05";

                if (settings.Email != null)
                    client.SetHITTypeNotification(hitType, notification, true);
                else
                {
                    notification.Destination = "*****@*****.**";
                    client.SetHITTypeNotification(hitType, notification, false);
                }
            }
            catch (Exception ex)
            {
                AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
                ea.Reason = ex.Message;
                context.OnExecutionProgress("MechanicalTurk", ea);

                if (ea.Cancel)
                    return;
            }

            string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
            string template = File.ReadAllText(questionFile);
            QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);

            int processed = 0;
            int failed = 0;
            foreach (KeyValuePair<string, ResultSet> document in context.Results)
            {
                formTemplate.Question[0].QuestionIdentifier = document.Key;
                formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
                string question = QuestionUtil.SerializeQuestionForm(formTemplate);

                HIT hit = new HIT();
                hit.Expiration = DateTime.Now.AddDays(1);
                hit.ExpirationSpecified = true;
                hit.HITGroupId = "SentimentAnalysis";
                hit.HITTypeId = hitType;
                hit.MaxAssignments = settings.Assignments;
                hit.MaxAssignmentsSpecified = true;
                hit.Question = question;

                HIT serverHit = null;
                try
                {
                    processed++;
                    if (context.UseDebugMode)
                    {
                        TimeSpan time = TimeSpan.Zero;
                        serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                        {
                            return client.CreateHIT(hit);
                        }), null, out time) as HIT;
                        Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
                    }
                    else
                        serverHit = client.CreateHIT(hit);

                    document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
                    AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
                    context.OnExecutionProgress("MechanicalTurk", e);

                    if (e.Cancel)
                        break;
                }
                catch (Exception ex)
                {
                    failed++;
                    document.Value.AddOutput("MechanicalTurk", 0, "failed");
                    AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
                    ea.Reason = ex.Message;
                    context.OnExecutionProgress("MechanicalTurk", ea);

                    if (ea.Cancel)
                        break;
                }
            }

            context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
        }
Exemple #4
0
        private void btProcess_Click(object sender, EventArgs e)
        {
            if (CancelIfNecessary() || !VerifyInput())
                return;

            try
            {
                ReadTheSource(tbSource.Text);
            }
            catch (Exception ex)
            {
                MessageBox.Show(ex.Message, "Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
                return;
            }

            btProcess.Text = "Cancel";
            btClose.Enabled = false;
            _cancelFlag = false;

            AnalysisExecutionContext context = new AnalysisExecutionContext(_documents);
            context.ExecutionProgress += ExecutionProgress;
            context.Key = tbMechanicalTurkKey.Text;
            context.Secret = tbMechanicalTurkSecret.Text;
            context.Language = cbLanguage.Text;

            MechanicalTurkSettings settings = new MechanicalTurkSettings(null,
                int.Parse(cbMechanicalTurkAssignments.Text), decimal.Parse(tbMechanicalTurkReward.Text),
                (int.Parse(tbMechanicalTurkTime.Text) * 60), (int.Parse(tbMechanicalTurkApprove.Text) * 60),
                int.Parse(tbMechanicalTurkPercent.Text));
            if (cbMechanicalTurkNotification.Checked)
                settings.Email = tbMechanicalTurkEmail.Text;
            if (cbMechanicalTurkLocale.SelectedIndex > -1)
                settings.Locale = LocaleHelper.GetCountryAbbreviation(cbMechanicalTurkLocale.SelectedItem as string);
            context.CustomField = settings;

            if (UseDebugMode && Benchmark)
                context.UseDebugMode = true;

            WriteDebugInfo("Running the dedicated thread for MechanicalTurk context serving...");
            bool isFirstRun = !_documents.First().Value.GetServices().Contains("MechanicalTurk");
            MechanicalTurkExecutor executor = new MechanicalTurkExecutor();

            ThreadPool.QueueUserWorkItem(new WaitCallback(delegate(object obj)
            {
                if (isFirstRun)
                    executor.Execute(context);
                else
                    executor.Request(context);
            }), null);
        }
        public void Request(AnalysisExecutionContext context)
        {
            _context = context;

            if (context.Results.Count <= 0)
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
            if (settings == null)
                settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);

            string key = context.Key;
            string secret = context.Secret;

            MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
            SimpleClient client = new SimpleClient(config);

            int failed = 0;
            int processed = 0;
            try
            {
                foreach (KeyValuePair<string, ResultSet> document in context.Results)
                {
                    string id = document.Value.GetPolarity("MechanicalTurk");
                    double count = document.Value.GetScore("MechanicalTurk");
                    count = (double.IsNaN(count)) ? 0 : count;
                    if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
                        continue;

                    IList<Assignment> assignments = null;
                    if (context.UseDebugMode)
                    {
                        TimeSpan time = TimeSpan.Zero;
                        assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                        {
                            return client.GetAssignmentsForHIT(id, 1, true);
                        }), null, out time) as IList<Assignment>;
                        Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
                    }
                    else
                        assignments = client.GetAssignmentsForHIT(id, 1, true);

                    if (assignments.Count < count)
                    {
                        processed++;
                        continue;
                    }

                    double confidence = 0;
                    IList<string> answers = DeserializeAnswers(assignments);
                    string polarity = MergeSentimentPolarity(answers, out confidence);

                    processed++;
                    document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
                    document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
                    AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
                    context.OnExecutionProgress("MechanicalTurk", e);

                    if (e.Cancel)
                        break;
                }
            }
            catch (Exception ex)
            {
                failed++;
                AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
                ea.Reason = ex.Message;
                context.OnExecutionProgress("MechanicalTurk", ea);

                if (ea.Cancel)
                    return;
            }

            context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
        }