Exemplo n.º 1
0
        /// <summary>
        /// Create a HIT from a stored template.
        /// </summary>
        /// <param name="hitId">The Hit Id</param>
        public void CreateHit(int hitId)
        {
            using (var ctx = new TurkRContext())
            {
                var hit = ctx.Hits.FirstOrDefault(h => h.HitId == hitId);
                if (hit != null)
                {
                    SimpleClient myClient = new SimpleClient(_mturkConfig);

                    // Setup qualifications
                    List<QualificationRequirement> qualifications = new List<QualificationRequirement>();

                    // First register the HitType
                    string hitTypeId = myClient.RegisterHITType(hit.Title, hit.Description, (long)hit.AutoApprovalDelay, (long)hit.Duration, (decimal)hit.Reward, hit.Keywords, qualifications);

                    // Define the external question
                    ExternalQuestion eq = new ExternalQuestion();
                    eq.ExternalURL = "https://www.descil.ethz.ch/apps/turkr/start/" + hit.HitCode;
                    eq.FrameHeight = "800";

                    // More definitions
                    string requesterAnnotation = hit.HitCode;
                    string[] responseGroup = null;

                    // Creat the HIT
                    var amtHit = myClient.CreateHIT(hitTypeId, hit.Title, hit.Description, hit.Keywords, eq, (decimal)hit.Reward, (long)hit.Duration, (long)hit.AutoApprovalDelay, (long)hit.Lifetime, hit.Assignments, requesterAnnotation, qualifications, responseGroup);

                    hit.AmtHitId = amtHit.HITId;
                    hit.CreationTime = DateTime.UtcNow;
                    ctx.SaveChanges();
                }
            }
        }
Exemplo n.º 2
0
        public void Execute(AnalysisExecutionContext context)
        {
            _context = context;

            if (context.Results.Count <= 0)
            {
                context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
                return;
            }

            MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
            if (settings == null)
                settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);

            string key = context.Key;
            string secret = context.Secret;

            MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
            SimpleClient client = new SimpleClient(config);

            List<QualificationRequirement> requirements = new List<QualificationRequirement>();
            QualificationRequirement sucRequirement = new QualificationRequirement();
            sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
            sucRequirement.IntegerValueSpecified = true;
            sucRequirement.IntegerValue = settings.PercentOfSuccess;
            sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
            requirements.Add(sucRequirement);

            if (settings.Locale != null)
            {
                QualificationRequirement qualReq = new QualificationRequirement();
                qualReq.LocaleValue = new Locale() { Country = settings.Locale };
                qualReq.Comparator = Comparator.EqualTo;
                qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
                requirements.Add(qualReq);
            }

            string hitType = string.Empty;
            try
            {
                if (context.UseDebugMode)
                {
                    TimeSpan time = TimeSpan.Zero;
                    hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                    {
                        return hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                            settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements);
                    }), null, out time) as string;
                    Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
                }
                else
                    hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
                        settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);

                NotificationSpecification notification = new NotificationSpecification();
                notification.Transport = NotificationTransport.Email;
                notification.EventType = new EventType[] { EventType.AssignmentReturned };
                notification.Destination = settings.Email;
                notification.Version = "2006-05-05";

                if (settings.Email != null)
                    client.SetHITTypeNotification(hitType, notification, true);
                else
                {
                    notification.Destination = "*****@*****.**";
                    client.SetHITTypeNotification(hitType, notification, false);
                }
            }
            catch (Exception ex)
            {
                AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
                ea.Reason = ex.Message;
                context.OnExecutionProgress("MechanicalTurk", ea);

                if (ea.Cancel)
                    return;
            }

            string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
            string template = File.ReadAllText(questionFile);
            QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);

            int processed = 0;
            int failed = 0;
            foreach (KeyValuePair<string, ResultSet> document in context.Results)
            {
                formTemplate.Question[0].QuestionIdentifier = document.Key;
                formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
                string question = QuestionUtil.SerializeQuestionForm(formTemplate);

                HIT hit = new HIT();
                hit.Expiration = DateTime.Now.AddDays(1);
                hit.ExpirationSpecified = true;
                hit.HITGroupId = "SentimentAnalysis";
                hit.HITTypeId = hitType;
                hit.MaxAssignments = settings.Assignments;
                hit.MaxAssignmentsSpecified = true;
                hit.Question = question;

                HIT serverHit = null;
                try
                {
                    processed++;
                    if (context.UseDebugMode)
                    {
                        TimeSpan time = TimeSpan.Zero;
                        serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
                        {
                            return client.CreateHIT(hit);
                        }), null, out time) as HIT;
                        Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
                    }
                    else
                        serverHit = client.CreateHIT(hit);

                    document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
                    AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
                    context.OnExecutionProgress("MechanicalTurk", e);

                    if (e.Cancel)
                        break;
                }
                catch (Exception ex)
                {
                    failed++;
                    document.Value.AddOutput("MechanicalTurk", 0, "failed");
                    AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
                    ea.Reason = ex.Message;
                    context.OnExecutionProgress("MechanicalTurk", ea);

                    if (ea.Cancel)
                        break;
                }
            }

            context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
        }