private void session_Error(object sender, ResponseErrorEventArgs e)
{
if (_context.UseDebugMode)
{
Console.WriteLine("\tSemantria executor got an error with {0} status code. The reason is: {1}", e.Status, e.Message);
}
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, 0, 0);
ea.Reason = e.Message;
_context.OnExecutionProgress("Semantria", ea);
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Viralheat", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
if (document.Value.Source.Length > 360)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
{
break;
}
continue;
}
StringBuilder builder = new StringBuilder();
builder.Append("http://www.viralheat.com/api/sentiment/review.json?");
builder.AppendFormat("api_key={0}&", context.Key);
builder.AppendFormat("text={0}", HttpUtility.UrlEncode(document.Value.Source));
try
{
HttpWebResponse response = null;
WebRequest request = WebRequest.Create(builder.ToString());
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(request.GetResponse());
}), null, out time) as HttpWebResponse;
Console.WriteLine("\tViralheat: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
{
response = request.GetResponse() as HttpWebResponse;
}
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.OK)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
response.Close();
if (ea.Cancel)
{
break;
}
}
else
{
using (Stream stream = response.GetResponseStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(ViralheatSentiment));
ViralheatSentiment sentiment = (ViralheatSentiment)serializer.ReadObject(stream);
processed++;
document.Value.AddOutput("Viralheat", sentiment.Prob, sentiment.Mood);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
{
break;
}
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("Viralheat", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Chatterbox", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
if (document.Value.Source.Length > 300)
{
failed++;
document.Value.AddOutput("Chatterbox", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Chatterbox", ea);
if (ea.Cancel)
{
break;
}
continue;
}
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("lang", LocaleHelper.GetDoubleLanguageAbbreviation(context.Language));
parameters.Add("text", Encoding.UTF8.GetString(Encoding.UTF8.GetBytes(document.Value.Source)));
WebRequest request = WebRequest.Create("https://chatterbox-analytics-sentiment-analysis-free.p.mashape.com/sentiment/current/classify_text/");
request.Headers.Add("X-Mashape-Authorization", context.Key);
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
using (StreamWriter writer = new StreamWriter(request.GetRequestStream()))
{
writer.Write(FormatParameters(parameters));
writer.Flush();
}
try
{
HttpWebResponse response = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(request.GetResponse());
}), null, out time) as HttpWebResponse;
Console.WriteLine("\tChatterbox: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
{
response = request.GetResponse() as HttpWebResponse;
}
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.OK)
{
failed++;
document.Value.AddOutput("Chatterbox", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Chatterbox", ea);
response.Close();
if (ea.Cancel)
{
break;
}
}
else
{
using (Stream stream = response.GetResponseStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(ChatterboxSentiment));
ChatterboxSentiment sentiment = (ChatterboxSentiment)serializer.ReadObject(stream);
processed++;
string polarity = GetSentimentPolarity(sentiment.Value);
document.Value.AddOutput("Chatterbox", sentiment.Value, polarity);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Chatterbox", ea);
if (ea.Cancel)
{
break;
}
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Chatterbox", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Chatterbox", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("Chatterbox", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Viralheat", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
if (document.Value.Source.Length > 360)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
break;
continue;
}
StringBuilder builder = new StringBuilder();
builder.Append("http://www.viralheat.com/api/sentiment/review.json?");
builder.AppendFormat("api_key={0}&", context.Key);
builder.AppendFormat("text={0}", HttpUtility.UrlEncode(document.Value.Source));
try
{
HttpWebResponse response = null;
WebRequest request = WebRequest.Create(builder.ToString());
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return request.GetResponse();
}), null, out time) as HttpWebResponse;
Console.WriteLine("\tViralheat: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
response = request.GetResponse() as HttpWebResponse;
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.OK)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
response.Close();
if (ea.Cancel)
break;
}
else
{
using (Stream stream = response.GetResponseStream())
{
DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(ViralheatSentiment));
ViralheatSentiment sentiment = (ViralheatSentiment)serializer.ReadObject(stream);
processed++;
document.Value.AddOutput("Viralheat", sentiment.Prob, sentiment.Mood);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
break;
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Viralheat", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Viralheat", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("Viralheat", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Bitext", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
if (document.Value.Source.Length > 8192)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
break;
continue;
}
Dictionary<string, string> parameters = new Dictionary<string, string>();
parameters.Add("User", context.Key);
parameters.Add("Pass", context.Secret);
parameters.Add("OutFormat", context.Format.ToString());
parameters.Add("Detail", "Global");
parameters.Add("Normalized", "No");
parameters.Add("Theme", "Gen");
parameters.Add("ID", document.Key);
parameters.Add("Lang", LocaleHelper.GetTripleLanguageAbbreviation(context.Language));
parameters.Add("Text", HttpUtility.UrlEncode(document.Value.Source));
byte[] data = Encoding.UTF8.GetBytes(FormatParameters(parameters));
WebRequest request = WebRequest.Create("http://svc9.bitext.com/WS_NOps_Val/Service.aspx");
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
request.ContentLength = data.Length;
using (Stream writer = request.GetRequestStream())
{
writer.Write(data, 0, data.Length);
writer.Flush();
}
try
{
HttpWebResponse response = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return request.GetResponse();
}), null, out time) as HttpWebResponse;
Console.WriteLine("Bitext: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
response = request.GetResponse() as HttpWebResponse;
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.OK)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
response.Close();
if (ea.Cancel)
break;
}
else
{
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
string result = reader.ReadToEnd();
result = result.Replace("\r\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\n", string.Empty)
.Replace(">\"", ">")
.Replace("\"<", "<")
.Replace("&", "&");
Regex regex = new Regex(@"(?<=\bencoding="")[^""]*");
Match match = regex.Match(result);
Encoding encoding = null;
if (match.Success)
encoding = Encoding.GetEncoding(match.Value);
else
encoding = Encoding.UTF8;
BitextSentiment sentiment = null;
using (Stream stream = new MemoryStream(encoding.GetBytes(result)))
{
XmlSerializer serializer = new XmlSerializer(typeof(BitextSentiment));
sentiment = (BitextSentiment)serializer.Deserialize(stream);
}
processed++;
double score = MergeSentimentScore(sentiment);
string polarity = GetSentimentPolarity(score);
document.Value.AddOutput("Bitext", score, polarity);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
break;
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("Bitext", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Repustate", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
bool isTrial = false;
try
{
string lang = LocaleHelper.GetDoubleLanguageAbbreviation(context.Language);
RepustateClient Repustate = new RepustateClient(context.Key);
Dictionary<string, string> TotalScoreDataMap = new Dictionary<string, string>();
int count = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
count++;
string ky = "text" + count;
TotalScoreDataMap.Add(ky, document.Value.ToString());
}
int BatchSize = 500;
int processedBatches = 0;
if (TotalScoreDataMap.Count < BatchSize)
{
Dictionary<string, string> scoreDataMap = new Dictionary<string, string>();
scoreDataMap = Repustate.GetDocumentsQueue(processedBatches, BatchSize, TotalScoreDataMap);
scoreDataMap.Add("lang", lang);
JavaScriptSerializer serializer = new JavaScriptSerializer();
Array RepustateSentiments;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
RepustateSentiments = (Array)BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary<string, object> deserializedResponse = serializer.Deserialize<IDictionary<string, object>>(response) as IDictionary<string, object>;
return deserializedResponse["results"] as Array;
}), null, out time);
Console.WriteLine("\tRepustate: Batch of {0} documents has been recieved. Eexecution time is: {1} ms", scoreDataMap.Count - 1, time.TotalMilliseconds);
}
else
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary<string, object> deserializedResponse = serializer.Deserialize<IDictionary<string, object>>(response) as IDictionary<string, object>;
RepustateSentiments = deserializedResponse["results"] as Array;
}
SortedDictionary<int, string> rs = new SortedDictionary<int, string>();
for (int i = 0; i < RepustateSentiments.Length; i++)
{
int id = Int32.Parse(((Dictionary<string, object>)RepustateSentiments.GetValue(i))["id"].ToString());
string score = ((Dictionary<string, object>)RepustateSentiments.GetValue(i))["score"].ToString();
rs.Add(id, score);
}
int c = 1;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
double score = double.Parse(rs[c]);
processed++;
if (score <= -0.05)
document.Value.AddOutput("Repustate", score, "negative");
if (score >= 0.05)
document.Value.AddOutput("Repustate", score, "positive");
else
document.Value.AddOutput("Repustate", score, "neutral");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Repustate", ea);
c++;
if (ea.Cancel)
break;
}
}
else
{
int totalBatches = (TotalScoreDataMap.Count / BatchSize) + 1;
processedBatches = 0;
for (int i = 0; i < totalBatches; i++)
{
Dictionary<string, string> scoreDataMap = new Dictionary<string, string>();
scoreDataMap = Repustate.GetDocumentsQueue(processedBatches, BatchSize, TotalScoreDataMap);
scoreDataMap.Add("lang", lang);
JavaScriptSerializer serializer = new JavaScriptSerializer();
Array RepustateSentiments;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
RepustateSentiments = (Array)BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary<string, object> deserializedResponse = serializer.Deserialize<IDictionary<string, object>>(response) as IDictionary<string, object>;
return deserializedResponse["results"] as Array;
}), null, out time);
Console.WriteLine("\tRepustate: Batch of {0} documents has been recieved. Eexecution time is: {1} ms", scoreDataMap.Count - 1, time.TotalMilliseconds);
}
else
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary<string, object> deserializedResponse = serializer.Deserialize<IDictionary<string, object>>(response) as IDictionary<string, object>;
RepustateSentiments = deserializedResponse["results"] as Array;
}
SortedDictionary<int, string> rs = new SortedDictionary<int, string>();
for (int j = 0; j < RepustateSentiments.Length; j++)
{
int id = Int32.Parse(((Dictionary<string, object>)RepustateSentiments.GetValue(j))["id"].ToString());
string score = ((Dictionary<string, object>)RepustateSentiments.GetValue(j))["score"].ToString();
rs.Add(id, score);
}
int key = 1;
int UpperCounter = processedBatches;
int LowerCounter = BatchSize;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
if (UpperCounter > 0)
{
key++;
UpperCounter--;
continue;
}
if (LowerCounter < 1)
break;
LowerCounter--;
double score = double.Parse(rs[key]);
processed++;
if (score <= -0.05)
document.Value.AddOutput("Repustate", score, "negative");
if (score >= 0.05)
document.Value.AddOutput("Repustate", score, "positive");
else
document.Value.AddOutput("Repustate", score, "neutral");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Repustate", ea);
key++;
if (ea.Cancel)
break;
}
processedBatches = processedBatches + BatchSize;
}
}
}
catch (WebException ex)
{
if (!(ex.Response.ContentLength == -1))
{
isTrial = true;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, context.Results.Count, 0, 0);
ea.Reason = "Your Repustate account doesn’t support " + context.Language + " language";
context.OnExecutionProgress("Repustate", ea);
}
else
{
isTrial = false;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
failed++;
document.Value.AddOutput("Repustate", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Repustate", ea);
if (ea.Cancel)
break;
}
}
}
if(!isTrial)
context.OnExecutionProgress("Repustate", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Bitext", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
if (document.Value.Source.Length > 8192)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
{
break;
}
continue;
}
Dictionary <string, string> parameters = new Dictionary <string, string>();
parameters.Add("User", context.Key);
parameters.Add("Pass", context.Secret);
parameters.Add("OutFormat", context.Format.ToString());
parameters.Add("Detail", "Global");
parameters.Add("Normalized", "No");
parameters.Add("Theme", "Gen");
parameters.Add("ID", document.Key);
parameters.Add("Lang", LocaleHelper.GetTripleLanguageAbbreviation(context.Language));
parameters.Add("Text", HttpUtility.UrlEncode(document.Value.Source));
byte[] data = Encoding.UTF8.GetBytes(FormatParameters(parameters));
WebRequest request = WebRequest.Create("http://svc9.bitext.com/WS_NOps_Val/Service.aspx");
request.ContentType = "application/x-www-form-urlencoded";
request.Method = "POST";
request.ContentLength = data.Length;
using (Stream writer = request.GetRequestStream())
{
writer.Write(data, 0, data.Length);
writer.Flush();
}
try
{
HttpWebResponse response = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(request.GetResponse());
}), null, out time) as HttpWebResponse;
Console.WriteLine("Bitext: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
{
response = request.GetResponse() as HttpWebResponse;
}
if (response.StatusCode != HttpStatusCode.Accepted && response.StatusCode != HttpStatusCode.OK)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
response.Close();
if (ea.Cancel)
{
break;
}
}
else
{
using (StreamReader reader = new StreamReader(response.GetResponseStream()))
{
string result = reader.ReadToEnd();
result = result.Replace("\r\n", string.Empty)
.Replace("\r", string.Empty)
.Replace("\n", string.Empty)
.Replace(">\"", ">")
.Replace("\"<", "<")
.Replace("&", "&");
Regex regex = new Regex(@"(?<=\bencoding="")[^""]*");
Match match = regex.Match(result);
Encoding encoding = null;
if (match.Success)
{
encoding = Encoding.GetEncoding(match.Value);
}
else
{
encoding = Encoding.UTF8;
}
BitextSentiment sentiment = null;
using (Stream stream = new MemoryStream(encoding.GetBytes(result)))
{
XmlSerializer serializer = new XmlSerializer(typeof(BitextSentiment));
sentiment = (BitextSentiment)serializer.Deserialize(stream);
}
processed++;
double score = MergeSentimentScore(sentiment);
string polarity = GetSentimentPolarity(score);
document.Value.AddOutput("Bitext", score, polarity);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
{
break;
}
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Bitext", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Bitext", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("Bitext", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
List<QualificationRequirement> requirements = new List<QualificationRequirement>();
QualificationRequirement sucRequirement = new QualificationRequirement();
sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
sucRequirement.IntegerValueSpecified = true;
sucRequirement.IntegerValue = settings.PercentOfSuccess;
sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
requirements.Add(sucRequirement);
if (settings.Locale != null)
{
QualificationRequirement qualReq = new QualificationRequirement();
qualReq.LocaleValue = new Locale() { Country = settings.Locale };
qualReq.Comparator = Comparator.EqualTo;
qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
requirements.Add(qualReq);
}
string hitType = string.Empty;
try
{
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements);
}), null, out time) as string;
Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
}
else
hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);
NotificationSpecification notification = new NotificationSpecification();
notification.Transport = NotificationTransport.Email;
notification.EventType = new EventType[] { EventType.AssignmentReturned };
notification.Destination = settings.Email;
notification.Version = "2006-05-05";
if (settings.Email != null)
client.SetHITTypeNotification(hitType, notification, true);
else
{
notification.Destination = "*****@*****.**";
client.SetHITTypeNotification(hitType, notification, false);
}
}
catch (Exception ex)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
return;
}
string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
string template = File.ReadAllText(questionFile);
QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
formTemplate.Question[0].QuestionIdentifier = document.Key;
formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
string question = QuestionUtil.SerializeQuestionForm(formTemplate);
HIT hit = new HIT();
hit.Expiration = DateTime.Now.AddDays(1);
hit.ExpirationSpecified = true;
hit.HITGroupId = "SentimentAnalysis";
hit.HITTypeId = hitType;
hit.MaxAssignments = settings.Assignments;
hit.MaxAssignmentsSpecified = true;
hit.Question = question;
HIT serverHit = null;
try
{
processed++;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return client.CreateHIT(hit);
}), null, out time) as HIT;
Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
}
else
serverHit = client.CreateHIT(hit);
document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
break;
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("MechanicalTurk", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Alchemy", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
AlchemyAPI.AlchemyAPI session = new AlchemyAPI.AlchemyAPI();
session.SetAPIKey(context.Key);
AlchemyAPI_TargetedSentimentParams parameters = new AlchemyAPI_TargetedSentimentParams();
parameters.setOutputMode(AlchemyAPI_BaseParams.OutputMode.XML);
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
try
{
string strResult = string.Empty;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
strResult = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return session.TextGetTextSentiment(document.Value.Source, parameters);
}), null, out time) as string;
Console.WriteLine("\tAlchemyAPI: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
strResult = session.TextGetTextSentiment(document.Value.Source, parameters);
using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(strResult)))
{
XmlSerializer serializer = new XmlSerializer(typeof(AlchemySentimentResult));
AlchemySentimentResult result = (AlchemySentimentResult)serializer.Deserialize(stream);
processed++;
document.Value.AddOutput("Alchemy", result.SentimentDetails.Score, result.SentimentDetails.Type);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Alchemy", ea);
if (ea.Cancel)
break;
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Alchemy", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Alchemy", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("Alchemy", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Alchemy", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
AlchemyAPI.AlchemyAPI session = new AlchemyAPI.AlchemyAPI();
session.SetAPIKey(context.Key);
AlchemyAPI_TargetedSentimentParams parameters = new AlchemyAPI_TargetedSentimentParams();
parameters.setOutputMode(AlchemyAPI_BaseParams.OutputMode.XML);
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
try
{
string strResult = string.Empty;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
strResult = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(session.TextGetTextSentiment(document.Value.Source, parameters));
}), null, out time) as string;
Console.WriteLine("\tAlchemyAPI: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
{
strResult = session.TextGetTextSentiment(document.Value.Source, parameters);
}
using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(strResult)))
{
XmlSerializer serializer = new XmlSerializer(typeof(AlchemySentimentResult));
AlchemySentimentResult result = (AlchemySentimentResult)serializer.Deserialize(stream);
processed++;
document.Value.AddOutput("Alchemy", result.SentimentDetails.Score, result.SentimentDetails.Type);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Alchemy", ea);
if (ea.Cancel)
{
break;
}
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Alchemy", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Alchemy", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("Alchemy", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Skyttle", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
try {
Dictionary <string, string> scoreDataMap = new Dictionary <string, string>();
scoreDataMap.Add("text", document.Value.ToString());
string url = "https://sentinelprojects-skyttle20.p.mashape.com/";
string text = document.Value.ToString();
string lang = LocaleHelper.GetDoubleLanguageAbbreviation(context.Language);
string keywords = "1";
string sentiment = "1";
string annotate = "0";
string param = "text=" + HttpUtility.UrlEncode(text, System.Text.Encoding.UTF8) +
"&lang=" + HttpUtility.UrlEncode(lang, System.Text.Encoding.UTF8) +
"&keywords=" + HttpUtility.UrlEncode(keywords, System.Text.Encoding.UTF8) +
"&sentiment=" + HttpUtility.UrlEncode(sentiment, System.Text.Encoding.UTF8) +
"&annotate=" + HttpUtility.UrlEncode(annotate, System.Text.Encoding.UTF8);
string response = "";
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(MakeCall(url, param, context.Key));
}), null, out time) as string;
Console.WriteLine("\tSkyttle: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
{
response = MakeCall(url, param, context.Key);
}
JavaScriptSerializer serializer = new JavaScriptSerializer();
IDictionary <string, object> dict = serializer.Deserialize <IDictionary <string, object> >(response);
processed++;
IDictionary <string, object> obj = ((object[])dict["docs"])[0] as IDictionary <string, object>;
string language = obj["language"] as string;
IDictionary <string, object> sentiments = obj["sentiment_scores"] as IDictionary <string, object>;
decimal negScore = (decimal)sentiments["neg"];
decimal posScore = (decimal)sentiments["pos"];
decimal neuScore = (decimal)sentiments["neu"];
if (neuScore > 50)
{
document.Value.AddOutput("Skyttle", 0.0, "neutral");
}
else if (negScore > posScore)
{
document.Value.AddOutput("Skyttle", Convert.ToDouble(-1 * negScore / 100), "negative");
}
else if (posScore > negScore)
{
document.Value.AddOutput("Skyttle", Convert.ToDouble(posScore / 100), "positive");
}
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Skyttle", ea);
if (ea.Cancel)
{
break;
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Skyttle", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Skyttle", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("Skyttle", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Semantria", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
ISerializer serializer = new JsonSerializer();
if (context.Format == DataFormat.XML)
{
serializer = new XmlSerializer();
}
int index = 0;
int failed = 0;
try
{
using (Session session = Session.CreateSession(context.Key, context.Secret, serializer))
{
session.Error += session_Error;
Subscription subscription = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
subscription = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(session.GetSubscription());
}), null, out time) as Subscription;
Console.WriteLine("\tSemantria: Subscrition object has been obtained. Execution time is: {0} ms", time.TotalMilliseconds);
}
else
{
subscription = session.GetSubscription();
}
int docLimit = subscription.BasicSettings.CharactersLimit;
List <Document> documents = GetSourceData(context.Results, docLimit);
List <DocAnalyticData> output = new List <DocAnalyticData>(documents.Count);
int batchSize = subscription.BasicSettings.BatchLimit;
int respBatchLimit = subscription.BasicSettings.ProcessedBatchLimit;
Configuration config = GetOrCreateConfiguration(session, context.Language);
int pushProgress = 50 / ((documents.Count < batchSize) ? 1 : (documents.Count / batchSize));
batchSize = (documents.Count < batchSize) ? documents.Count : batchSize;
int progress = 0;
Dictionary <string, bool> queue = new Dictionary <string, bool>(documents.Count);
while (index < documents.Count)
{
int execResult = -1;
batchSize = ((index + batchSize) > documents.Count) ? (documents.Count % batchSize) : batchSize;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
execResult = (int)BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
List <Document> docs = documents.GetRange(index, batchSize);
foreach (Document doc in docs)
{
queue.Add(doc.Id, false);
}
return(session.QueueBatchOfDocuments(documents.GetRange(index, batchSize), config.ConfigId));
}), null, out time);
Console.WriteLine("\tSemantria: Batch of {0} documents has been queued. Eexecution time is: {1} ms", batchSize, time.TotalMilliseconds);
}
else
{
List <Document> docs = documents.GetRange(index, batchSize);
foreach (Document doc in docs)
{
queue.Add(doc.Id, false);
}
execResult = session.QueueBatchOfDocuments(documents.GetRange(index, batchSize), config.ConfigId);
}
progress = (queue.Count == 1) ? 1 : ((int)(queue.Count / 2));
if (execResult != -1)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, progress, failed);
context.OnExecutionProgress("Semantria", ea);
if (ea.Cancel)
{
break;
}
}
else
{
failed += batchSize;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, progress, failed);
context.OnExecutionProgress("Semantria", ea);
if (ea.Cancel)
{
break;
}
}
index += batchSize;
}
index = 0;
while (queue.Values.Contains(false))
{
Thread.Sleep(1000);
IList <DocAnalyticData> temp = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
temp = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(session.GetProcessedDocuments(config.ConfigId));
}), null, out time) as IList <DocAnalyticData>;
Console.WriteLine("\tSemantria: Batch of {0} documents has been received. Eexecution time is: {1} ms", temp.Count, time.TotalMilliseconds);
}
else
{
temp = session.GetProcessedDocuments(config.ConfigId);
}
foreach (DocAnalyticData data in temp)
{
if (context.Results.ContainsKey(data.Id))
{
context.Results[data.Id].AddOutput("Semantria", data.SentimentScore, data.SentimentPolarity);
queue[data.Id] = true;
}
}
index = queue.Values.Count(item => item == true);
progress += ((int)(100 * index) / queue.Count);
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, progress, failed);
context.OnExecutionProgress("Semantria", ea);
if (ea.Cancel)
{
break;
}
}
}
}
catch (Exception ex)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, index, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Semantria", ea);
if (ea.Cancel)
{
return;
}
}
context.OnExecutionProgress("Semantria", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, index, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Skyttle", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
try {
Dictionary<string, string> scoreDataMap = new Dictionary<string, string>();
scoreDataMap.Add("text", document.Value.ToString());
string url = "https://sentinelprojects-skyttle20.p.mashape.com/";
string text = document.Value.ToString();
string lang = LocaleHelper.GetDoubleLanguageAbbreviation(context.Language);
string keywords = "1";
string sentiment = "1";
string annotate = "0";
string param = "text=" + HttpUtility.UrlEncode(text, System.Text.Encoding.UTF8) +
"&lang=" + HttpUtility.UrlEncode(lang, System.Text.Encoding.UTF8) +
"&keywords=" + HttpUtility.UrlEncode(keywords, System.Text.Encoding.UTF8) +
"&sentiment=" + HttpUtility.UrlEncode(sentiment, System.Text.Encoding.UTF8) +
"&annotate=" + HttpUtility.UrlEncode(annotate, System.Text.Encoding.UTF8);
string response = "";
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
response = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return MakeCall(url, param, context.Key);
}), null, out time) as string;
Console.WriteLine("\tSkyttle: Sentiment for the document {0} has been retreived. Execution time is: {1}", document.Key, time.TotalMilliseconds);
}
else
response = MakeCall(url, param, context.Key);
JavaScriptSerializer serializer = new JavaScriptSerializer();
IDictionary<string, object> dict = serializer.Deserialize<IDictionary<string, object>>(response);
processed++;
IDictionary<string, object> obj = ((object[])dict["docs"])[0] as IDictionary<string, object>;
string language = obj["language"] as string;
IDictionary<string, object> sentiments = obj["sentiment_scores"] as IDictionary<string, object>;
decimal negScore = (decimal)sentiments["neg"];
decimal posScore = (decimal)sentiments["pos"];
decimal neuScore = (decimal)sentiments["neu"];
if (neuScore > 50)
document.Value.AddOutput("Skyttle", 0.0, "neutral");
else if (negScore > posScore)
document.Value.AddOutput("Skyttle", Convert.ToDouble(-1 * negScore / 100), "negative");
else if (posScore > negScore)
document.Value.AddOutput("Skyttle", Convert.ToDouble(posScore / 100), "positive");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Skyttle", ea);
if (ea.Cancel)
break;
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("Skyttle", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Skyttle", ea);
if (ea.Cancel)
break;
}
}
context.OnExecutionProgress("Skyttle", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
{
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
}
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
List <QualificationRequirement> requirements = new List <QualificationRequirement>();
QualificationRequirement sucRequirement = new QualificationRequirement();
sucRequirement.QualificationTypeId = MTurkSystemQualificationTypes.ApprovalRateQualification;
sucRequirement.IntegerValueSpecified = true;
sucRequirement.IntegerValue = settings.PercentOfSuccess;
sucRequirement.Comparator = Comparator.GreaterThanOrEqualTo;
requirements.Add(sucRequirement);
if (settings.Locale != null)
{
QualificationRequirement qualReq = new QualificationRequirement();
qualReq.LocaleValue = new Locale()
{
Country = settings.Locale
};
qualReq.Comparator = Comparator.EqualTo;
qualReq.QualificationTypeId = MTurkSystemQualificationTypes.LocaleQualification;
requirements.Add(qualReq);
}
string hitType = string.Empty;
try
{
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
hitType = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment nlp", requirements));
}), null, out time) as string;
Console.WriteLine("MechanicalTurk: HIT type for sentiment analysis has been created. HIT type ID is: {0} Execution time is: {1}", hitType, time.TotalMilliseconds);
}
else
{
hitType = client.RegisterHITType("Sentiment analysis", "Judge the sentiment expressed by the following text.",
settings.TimeToApprove, settings.TimeToFinish, settings.Reward, "sentiment, nlp", requirements);
}
NotificationSpecification notification = new NotificationSpecification();
notification.Transport = NotificationTransport.Email;
notification.EventType = new EventType[] { EventType.AssignmentReturned };
notification.Destination = settings.Email;
notification.Version = "2006-05-05";
if (settings.Email != null)
{
client.SetHITTypeNotification(hitType, notification, true);
}
else
{
notification.Destination = "*****@*****.**";
client.SetHITTypeNotification(hitType, notification, false);
}
}
catch (Exception ex)
{
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, 0, 0);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
return;
}
}
string questionFile = Path.Combine(AppDomain.CurrentDomain.SetupInformation.ApplicationBase, "SentimentTemplate.xml");
string template = File.ReadAllText(questionFile);
QuestionForm formTemplate = QuestionUtil.DeserializeQuestionForm(template);
int processed = 0;
int failed = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
formTemplate.Question[0].QuestionIdentifier = document.Key;
formTemplate.Question[0].QuestionContent.Items[0] = Encoding.UTF8.GetString(Encoding.Default.GetBytes(document.Value.Source));
string question = QuestionUtil.SerializeQuestionForm(formTemplate);
HIT hit = new HIT();
hit.Expiration = DateTime.Now.AddDays(1);
hit.ExpirationSpecified = true;
hit.HITGroupId = "SentimentAnalysis";
hit.HITTypeId = hitType;
hit.MaxAssignments = settings.Assignments;
hit.MaxAssignmentsSpecified = true;
hit.Question = question;
HIT serverHit = null;
try
{
processed++;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
serverHit = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(client.CreateHIT(hit));
}), null, out time) as HIT;
Console.WriteLine("MechanicalTurk: HIT {0} has been sent to Mechanical turk for processing. Execution time is: {1}", serverHit.HITId, time.TotalMilliseconds);
}
else
{
serverHit = client.CreateHIT(hit);
}
document.Value.AddOutput("MechanicalTurk", settings.Assignments, serverHit.HITId);
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
{
break;
}
}
catch (Exception ex)
{
failed++;
document.Value.AddOutput("MechanicalTurk", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
break;
}
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Execute(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("Repustate", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
int processed = 0;
int failed = 0;
bool isTrial = false;
try
{
string lang = LocaleHelper.GetDoubleLanguageAbbreviation(context.Language);
RepustateClient Repustate = new RepustateClient(context.Key);
Dictionary <string, string> TotalScoreDataMap = new Dictionary <string, string>();
int count = 0;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
count++;
string ky = "text" + count;
TotalScoreDataMap.Add(ky, document.Value.ToString());
}
int BatchSize = 500;
int processedBatches = 0;
if (TotalScoreDataMap.Count < BatchSize)
{
Dictionary <string, string> scoreDataMap = new Dictionary <string, string>();
scoreDataMap = Repustate.GetDocumentsQueue(processedBatches, BatchSize, TotalScoreDataMap);
scoreDataMap.Add("lang", lang);
JavaScriptSerializer serializer = new JavaScriptSerializer();
Array RepustateSentiments;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
RepustateSentiments = (Array)BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary <string, object> deserializedResponse = serializer.Deserialize <IDictionary <string, object> >(response) as IDictionary <string, object>;
return(deserializedResponse["results"] as Array);
}), null, out time);
Console.WriteLine("\tRepustate: Batch of {0} documents has been recieved. Eexecution time is: {1} ms", scoreDataMap.Count - 1, time.TotalMilliseconds);
}
else
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary <string, object> deserializedResponse = serializer.Deserialize <IDictionary <string, object> >(response) as IDictionary <string, object>;
RepustateSentiments = deserializedResponse["results"] as Array;
}
SortedDictionary <int, string> rs = new SortedDictionary <int, string>();
for (int i = 0; i < RepustateSentiments.Length; i++)
{
int id = Int32.Parse(((Dictionary <string, object>)RepustateSentiments.GetValue(i))["id"].ToString());
string score = ((Dictionary <string, object>)RepustateSentiments.GetValue(i))["score"].ToString();
rs.Add(id, score);
}
int c = 1;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
double score = double.Parse(rs[c]);
processed++;
if (score <= -0.05)
{
document.Value.AddOutput("Repustate", score, "negative");
}
if (score >= 0.05)
{
document.Value.AddOutput("Repustate", score, "positive");
}
else
{
document.Value.AddOutput("Repustate", score, "neutral");
}
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Repustate", ea);
c++;
if (ea.Cancel)
{
break;
}
}
}
else
{
int totalBatches = (TotalScoreDataMap.Count / BatchSize) + 1;
processedBatches = 0;
for (int i = 0; i < totalBatches; i++)
{
Dictionary <string, string> scoreDataMap = new Dictionary <string, string>();
scoreDataMap = Repustate.GetDocumentsQueue(processedBatches, BatchSize, TotalScoreDataMap);
scoreDataMap.Add("lang", lang);
JavaScriptSerializer serializer = new JavaScriptSerializer();
Array RepustateSentiments;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
RepustateSentiments = (Array)BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary <string, object> deserializedResponse = serializer.Deserialize <IDictionary <string, object> >(response) as IDictionary <string, object>;
return(deserializedResponse["results"] as Array);
}), null, out time);
Console.WriteLine("\tRepustate: Batch of {0} documents has been recieved. Eexecution time is: {1} ms", scoreDataMap.Count - 1, time.TotalMilliseconds);
}
else
{
string response = Repustate.GetSentimentBulk(scoreDataMap);
IDictionary <string, object> deserializedResponse = serializer.Deserialize <IDictionary <string, object> >(response) as IDictionary <string, object>;
RepustateSentiments = deserializedResponse["results"] as Array;
}
SortedDictionary <int, string> rs = new SortedDictionary <int, string>();
for (int j = 0; j < RepustateSentiments.Length; j++)
{
int id = Int32.Parse(((Dictionary <string, object>)RepustateSentiments.GetValue(j))["id"].ToString());
string score = ((Dictionary <string, object>)RepustateSentiments.GetValue(j))["score"].ToString();
rs.Add(id, score);
}
int key = 1;
int UpperCounter = processedBatches;
int LowerCounter = BatchSize;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
if (UpperCounter > 0)
{
key++;
UpperCounter--;
continue;
}
if (LowerCounter < 1)
{
break;
}
LowerCounter--;
double score = double.Parse(rs[key]);
processed++;
if (score <= -0.05)
{
document.Value.AddOutput("Repustate", score, "negative");
}
if (score >= 0.05)
{
document.Value.AddOutput("Repustate", score, "positive");
}
else
{
document.Value.AddOutput("Repustate", score, "neutral");
}
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("Repustate", ea);
key++;
if (ea.Cancel)
{
break;
}
}
processedBatches = processedBatches + BatchSize;
}
}
}
catch (WebException ex)
{
if (!(ex.Response.ContentLength == -1))
{
isTrial = true;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, context.Results.Count, 0, 0);
ea.Reason = "Your Repustate account doesn’t support " + context.Language + " language";
context.OnExecutionProgress("Repustate", ea);
}
else
{
isTrial = false;
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
failed++;
document.Value.AddOutput("Repustate", 0, "failed");
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("Repustate", ea);
if (ea.Cancel)
{
break;
}
}
}
}
if (!isTrial)
{
context.OnExecutionProgress("Repustate", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
}
public void Request(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
{
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
}
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
int failed = 0;
int processed = 0;
try
{
foreach (KeyValuePair <string, ResultSet> document in context.Results)
{
string id = document.Value.GetPolarity("MechanicalTurk");
double count = document.Value.GetScore("MechanicalTurk");
count = (double.IsNaN(count)) ? 0 : count;
if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
{
continue;
}
IList <Assignment> assignments = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return(client.GetAssignmentsForHIT(id, 1, true));
}), null, out time) as IList <Assignment>;
Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
}
else
{
assignments = client.GetAssignmentsForHIT(id, 1, true);
}
if (assignments.Count < count)
{
processed++;
continue;
}
double confidence = 0;
IList <string> answers = DeserializeAnswers(assignments);
string polarity = MergeSentimentPolarity(answers, out confidence);
processed++;
document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
{
break;
}
}
}
catch (Exception ex)
{
failed++;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
{
return;
}
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}
public void Request(AnalysisExecutionContext context)
{
_context = context;
if (context.Results.Count <= 0)
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
if (!context.Results.Values.First().GetServices().Contains("MechanicalTurk"))
{
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Canceled, 0, 0, 0));
return;
}
MechanicalTurkSettings settings = context.CustomField as MechanicalTurkSettings;
if (settings == null)
settings = new MechanicalTurkSettings(null, 3, new decimal(0.10), 300, 1800, 75);
string key = context.Key;
string secret = context.Secret;
MTurkConfig config = new MTurkConfig(_serviceURL, key, secret);
SimpleClient client = new SimpleClient(config);
int failed = 0;
int processed = 0;
try
{
foreach (KeyValuePair<string, ResultSet> document in context.Results)
{
string id = document.Value.GetPolarity("MechanicalTurk");
double count = document.Value.GetScore("MechanicalTurk");
count = (double.IsNaN(count)) ? 0 : count;
if (id == "negative" || id == "neutral" || id == "positive" || id == "failed" || id == "undefined")
continue;
IList<Assignment> assignments = null;
if (context.UseDebugMode)
{
TimeSpan time = TimeSpan.Zero;
assignments = BenchmarkHelper.Invoke(new InvokeBenchmarkHandler(delegate(object state)
{
return client.GetAssignmentsForHIT(id, 1, true);
}), null, out time) as IList<Assignment>;
Console.WriteLine("MechanicalTurk: Answers for {0} HIT have been received. Execution time is: {1}", id, time.TotalMilliseconds);
}
else
assignments = client.GetAssignmentsForHIT(id, 1, true);
if (assignments.Count < count)
{
processed++;
continue;
}
double confidence = 0;
IList<string> answers = DeserializeAnswers(assignments);
string polarity = MergeSentimentPolarity(answers, out confidence);
processed++;
document.Value.AddOutput("MechanicalTurk", double.NaN, polarity, confidence);
document.Value.AddReferencePolarity(polarity, "MechanicalTurk");
AnalysisExecutionProgressEventArgs e = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Processed, context.Results.Count, processed, failed);
context.OnExecutionProgress("MechanicalTurk", e);
if (e.Cancel)
break;
}
}
catch (Exception ex)
{
failed++;
AnalysisExecutionProgressEventArgs ea = new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Failed, context.Results.Count, processed, failed);
ea.Reason = ex.Message;
context.OnExecutionProgress("MechanicalTurk", ea);
if (ea.Cancel)
return;
}
context.OnExecutionProgress("MechanicalTurk", new AnalysisExecutionProgressEventArgs(AnalysisExecutionStatus.Success, context.Results.Count, processed, failed));
}