public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// Assumpltion: the input is a folder videos, convert to mp4, and upload to satyam blob
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
SingleObjectLabelingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <SingleObjectLabelingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
//var client = new WebClient();
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
FFMpegWrappers.ConvertVideoURLToMP4(DirectoryConstants.ffmpeg, video, guidFolder);
}
satyamStorage.uploadALocalFolder(guidFolder, satyamContainerName, satyamDirectoryName);
return(true);
}
//public static Dictionary<string, int> getRemainingHITsFinishedTasksFromResultDB(string guid, int taskPerHIT)
//{
// SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> entries = resultDB.getEntriesByGUID(guid);
// Dictionary<string, int> noResultPerHITID = new Dictionary<string, int>();
// //int noFinishedHIT = 0;
// foreach (SatyamResultsTableEntry entry in entries)
// {
// string HITId = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString).amazonInfo.HITID;
// if (!noResultPerHITID.ContainsKey(HITId))
// {
// noResultPerHITID.Add(HITId,1);
// }
// else
// {
// noResultPerHITID[HITId]++;
// //if (noResultPerHITID[HITId] == taskPerHIT)
// //{
// // noFinishedHIT++;
// //}
// }
// }
// return noResultPerHITID;
//}
//makes sure that there are enough HITS in Amazon to ensure that jobs will be done
public static void LaunchAmazonHITsFromTaskTable(int max = int.MaxValue)
{
//Group them by jobguids since each job can have many tasks
Dictionary <string, List <SatyamTaskTableEntry> > TasksByGUID = OrganizeMTurkTasksByGUID();
if (TasksByGUID == null) //there is nothing to do!!
{
return;
}
//after running the aggregator, the number of tasks in each guid <= no of HITS pending
//so if there are less HITS pending we launch more and if there are more we leave them alone until the job is done
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
//now check how manyy hits are pending and launch to ensure that there are enough hits in amazon
foreach (KeyValuePair <string, List <SatyamTaskTableEntry> > entry in TasksByGUID)
{
String jobGUID = entry.Key;
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.Value[0].TaskParametersString);
SatyamJob job = task.jobEntry;
//// filter task templates -- temporary filter
//if (job.JobTemplateType == TaskConstants.Segmentation_Image_MTurk) continue;
string AmazonAccessKeyID = job.amazonHITInformation.AmazonAccessKeyID;
string AmazonSecretAccessKeyID = job.amazonHITInformation.AmazonSecretAccessKeyID;
string JobTemplateType = entry.Value[0].JobTemplateType;
Dictionary <string, Object> hitParams = new Dictionary <string, object>();
hitParams.Add("Title", job.amazonHITInformation.AmazonMTurkTaskTitle);
hitParams.Add("TaskDescription", job.amazonHITInformation.AmazonMTurkTaskDescription);
hitParams.Add("KeyWords", job.amazonHITInformation.AmazonMTurkTaskKeywords);
hitParams.Add("Reward", job.amazonHITInformation.Price); //convert cents to dollars
if (TaskConstants.masterGUIDs.Contains(job.JobGUIDString))
{
hitParams.Add("Masters", true);
}
int tasksPerHIT = job.TasksPerJob;
List <SatyamAmazonHITTableAccessEntry> hitEntries = hitDB.getAllEntriesByJobGUIDAndStatus(jobGUID, HitStatus.pending);
// the task will be marked as submitted when the webpage exit
int noHITsRemaining = hitEntries.Count;
////////////////////////// check and deal with pending stragglers, the actual hit might have expired.
//int noStragglers = 0;
//foreach (SatyamAmazonHITTableAccessEntry pendingEntry in hitEntries)
//{
// if ((DateTime.Now - pendingEntry.CreationTime).TotalHours > 1)
// {
// // the hit will be finished in 1hr if taken by a user,
// // there is a higher chance that this hit have already been expired
// // launch more for stragglers
// noStragglers++;
// }
//}
//noHITsRemaining -= noStragglers;
///////////////////////////////
//////////////////////////////////// the query amazon way
//AmazonMTurkHIT hit = new AmazonMTurkHIT();
//hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false);
////an alternative without querying amazon
////Dictionary<string, int> noHITsFinishedTasks = getRemainingHITsFinishedTasksFromResultDB(jobGUID, tasksPerHIT);
////int noHITsRemaining = 0;
//noHITsRemaining = 0;
//int noPendingHITDone = 0;
//foreach (SatyamAmazonHITTableAccessEntry hitEntry in hitEntries)
//{
// int noTasksPending = hit.getNumTasksRemaining(hitEntry.HITID);
// noHITsRemaining += noTasksPending;
// //an alternative without querying amazon
// //if (noHITsFinishedTasks.ContainsKey(hitEntry.HITID) && noHITsFinishedTasks[hitEntry.HITID] == tasksPerHIT)
// //{
// // //HIT done
// // noPendingHITDone++;
// //}
//}
////int noHITsRemainingFromResultDB = hitEntries.Count - noPendingHITDone;
////Console.WriteLine("Job {0}, HITS Remaining/fromDB/Pending/noTasks: {1}/{2}/{3}",
//// jobGUID, noHITsRemaining, noHITsRemainingFromResultDB, hitEntries.Count, TasksByGUID[jobGUID].Count);
////if (noHITsRemainingFromResultDB!= noHITsRemaining)
////{
//// // not gonna work, what is the hit has less tasks than tasksPerHIT??
//// Console.WriteLine("Database method is wrong");
////}
/////////////////////////////////////////////////////////
int NoParallelTurker = 3; // too large will make the task stale on AMT, too small will hurt latency
int difference = (int)Math.Ceiling((double)(TasksByGUID[jobGUID].Count * NoParallelTurker - noHITsRemaining * tasksPerHIT) / (double)tasksPerHIT);
//int difference = 50;
//difference *= 2; // boost for parallelism
//difference = 10;
//difference = Math.Min(difference, max) * NoParallelTurker;
if (difference > 0)
{
AmazonMTurkHITBatchLauncher hitLauncher = new AmazonMTurkHITBatchLauncher(AmazonAccessKeyID, AmazonSecretAccessKeyID);
string url = TaskConstants.JobTemplateTypeToTaskURI[JobTemplateType];
double reward = job.amazonHITInformation.Price;
url = url + "?reward=" + reward;
List <string> hitIDs = null;
string hitParametersString = JSonUtils.ConvertObjectToJSon <Dictionary <string, object> >(hitParams);
hitLauncher.LaunchBatches(url, hitParams, difference, out hitIDs);
if (hitIDs != null)
{
foreach (string hitID in hitIDs)
{
hitDB.AddEntry(JobTemplateType, job.UserID, hitID, job.JobGUIDString, DateTime.Now, hitParametersString, AmazonAccessKeyID, AmazonSecretAccessKeyID);
}
jobDB.UpdateEntryStatus(entry.Key, JobStatus.launched);
}
}
}
jobDB.close();
hitDB.close();
}
private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
if (Submit_Button.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
// TaskConstants.TrackletLabeling_MTurk, price);
entry = taskTableDB.getTopKNewEntryForWorkerIDByTemplateAndPrice(50, Hidden_AmazonWorkerID.Value,
TaskConstants.TrackletLabeling_MTurk, price);
}
else
{
entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.TrackletLabeling_MTurk);
}
taskTableDB.close();
if (entry == null)
{
return(false);
}
taskTableDB = new SatyamTaskTableAccess();
taskTableDB.IncrementDoneScore(entry.ID);
taskTableDB.close();
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string videoDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(videoDir, new List <string>()
{
"jpg"
});
string annotationFilePath = task.SatyamURI;
//string urls = "";
//for (int i=0;i<ImageURLs.Count;i++)
//{
// urls += ImageURLs[i];
// if (i == ImageURLs.Count - 1) break;
// urls += ',';
//}
Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ',');
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters);
Dictionary <string, List <string> > subcategories = job.Categories;
List <string> categories = subcategories.Keys.ToList();
//CategorySelection_RakdioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
//CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
//DescriptionPanel.Visible = true;
//DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
//Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
// pass parameters from old template
Slug_Hidden.Value = "null";
Start_Hidden.Value = "0";
Stop_Hidden.Value = (ImageURLs.Count - 1).ToString();
Skip_Hidden.Value = "0";
PerObject_Hidden.Value = "0.1";
Completion_Hidden.Value = "0.5";
BlowRadius_Hidden.Value = "0";
JobId_Hidden.Value = "1";
LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ',');
Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_');
Training_Hidden.Value = "0";
fps_Hidden.Value = job.FrameRate.ToString();
Hidden_ChunkDuration.Value = job.ChunkDuration.ToString();
var web = new WebClient();
System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0]));
ImageWidth_Hidden.Value = x.Width.ToString();
ImageHeight_Hidden.Value = x.Height.ToString();
// image boundary for now
//string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" };
string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" };
RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ',');
// temp test
List <VATIC_Tracklet> prevTracesTemp = new List <VATIC_Tracklet>();
WebClient client = new WebClient();
Stream stream = client.OpenRead(annotationFilePath);
StreamReader reader = new StreamReader(stream);
List <string> trace = new List <string>();
while (reader.Peek() >= 0)
{
string content = reader.ReadLine();
trace.Add(content);
}
Dictionary <string, VATIC_Tracklet> tracklets = VATIC_Tracklet.ReadTrackletsFromVIRAT(trace);
foreach (string id in tracklets.Keys)
{
//string output = JSonUtils.ConvertObjectToJSon(tracklets[id]);
prevTracesTemp.Add(tracklets[id]);
}
string output = JSonUtils.ConvertObjectToJSon(prevTracesTemp);
PreviousTrackString_Hidden.Value = output;
return(true);
}
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
double IoUTreshold,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
// Organize by taskID
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!EntriesPerTask.ContainsKey(taskEntryID))
{
EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
EntriesPerTask[taskEntryID].Add(entry);
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
}
foreach (int taskEntryID in EntriesPerTask.Keys)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
List <int> taskidfilter = new List <int>()
{
40430,
40432,
40433,
40434,
40440,
40447,
40451,
40460,
};
//if (fileName != "2007_000549") continue;
if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
{
continue;
}
Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = satyamResult.TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = job.JobGUIDString;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
}
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job
// All we need to do: (IN STRICT ORDER)
// clear all results back to inconclusive,
// remove all aggregated results
// restore the task table as it was for the guid,
// change the job status back to launched, and
// WARNING:
// for safety, this has to be run atomically synchronously, without any parallel process. for now.
public static void reopenJobForMoreResults(string guid)
{
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid);
jobDB.close();
if (jobEntry.JobStatus != JobStatus.completed)
{
Console.WriteLine("Job not completed yet!");
return;
}
SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess();
bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid);
aggResultDB.close();
if (!delSuccess)
{
Console.WriteLine("Delete Agg Result DB Failed");
return;
}
SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess();
if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive))
{
Console.WriteLine("Update Result DB Failed");
//resultDB.close();
//return;
}
List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid);
resultDB.close();
Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>();
foreach (SatyamResultsTableEntry result in results)
{
if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
continue;
}
SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString);
taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task);
}
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
foreach (int taskID in taskParamsByTaskID.Keys)
{
SatyamTask task = taskParamsByTaskID[taskID];
SatyamJob job = task.jobEntry;
bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task),
job.JobSubmitTime, job.amazonHITInformation.Price);
if (!suc)
{
Console.WriteLine("Update Task Table Failed");
taskDB.close();
return;
}
}
taskDB.close();
jobDB = new SatyamJobSubmissionsTableAccess();
bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched);
jobDB.close();
if (!success)
{
Console.WriteLine("Update Job Entry Failed");
return;
}
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
protected void JobSubmitButton_Click(object sender, EventArgs e)
{
string AzureBlobStorageConnectionString = AzureBlobStorageConnectionStringTextBox.Text;
string AzureContainer = AzureBlobStorageContainerNameTextBox.Text;
string AzureContainerDirectory = AzureBlobStorageContainerDirectoryNameTextBox.Text;
string AmazonAccessKeyIDValue = AmazonAccessKeyID.Text;
string AmazonSecretAccessKeyIDValue = AmazonSecretAccessKeyID.Text;
string Description = CategoryDescription.Text;
string jobGuid = NewJobGUID.Text;
string AmazonTaskTitle = AmazonTaskTitleTextBox.Text;
string AmazonTaskDescription = AmazonTaskDescriptionTextBox.Text;
string AmazonTaskKeywords = AmazonTaskKeywordsTextBox.Text;
double pricePerObject = 0; //default
int numObjectPerImage = 0; //default
double price = 0; //default
int tasksPerHIT = 1; //defuault
if (AzureBlobStorageConnectionString == "" || AzureContainer == "")
{
Template_ErrorMessageLabel.Text = "Error : Azure Connection String and Container are mandatory fields.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureConnectionInfo connectionInfo = new AzureConnectionInfo(AzureBlobStorageConnectionString, AzureContainer, AzureContainerDirectory);
int noFiles = connectionInfo.getNoFiles();
if (noFiles == -1)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
else if (noFiles == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are 0 files at the Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonMTurkHIT hit = new AmazonMTurkHIT();
bool success = hit.setAccount(AmazonAccessKeyIDValue, AmazonSecretAccessKeyIDValue, false);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double balance = hit.getAccountBalance();
if (balance < 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (balance < 0.1)
{
Template_ErrorMessageLabel.Text = "Error : Insufficient money in Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
}
if (categories.Count == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are no Categories";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
List <LineSegment> BoundaryLines = new List <LineSegment>();
if (BoundaryTextBox.Text != "")
{
bool error = false;
string errorMessage = "";
string[] boundaryFields = BoundaryTextBox.Text.Split(',');
for (int i = 0; i < boundaryFields.Length; i++)
{
string[] coords = boundaryFields[i].Split('-');
if (coords.Length != 4)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : does not have 4 values.";
error = true;
break;
}
int x1, y1, x2, y2;
bool success = int.TryParse(coords[0], out x1);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[0] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[1], out y1);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[1] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[2], out x2);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[2] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[3], out y2);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[3] + " must be an integer";
error = true;
break;
}
LineSegment ls = new LineSegment(x1, y1, x2, y2);
BoundaryLines.Add(ls);
}
if (error)
{
Template_ErrorMessageLabel.Text = errorMessage;
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
}
if (Hidden_MechanicalTurk.Value == "true")
{
bool success = Double.TryParse(PriceTextBox.Text, out pricePerObject);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical price entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
pricePerObject = pricePerObject / 100; //covert to dollars
success = Int32.TryParse(NoObjectsPerImageTextBox.Text, out numObjectPerImage);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical number of objects per image entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
price = pricePerObject * numObjectPerImage;
if (price <= 0.0)
{
Template_ErrorMessageLabel.Text = "Error : The Price is Zero";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
//need to round to cents
price = Math.Floor(price * 100) / 100;
AmazonMTurkHIT hit = new AmazonMTurkHIT();
success = hit.setAccount(AmazonAccessKeyIDValue, AmazonSecretAccessKeyIDValue, false);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double balance = hit.getAccountBalance();
if (balance < 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double moneyNeeded = 4 * price * noFiles / tasksPerHIT;
if (balance < moneyNeeded)
{
Template_ErrorMessageLabel.Text = "Error : Insufficient money in Amazon Turk Account, you will need atleast " + moneyNeeded + "$.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Image Segmentation";
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Earn money quickly by drawing polygons around objects of interest in an image!";
}
if (AmazonTaskKeywords == "Image Segmentation")
{
Template_ErrorMessageLabel.Text = "";
}
}
AzureInformation AzureInfo = new AzureInformation();
AzureInfo.AzureBlobStorageConnectionString = AzureBlobStorageConnectionString;
AzureInfo.AzureBlobStorageContainerName = AzureContainer;
AzureInfo.AzureBlobStorageContainerDirectoryName = AzureContainerDirectory;
AmazonMTurkHITInformation AmazonInfo = new AmazonMTurkHITInformation();
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonInfo.AmazonAccessKeyID = AmazonAccessKeyIDValue;
AmazonInfo.AmazonSecretAccessKeyID = AmazonSecretAccessKeyIDValue;
AmazonInfo.Price = price;
AmazonInfo.AmazonMTurkTaskTitle = AmazonTaskTitle;
AmazonInfo.AmazonMTurkTaskDescription = AmazonTaskDescription;
AmazonInfo.AmazonMTurkTaskKeywords = AmazonTaskKeywords;
}
else
{
AmazonInfo.AmazonAccessKeyID = "";
AmazonInfo.AmazonSecretAccessKeyID = "";
AmazonInfo.Price = 0;
AmazonInfo.AmazonMTurkTaskTitle = "";
AmazonInfo.AmazonMTurkTaskDescription = "";
AmazonInfo.AmazonMTurkTaskKeywords = "";
}
ImageSegmentationSubmittedJob jobParameters = new ImageSegmentationSubmittedJob();
jobParameters.Categories = categories;
jobParameters.Description = Description;
jobParameters.BoundaryLines = BoundaryLines;
SatyamJob job = new SatyamJob();
job.UserID = userName;
job.amazonHITInformation = AmazonInfo;
job.azureInformation = AzureInfo;
job.JobGUIDString = jobGuid;
if (Hidden_MechanicalTurk.Value == "true")
{
job.JobTemplateType = TaskConstants.Segmentation_Image_MTurk;
job.TasksPerJob = tasksPerHIT;
}
else
{
job.JobTemplateType = TaskConstants.Segmentation_Image;
job.TasksPerJob = 0;
}
job.JobSubmitTime = DateTime.Now;
job.JobParameters = JSonUtils.ConvertObjectToJSon <ImageSegmentationSubmittedJob>(jobParameters);
string jobDefinition = JSonUtils.ConvertObjectToJSon <SatyamJob>(job);
SatyamJobSubmissionsTableAccess dbAccess = new SatyamJobSubmissionsTableAccess();
dbAccess.AddEntry(job.JobTemplateType, job.UserID, job.JobGUIDString, jobDefinition, job.JobSubmitTime);
dbAccess.close();
Response.Redirect("NewJobSubmitted.aspx");
}
private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
if (SubmitButton.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice("SINGLE_OBJECT_LABLING_MTURK", price);
entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
TaskConstants.Classification_Video_MTurk, price);
}
else
{
entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Classification_Video_MTurk);
}
if (entry != null)
{
taskTableDB.IncrementDoneScore(entry.ID);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string uri = task.SatyamURI;
Hidden_VideoURL.Value = uri;
SatyamJob jobDefinitionEntry = task.jobEntry;
SingleObjectLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <SingleObjectLabelingSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
CategorySelection_RadioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
DescriptionPanel.Visible = true;
DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
//DescriptionLabel.Text = job.Description;
}
Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString();
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
NoLabeled.Text = Hidden_NoImagesDone.Value;
taskTableDB.close();
return(true);
}
else
{
taskTableDB.close();
return(false);
}
}
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
//bool saveImage = false,
bool prepareTrainingSet = false, string outputDirectory = null,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
)
{
if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
{
Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
}
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
//int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
/// // record logs
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
double totalError, totalGroundtruth;
//ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
// MinHeight, MaxOcclusion, Max_Truncation);
ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
"_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
if (jobParams.DataSrcFormat == DataFormat.Video)
{
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(video);
string videonameExtension = URIUtilities.filenameFromURI(video);
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
client.DownloadFile(video, outputDirectory + "\\" + videonameExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
Console.WriteLine("deleting downloaded file...");
File.Delete(outputDirectory + "\\" + videonameExtension);
GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
if (jobParams.DataSrcFormat == DataFormat.VideoFrame)
{
int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
// chunk according to parameters
List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >();
foreach (string url in frameUrls)
{
// assumed hierarchy: blob/directory.../videoname/frameid.jpg
//string frameName = URIUtilities.filenameFromURINoExtension(url);
string[] urlparts = url.Split('/');
string videoName = urlparts[urlparts.Length - 2];
if (!framesPerVideo.ContainsKey(videoName))
{
framesPerVideo.Add(videoName, new List <string>());
}
framesPerVideo[videoName].Add(url);
}
foreach (string video in framesPerVideo.Keys)
{
string outputDirectory = guidFolder + "\\" + video;
Directory.CreateDirectory(outputDirectory);
foreach (string frameURL in framesPerVideo[video])
{
string frameName = URIUtilities.filenameFromURI(frameURL);
client.DownloadFile(frameURL, outputDirectory + "\\" + frameName);
}
GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
return(true);
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = fields[0];
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters);
Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = jobDefinition.Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
//fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID;
fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID;
}
ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName);
}
}
public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID);
resultsDB.close();
Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>();
Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>();
SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >();
SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >();
Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>();
Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>();
Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>();
Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>();
int totalResults = 0;
int noTasksWithDuplicateResultsFromSameWorker = 0;
int noTasksWithMixedResultsFromSameWorker = 0;
int noTaskWithDuplicateResultsAsMajority = 0;
int noTaskWhoseDuplicateResultsChangedAggregation = 0;
int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0;
int noCorrectDecisionsAmongDuplicates = 0;
//noCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noWorkerSwitchedToCorrect = 0;
int noWorkerSwitchedToCorrectAndMaintained = 0;
int noTasksWithWorkerSwitchedToCorrect = 0;
int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0;
List <double> timeTakenPerResult = new List <double>();
List <double> acceptedTimeTakenPerResult = new List <double>();
List <int> SequenceNumberOfResultPerTask = new List <int>();
List <int> SequenceNumberOfResultPerWorker = new List <int>();
string JobGUID = entries[0].JobGUID;
Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>();
List <double> resultArrivalTimes = new List <double>();
totalResults = entries.Count;
///imagenet only
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!noResultsPerTask.ContainsKey(taskEntryID))
{
noResultsPerTask.Add(taskEntryID, 0);
noAcceptedResultsPerTask.Add(taskEntryID, 0);
noPaidResultsPerTask.Add(taskEntryID, 0);
moneyPaidPerTask.Add(taskEntryID, 0);
finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
}
ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
}
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
List <int> taskIDs = noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID])
{
//has multiple results from same turker
noTasksWithDuplicateResultsFromSameWorker++;
// the aggregation result
List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult();
// the aggregation if without duplicate results
List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult();
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString));
}
OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0]));
}
aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask);
aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask);
if (aggregatedResultPerTask != null)
{
if (aggregatedOnlyFirstResultOfTurkerPerTask == null ||
aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category)
{
noTaskWhoseDuplicateResultsChangedAggregation++;
if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask)))
{
noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++;
}
}
}
bool hasMixedResults = false;
bool hasCorrectResult = false;
bool atLeastOneShiftedFromIncorrectToCorrect = false;
bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false;
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count))
{
ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0);
}
ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++;
double superMajority = 0.6; // tunable
if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority)
{
noTaskWithDuplicateResultsAsMajority++;
}
///////////// imagenet only //////////////////
if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1)
{
// multiple choices given
hasMixedResults = true;
bool incorrect = false;
bool switchToCorrect = false;
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString))
{
hasCorrectResult = true;
noCorrectDecisionsAmongDuplicates++;
if (incorrect)
{
switchToCorrect = true;
}
incorrect = false;
}
else
{
incorrect = true;
}
}
if (switchToCorrect)
{
// been incorrect and swithed to correct
noWorkerSwitchedToCorrect++;
atLeastOneShiftedFromIncorrectToCorrect = true;
if (!incorrect)
{
//switch to correct and maintain till the end
noWorkerSwitchedToCorrectAndMaintained++;
atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true;
}
}
}
////////////////////////////////////////////////
}
if (hasMixedResults)
{
noTasksWithMixedResultsFromSameWorker++;
}
if (hasCorrectResult)
{
noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++;
}
if (atLeastOneShiftedFromIncorrectToCorrect)
{
noTasksWithWorkerSwitchedToCorrect++;
}
if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained)
{
noTasksWithWorkerSwitchedToCorrectAndMaintained++;
}
}
}
Console.WriteLine("DuplicateResultsHistogram");
foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys)
{
Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]);
}
Console.WriteLine(
//"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" +
//"\t{3} images have >=1 workers making more than majority number of results.\n" +
//"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
//"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
"\t\t{6} images has correct decision among duplicate decisions.\n" +
"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
//noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates,
//noTaskWithDuplicateResultsAsMajority,
//noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
//noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask,
noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect,
noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained,
noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
);
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}
protected void Object_Counting_JobSubmitButton_Click(object sender, EventArgs e)
{
string AzureBlobStorageConnectionString = AzureBlobStorageConnectionStringTextBox.Text;
string AzureContainer = AzureBlobStorageContainerNameTextBox.Text;
string AzureContainerDirectory = AzureBlobStorageContainerDirectoryNameTextBox.Text;
string AmazonAccessKeyIDValue = AmazonAccessKeyID.Text;
string AmazonSecretAccessKeyIDValue = AmazonSecretAccessKeyID.Text;
string Description = CategoryDescription.Text;
double pricePerImage = 0;
double price = 0;
string jobGuid = NewJobGUID.Text;
string AmazonTaskTitle = AmazonTaskTitleTextBox.Text;
string AmazonTaskDescription = AmazonTaskDescriptionTextBox.Text;
string AmazonTaskKeywords = AmazonTaskKeywordsTextBox.Text;
if (AzureBlobStorageConnectionString == "" || AzureContainer == "")
{
Template_ErrorMessageLabel.Text = "Error : Azure Connection String and Container are mandatory fields.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureConnectionInfo connectionInfo = new AzureConnectionInfo(AzureBlobStorageConnectionString, AzureContainer, AzureContainerDirectory);
int noFiles = connectionInfo.getNoFiles();
if (noFiles == -1)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
else if (noFiles == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are 0 files at the Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Hidden_MechanicalTurk.Value == "true")
{
bool success = Double.TryParse(PriceTextBox.Text, out pricePerImage);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical price entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
pricePerImage = pricePerImage / 100; //convert to dollars
price = pricePerImage * TaskConstants.OBJECT_COUNTING_MTURK_MAX_IMAGES_PER_TASK;
//need to round to cents
price = Math.Floor(price * 100) / 100;
if (price <= 0.0)
{
Template_ErrorMessageLabel.Text = "Error : The Price is Zero";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AmazonMTurkHIT hit = new AmazonMTurkHIT();
success = hit.setAccount(AmazonAccessKeyIDValue, AmazonSecretAccessKeyIDValue, false);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double balance = hit.getAccountBalance();
if (balance < 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double moneyNeeded = 4 * price * noFiles / TaskConstants.OBJECT_COUNTING_MTURK_MAX_IMAGES_PER_TASK;
if (balance < moneyNeeded)
{
Template_ErrorMessageLabel.Text = "Error : Insufficient money in Amazon Turk Account. You will need at least " + moneyNeeded + "$.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Counting Objects in an Image";
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Earn money quickly by simply counting the number of objects of interest in an image!";
}
if (AmazonTaskKeywords == "Object Counting")
{
Template_ErrorMessageLabel.Text = "";
}
}
string objectName = ObjectNameTextBox.Text;
if (objectName == "")
{
Template_ErrorMessageLabel.Text = "Error : No Object Name Entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureInformation AzureInfo = new AzureInformation();
AzureInfo.AzureBlobStorageConnectionString = AzureBlobStorageConnectionString;
AzureInfo.AzureBlobStorageContainerName = AzureContainer;
AzureInfo.AzureBlobStorageContainerDirectoryName = AzureContainerDirectory;
AmazonMTurkHITInformation AmazonInfo = new AmazonMTurkHITInformation();
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonInfo.AmazonAccessKeyID = AmazonAccessKeyIDValue;
AmazonInfo.AmazonSecretAccessKeyID = AmazonSecretAccessKeyIDValue;
AmazonInfo.Price = price; //in dollars
AmazonInfo.AmazonMTurkTaskTitle = AmazonTaskTitle;
AmazonInfo.AmazonMTurkTaskDescription = AmazonTaskDescription;
AmazonInfo.AmazonMTurkTaskKeywords = AmazonTaskKeywords;
}
else
{
AmazonInfo.AmazonAccessKeyID = "";
AmazonInfo.AmazonSecretAccessKeyID = "";
AmazonInfo.Price = 0;
AmazonInfo.AmazonMTurkTaskTitle = "";
AmazonInfo.AmazonMTurkTaskDescription = "";
AmazonInfo.AmazonMTurkTaskKeywords = "";
}
ObjectCountingSubmittedJob jobParameters = new ObjectCountingSubmittedJob();
jobParameters.ObjectName = objectName;
jobParameters.Description = Description;
SatyamJob job = new SatyamJob();
job.UserID = userName;
job.amazonHITInformation = AmazonInfo;
job.azureInformation = AzureInfo;
job.JobGUIDString = jobGuid;
if (Hidden_MechanicalTurk.Value == "true")
{
job.JobTemplateType = TaskConstants.Counting_Image_MTurk;
job.TasksPerJob = TaskConstants.OBJECT_COUNTING_MTURK_MAX_IMAGES_PER_TASK;
}
else
{
job.JobTemplateType = TaskConstants.Counting_Image;
job.TasksPerJob = 0;
}
job.JobSubmitTime = DateTime.Now;
job.JobParameters = JSonUtils.ConvertObjectToJSon <ObjectCountingSubmittedJob>(jobParameters);
if (Hidden_AdaptivePricing.Value == "true")
{
job.AdaptivePricing = true;
bool success = Double.TryParse(TargetPricePerHourTextBox.Text, out job.TargetPricePerTask);
if (!success)
{
job.TargetPricePerTask = 13.56; // default for now
}
success = Double.TryParse(BudgetTextBox.Text, out job.TotalBudget);
if (!success)
{
job.TotalBudget = -1; //default
}
}
else
{
job.AdaptivePricing = false;
job.TargetPricePerTask = -1;
job.TotalBudget = -1;
}
string jobDefinition = JSonUtils.ConvertObjectToJSon <SatyamJob>(job);
SatyamJobSubmissionsTableAccess dbAccess = new SatyamJobSubmissionsTableAccess();
dbAccess.AddEntry(job.JobTemplateType, job.UserID, job.JobGUIDString, jobDefinition, job.JobSubmitTime);
dbAccess.close();
Response.Redirect("NewJobSubmitted.aspx");
}
//protected void CategoryListBox_SelectedIndexChanged(object sender, EventArgs e)
//{
// SubCategoryListBox.Items.Clear();
// string selectedItem = CategoryListBox.SelectedValue;
// List<string> items = subCategories[selectedItem];
// for (int i = 0; i < items.Count; i++)
// {
// ListItem item = new ListItem(items[i]);
// SubCategoryListBox.Items.Add(item);
// }
//}
//protected void AddSubCategoryButton_Click(object sender, EventArgs e)
//{
// string category = CategoryListBox.SelectedValue;
// string newSubCategory = AddSubCategoryTextBox.Text;
// if (newSubCategory == "")
// {
// return;
// }
// else if (subCategories[category].Contains(newSubCategory))
// {
// return;
// }
// ListItem item = new ListItem(newSubCategory);
// SubCategoryListBox.Items.Add(item);
// AddSubCategoryTextBox.Text = "";
// subCategories[category].Add(newSubCategory);
// Hidden_SubCategories.Value = JSonUtils.ConvertObjectToJSon<Dictionary<String, List<String>>>(subCategories);
//}
//protected void DeleteSubCategoryButton_Click(object sender, EventArgs e)
//{
// string category = CategoryListBox.SelectedValue;
// List<ListItem> subcategoriesToBeDeleted = new List<ListItem>();
// for (int i = 0; i < SubCategoryListBox.Items.Count; i++)
// {
// if (SubCategoryListBox.Items[i].Selected)
// {
// subcategoriesToBeDeleted.Add(SubCategoryListBox.Items[i]);
// }
// }
// for (int i = 0; i < subcategoriesToBeDeleted.Count; i++)
// {
// subCategories[category].Remove(subcategoriesToBeDeleted[i].Text);
// SubCategoryListBox.Items.Remove(subcategoriesToBeDeleted[i]);
// }
// Hidden_SubCategories.Value = JSonUtils.ConvertObjectToJSon<Dictionary<String, List<String>>>(subCategories);
//}
protected void JobSubmitButton_Click(object sender, EventArgs e)
{
string AzureBlobStorageConnectionString = AzureBlobStorageConnectionStringTextBox.Text;
string AzureContainer = AzureBlobStorageContainerNameTextBox.Text;
string AzureContainerDirectory = AzureBlobStorageContainerDirectoryNameTextBox.Text;
string AmazonAccessKeyIDValue = AmazonAccessKeyID.Text;
string AmazonSecretAccessKeyIDValue = AmazonSecretAccessKeyID.Text;
string Description = CategoryDescription.Text;
string jobGuid = NewJobGUID.Text;
string AmazonTaskTitle = AmazonTaskTitleTextBox.Text;
string AmazonTaskDescription = AmazonTaskDescriptionTextBox.Text;
string AmazonTaskKeywords = AmazonTaskKeywordsTextBox.Text;
double pricePerObjectPerChunk = 0;
int noObjectsPerChunk = 0;
string FrameRate = TargetFrameRate.Text;
string chunkLength = ChunkDuration.Text;
string chunkOverlap = ChunkOverlap.Text;
if (AzureBlobStorageConnectionString == "" || AzureContainer == "")
{
Template_ErrorMessageLabel.Text = "Error : Azure Connection String and Container are mandatory fields.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureConnectionInfo connectionInfo = new AzureConnectionInfo(AzureBlobStorageConnectionString, AzureContainer, AzureContainerDirectory);
int noFiles = connectionInfo.getNoFiles();
if (noFiles == -1)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
else if (noFiles == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are 0 files at the Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonMTurkHIT hit = new AmazonMTurkHIT();
bool success = hit.setAccount(AmazonAccessKeyIDValue, AmazonSecretAccessKeyIDValue, false);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double balance = hit.getAccountBalance();
if (balance < 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (balance < 0.1)
{
Template_ErrorMessageLabel.Text = "Error : Insufficient money in Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
}
if (subCategories.Count == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are no Categories";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
List <LineSegment> BoundaryLines = new List <LineSegment>();
if (BoundaryString.Text != "")
{
bool error = false;
string errorMessage = "";
string[] boundaryFields = BoundaryString.Text.Split(',');
for (int i = 0; i < boundaryFields.Length; i++)
{
string[] coords = boundaryFields[i].Split('-');
if (coords.Length != 4)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : does not have 4 values.";
error = true;
break;
}
int x1, y1, x2, y2;
bool success = int.TryParse(coords[0], out x1);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[0] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[1], out y1);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[1] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[2], out x2);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[2] + " must be an integer";
error = true;
break;
}
success = int.TryParse(coords[3], out y2);
if (!success)
{
errorMessage = "In Boundary String, Line No " + (i + 1) + " " + boundaryFields[i] + " : " + coords[3] + " must be an integer";
error = true;
break;
}
LineSegment ls = new LineSegment(x1, y1, x2, y2);
BoundaryLines.Add(ls);
}
if (error)
{
Template_ErrorMessageLabel.Text = errorMessage;
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
}
if (Hidden_MechanicalTurk.Value == "true")
{
bool success = Double.TryParse(PriceTextBox.Text, out pricePerObjectPerChunk);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical price entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
pricePerObjectPerChunk = pricePerObjectPerChunk / 100; //covert to dollars
success = Int32.TryParse(NoObjectsPerChunkTextBox.Text, out noObjectsPerChunk);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical number of objects per image entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (pricePerObjectPerChunk <= 0.0)
{
Template_ErrorMessageLabel.Text = "Error : Illegal Price";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (noObjectsPerChunk <= 0.0)
{
Template_ErrorMessageLabel.Text = "Error : Illegal number of objects";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
}
if (!InputFormatImage.Checked && !InputFormatVideo.Checked)
{
Template_ErrorMessageLabel.Text = "Error : Please select input format";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Convert.ToDouble(FrameRate) <= 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Target Frame Rate";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Convert.ToInt32(chunkLength) <= 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Chunk Duration";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Convert.ToDouble(chunkOverlap) <= 0 || Convert.ToDouble(chunkOverlap) > Convert.ToInt32(chunkLength) / 2)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Chunk Overlap, must be in (0,Chunk/2]";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureInformation AzureInfo = new AzureInformation();
AzureInfo.AzureBlobStorageConnectionString = AzureBlobStorageConnectionString;
AzureInfo.AzureBlobStorageContainerName = AzureContainer;
AzureInfo.AzureBlobStorageContainerDirectoryName = AzureContainerDirectory;
AmazonMTurkHITInformation AmazonInfo = new AmazonMTurkHITInformation();
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonInfo.AmazonAccessKeyID = AmazonAccessKeyIDValue;
AmazonInfo.AmazonSecretAccessKeyID = AmazonSecretAccessKeyIDValue;
AmazonInfo.Price = Math.Floor(pricePerObjectPerChunk * (double)noObjectsPerChunk * 100) / 100;
AmazonInfo.AmazonMTurkTaskTitle = AmazonTaskTitle;
AmazonInfo.AmazonMTurkTaskDescription = AmazonTaskDescription;
AmazonInfo.AmazonMTurkTaskKeywords = AmazonTaskKeywords;
}
else
{
AmazonInfo.AmazonAccessKeyID = "";
AmazonInfo.AmazonSecretAccessKeyID = "";
AmazonInfo.Price = 0;
AmazonInfo.AmazonMTurkTaskTitle = "";
AmazonInfo.AmazonMTurkTaskDescription = "";
AmazonInfo.AmazonMTurkTaskKeywords = "";
}
MultiObjectTrackingSubmittedJob jobParameters = new MultiObjectTrackingSubmittedJob();
jobParameters.Categories = subCategories;
jobParameters.Description = Description;
jobParameters.BoundaryLines = BoundaryLines;
jobParameters.FrameRate = Convert.ToInt32(FrameRate);
jobParameters.ChunkDuration = Convert.ToInt32(chunkLength);
jobParameters.ChunkOverlap = Convert.ToDouble(chunkOverlap);
if (InputFormatVideo.Checked)
{
jobParameters.DataSrcFormat = Constants.DataFormat.Video;
}
else if (InputFormatImage.Checked)
{
jobParameters.DataSrcFormat = Constants.DataFormat.VideoFrame;
}
SatyamJob job = new SatyamJob();
job.UserID = userName;
job.amazonHITInformation = AmazonInfo;
job.azureInformation = AzureInfo;
job.JobGUIDString = jobGuid;
if (Hidden_MechanicalTurk.Value == "true")
{
job.JobTemplateType = TaskConstants.Tracking_MTurk;
job.TasksPerJob = 1;
}
else
{
job.JobTemplateType = TaskConstants.Tracking;
job.TasksPerJob = 0;
}
job.JobSubmitTime = DateTime.Now;
job.JobParameters = JSonUtils.ConvertObjectToJSon <MultiObjectTrackingSubmittedJob>(jobParameters);
string jobDefinition = JSonUtils.ConvertObjectToJSon <SatyamJob>(job);
SatyamJobSubmissionsTableAccess dbAccess = new SatyamJobSubmissionsTableAccess();
dbAccess.AddEntry(job.JobTemplateType, job.UserID, job.JobGUIDString, jobDefinition, job.JobSubmitTime);
dbAccess.close();
Response.Redirect("NewJobSubmitted.aspx");
}
private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
//Thread.Sleep(5000);
int maxDoneScore = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MAX_DONE_SCORE;
if (SubmitButton.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
// TaskConstants.Detection_Image_MTurk, price);
entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPriceAndMaxDoneScore(Hidden_AmazonWorkerID.Value,
TaskConstants.Detection_Image_MTurk, price, MaxDoneScore: maxDoneScore);
}
else
{
//entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk);
entry = taskTableDB.getMinimumTriedEntryByTemplateAndMaxDoneScore(TaskConstants.Detection_Image_MTurk, MaxDoneScore: maxDoneScore);
}
if (entry == null)
{
taskTableDB.close();
return(false);
}
//doneScore = entry.DoneScore;
//taskTableDB.IncrementDoneScore(entry.ID);
taskTableDB.close();
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string uri = task.SatyamURI;
DisplayImage.ImageUrl = uri;
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
CategorySelection_RadioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
DescriptionPanel.Visible = true;
DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
NoLabeled.Text = Hidden_NoImagesDone.Value;
Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString();
/////////////////////////////////Load Previous Turker Results
//MultiObjectLocalizationAndLabelingResult res = LoadLatestTurkerResult(entry);
///////////////////////////////Load Previous Aggregation Results
MultiObjectLocalizationAndLabelingResult res = LoadLatestProgressiveAggregationResult(entry);
//TFServing Backend
if (res == null && TFServingBackend)
{
res = LoadTFServingResult(entry);
Hidden_PrevResultID.Value = "-1"; // means TF
}
if (res == null)
{
Hidden_PrevResults.Value = "[]";
Hidden_PrevResultID.Value = 0.ToString();
}
else
{
string prevBoxes = JSonUtils.ConvertObjectToJSon(res.objects);
Hidden_ImageHeight.Value = res.imageHeight.ToString();
Hidden_ImageWidth.Value = res.imageWidth.ToString();
Hidden_PrevResults.Value = prevBoxes;
}
return(true);
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
// sort by task id
SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
for (int i = 0; i < entries.Count; i++)
{
if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID))
{
taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]);
}
foreach (int taskID in taskResults.Keys)
{
for (int i = 0; i < taskResults[taskID].Count; i++)
{
SatyamResultsTableEntry entry = taskResults[taskID][i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (res == null)
{
continue;
}
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID;
}
fileName += "-" + entry.ID;
if (File.Exists(directoryName + "\\" + fileName + ".jpg"))
{
continue;
}
Console.WriteLine("Saving Turker Result {0}", fileName);
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultsOnImage(res, originalImage);
byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res);
if (ResultImage == null)
{
continue;
}
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png);
}
}
}
public static void CreateDemoJob_MULTI_OBJECT_LOCALIZATION_AND_LABLING()
{
//first Job
SatyamJob job = new SatyamJob();
AzureInformation AzureInfo = new AzureInformation();
AzureInfo.AzureBlobStorageConnectionString = SatyamJobStorageAccountAccess.connection_string;
AzureInfo.AzureBlobStorageContainerName = "demo";
AzureInfo.AzureBlobStorageContainerDirectoryName = "catsAndDogs";
AmazonMTurkHITInformation AmazonInfo = new AmazonMTurkHITInformation();
MultiObjectLocalizationAndLabelingSubmittedJob template = new MultiObjectLocalizationAndLabelingSubmittedJob();
List <string> categories = new List <string>()
{
"Cat", "Dog"
};
template.Categories = categories;
template.Description = "";
template.BoundaryLines = new List <LineSegment>();
job.JobGUIDString = "MULTI_OBJECT_LOCALIZATION_AND_LABLING_DEMO";
job.amazonHITInformation = AmazonInfo;
job.azureInformation = AzureInfo;
job.JobSubmitTime = DateTime.Now;
job.JobTemplateType = "MULTI_OBJECT_LOCALIZATION_AND_LABLING_DEMO";
job.UserID = TaskConstants.AdminName;
job.JobParameters = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingSubmittedJob>(template);
string JobParametersString = JSonUtils.ConvertObjectToJSon <SatyamJob>(job);
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
jobDB.AddEntry(job.JobTemplateType, job.UserID, job.JobGUIDString, JobParametersString, job.JobSubmitTime);
//second job
SatyamJob job1 = new SatyamJob();
AzureInformation AzureInfo1 = new AzureInformation();
AzureInfo1.AzureBlobStorageConnectionString = SatyamJobStorageAccountAccess.connection_string;
AzureInfo1.AzureBlobStorageContainerName = "demo";
AzureInfo1.AzureBlobStorageContainerDirectoryName = "TrafficExample-EastGate";
AmazonMTurkHITInformation AmazonInfo1 = new AmazonMTurkHITInformation();
MultiObjectLocalizationAndLabelingSubmittedJob template1 = new MultiObjectLocalizationAndLabelingSubmittedJob();
List <string> categories1 = new List <string>()
{
"Vehicle", "Person"
};
template1.Categories = categories1;
template1.Description = "Vehicles can be buses, cars, trucks, motorbikes but not bi-cylces or wheelchairs. Person can be pedestrian or a cylist.";
template1.BoundaryLines = new List <LineSegment>();
template1.BoundaryLines.Add(new LineSegment(1, 489, 291, 191));
template1.BoundaryLines.Add(new LineSegment(291, 191, 435, 103));
template1.BoundaryLines.Add(new LineSegment(435, 103, 664, 68));
template1.BoundaryLines.Add(new LineSegment(664, 68, 883, 127));
template1.BoundaryLines.Add(new LineSegment(883, 127, 915, 205));
template1.BoundaryLines.Add(new LineSegment(915, 205, 1276, 497));
template1.BoundaryLines.Add(new LineSegment(1276, 497, 1276, 694));
template1.BoundaryLines.Add(new LineSegment(1276, 694, 1, 696));
template1.BoundaryLines.Add(new LineSegment(1, 696, 1, 489));
job1.JobGUIDString = "MULTI_OBJECT_LOCALIZATION_AND_LABLING_DEMO";
job1.amazonHITInformation = AmazonInfo1;
job1.azureInformation = AzureInfo1;
job1.JobSubmitTime = DateTime.Now;
job1.JobTemplateType = "MULTI_OBJECT_LOCALIZATION_AND_LABLING_DEMO";
job1.UserID = TaskConstants.AdminName;
job1.JobParameters = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingSubmittedJob>(template1);
string JobParametersString1 = JSonUtils.ConvertObjectToJSon <SatyamJob>(job1);
jobDB.AddEntry(job1.JobTemplateType, job1.UserID, job1.JobGUIDString, JobParametersString1, job1.JobSubmitTime);
jobDB.close();
}
public static SatyamResultPerJobDataAnalysis AnalyzeFromResultsTable(List <string> jobGUIDs)
{
Console.WriteLine("Analyzing Results Statistics of {0}", jobGUIDs[0]);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in jobGUIDs)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
if (entries.Count == 0)
{
return(null);
}
SatyamResultPerJobDataAnalysis ana = new SatyamResultPerJobDataAnalysis();
ana.JobGUID = entries[0].JobGUID;
ana.totalResults = entries.Count;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (ana.job == null)
{
ana.job = job;
}
ana.jobSubmitTime = job.JobSubmitTime;
double moneyPerResult = 0;
if (job.TasksPerJob > 0)
{
moneyPerResult = (double)satyamResult.amazonInfo.PricePerHIT / (double)job.TasksPerJob;
}
ana.PricePerResult.Add(entry.ID, moneyPerResult);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!ana.noResultsPerTask.ContainsKey(taskEntryID))
{
ana.noResultsPerTask.Add(taskEntryID, 0);
ana.noAcceptedResultsPerTask.Add(taskEntryID, 0);
ana.noPaidResultsPerTask.Add(taskEntryID, 0);
ana.moneyPaidPerTask.Add(taskEntryID, 0);
ana.finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ana.ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
ana.noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
ana.taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
ana.noResultsPerTask[taskEntryID]++;
ana.SequenceNumberOfResultPerTask.Add(ana.noResultsPerTask[taskEntryID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedResultsPerTask[taskEntryID]++;
ana.totalAcceptedResults++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidResultsPerTask[taskEntryID]++;
ana.totalpaidResults++;
ana.moneyPaidPerTask[taskEntryID] += moneyPerResult;
ana.totalMoneyPaid += moneyPerResult;
}
if (ana.finalTaskEndTime[taskEntryID] < entry.SubmitTime)
{
ana.finalTaskEndTime[taskEntryID] = entry.SubmitTime;
}
if (ana.lastTaskEndTime < entry.SubmitTime)
{
ana.lastTaskEndTime = entry.SubmitTime;
}
TimeSpan dt = satyamResult.TaskEndTime - satyamResult.TaskStartTime;
ana.timeTakenPerResult.Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerSequenceNumber.ContainsKey(ana.noResultsPerTask[taskEntryID]))
{
ana.AcceptedTimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
ana.TimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
}
//ana.acceptedTimeTakenPerResultOrganizedByResultSequencePerTask[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
ana.TimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerTask.ContainsKey(taskEntryID))
{
ana.AcceptedTimePerTask.Add(taskEntryID, new List <double>());
ana.TimePerTask.Add(taskEntryID, new List <double>());
}
//ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.TimePerTask[taskEntryID].Add(dt.TotalSeconds);
if (ana.lastTaskEndTime < satyamResult.TaskEndTime)
{
ana.lastTaskEndTime = satyamResult.TaskEndTime;
}
if (!ana.finalHITEndTime.ContainsKey(satyamResult.amazonInfo.HITID))
{
ana.finalHITEndTime.Add(satyamResult.amazonInfo.HITID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
if (ana.finalHITEndTime[satyamResult.amazonInfo.HITID] < (entry.SubmitTime - job.JobSubmitTime).TotalSeconds)
{
ana.finalHITEndTime[satyamResult.amazonInfo.HITID] = (entry.SubmitTime - job.JobSubmitTime).TotalSeconds;
}
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.acceptedTimeTakenPerResult.Add(dt.TotalSeconds);
ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.AcceptedTimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
}
ana.resultArrivalTimes.Add((entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
if (!ana.noJobsPerWorker.ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.noJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noAcceptedJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noPaidJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
}
ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
ana.SequenceNumberOfResultPerWorker.Add(ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
else
{
// this worker has done this task before
ana.noDuplicateResultsFromSameWorker++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
if (ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID] > 1)
{
ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask++;
}
}
ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
///////////////// imagenet only
//List<SingleObjectLabelingResult> CurrentResultsOfThisTask = new List<SingleObjectLabelingResult>();
//foreach(string worker in ana.ResultsPerWorkerPerTask[taskEntryID].Keys)
//{
// foreach(string workerResult in ana.ResultsPerWorkerPerTask[taskEntryID][worker])
// {
// CurrentResultsOfThisTask.Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(workerResult));
// }
//}
//if (SingleObjectLabelingAggregator.getAggregatedResult(CurrentResultsOfThisTask) != null)
//{
// // aggregation happened
// if (!ana.timeTakenTillAggregationPerTask.ContainsKey(taskEntryID))
// {
// // for the first time aggregated
// ana.timeTakenTillAggregationPerTask.Add(taskEntryID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
// }
//}
///////////////////////
}
ana.resultArrivalTimes.Sort();
ana.noTasks = ana.noResultsPerTask.Count;
ana.timeTakenPerResult.Sort();
ana.medianTimeTakenPerResultInSeconds = ana.timeTakenPerResult[ana.timeTakenPerResult.Count / 2];
if (ana.acceptedTimeTakenPerResult.Count != 0)
{
ana.acceptedTimeTakenPerResult.Sort();
ana.medianAcceptedTimeTakenPerResultInSeconds = ana.acceptedTimeTakenPerResult[ana.acceptedTimeTakenPerResult.Count / 2];
}
ana.totalTimeTakenForJob = ana.lastTaskEndTime - ana.jobSubmitTime;
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
ana.resultsPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
ana.moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
ana.ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
int noAggregatedTasks = 0;
List <int> taskIDs = ana.noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (!ana.resultsPerTaskHistogram.ContainsKey(ana.noResultsPerTask[taskID]))
{
ana.resultsPerTaskHistogram.Add(ana.noResultsPerTask[taskID], 0);
}
ana.resultsPerTaskHistogram[ana.noResultsPerTask[taskID]]++;
if (!ana.resultsAcceptedPerTaskHistogram.ContainsKey(ana.noAcceptedResultsPerTask[taskID]))
{
ana.resultsAcceptedPerTaskHistogram.Add(ana.noAcceptedResultsPerTask[taskID], 0);
}
ana.resultsAcceptedPerTaskHistogram[ana.noAcceptedResultsPerTask[taskID]]++;
if (!ana.resultsPaidPerTaskHistogram.ContainsKey(ana.noPaidResultsPerTask[taskID]))
{
ana.resultsPaidPerTaskHistogram.Add(ana.noPaidResultsPerTask[taskID], 0);
}
ana.resultsPaidPerTaskHistogram[ana.noPaidResultsPerTask[taskID]]++;
int moneyPaid = (int)Math.Floor(ana.moneyPaidPerTask[taskID] * 100);
if (!ana.moneyPaidPerTaskHistogram.ContainsKey(moneyPaid))
{
ana.moneyPaidPerTaskHistogram.Add(moneyPaid, 0);
}
ana.moneyPaidPerTaskHistogram[moneyPaid]++;
if (ana.ResultsPerWorkerPerTask[taskID].Count != ana.noResultsPerTask[taskID])
{
//has multiple results from same turker
ana.noTasksWithDuplicateResultsFromSameWorker++;
}
if (ana.noPaidResultsPerTask[taskID] != 0)
{
noAggregatedTasks++;
}
}
ana.avgNoPaidResultsPerTask = (double)ana.totalpaidResults / (double)noAggregatedTasks;
//////////// duplicate analysis
//Console.WriteLine("DuplicateResultsHistogram");
//foreach (int no in ana.ResultsPerWorkerPerTaskHistogram.Keys)
//{
// Console.WriteLine("{0}, {1}", no, ana.ResultsPerWorkerPerTaskHistogram[no]);
//}
//Console.WriteLine("{0} images (in total {1}({2})) has duplicate results from same turker, \n" +
// "\t{3} images have >=1 workers making more than majority number of results.\n" +
// "\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
// "\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
// //"\t\t{6} images has correct decision among duplicate decisions.\n" +
// //"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
// //"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
// ana.noTasksWithDuplicateResultsFromSameWorker, ana.noDuplicateResultsFromSameWorker, 0,
// ana.noTaskWithDuplicateResultsAsMajority,
// ana.noTasksWithMixedResultsFromSameWorker,ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
// ana.noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask
// //ana.noTasksWithWorkerSwitchedToCorrect, ana.noWorkerSwitchedToCorrect,
// //ana.noTasksWithWorkerSwitchedToCorrectAndMaintained, ana.noWorkerSwitchedToCorrectAndMaintained,
// //ana.noTaskWhoseDuplicateResultsChangedAggregation, ana.noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
//);
//////////////////////////////
string outputDir = DirectoryConstants.defaultTempDirectory + jobGUIDs[0] + "\\ResultStatistics\\";
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
//Console.WriteLine("timePerResult");
TextWriter tw = new StreamWriter(outputDir + "timePerResult.txt");
foreach (double time in ana.timeTakenPerResult)
{
tw.WriteLine(time);
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerSequenceNumber.txt");
foreach (int seq in ana.AcceptedTimePerSequenceNumber.Keys)
{
foreach (double time in ana.AcceptedTimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerTask.txt");
foreach (int taskid in ana.AcceptedTimePerTask.Keys)
{
foreach (double time in ana.AcceptedTimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerSequenceNumber.txt");
foreach (int seq in ana.TimePerSequenceNumber.Keys)
{
foreach (double time in ana.TimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerTask.txt");
foreach (int taskid in ana.TimePerTask.Keys)
{
foreach (double time in ana.TimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
//Console.WriteLine("resultArrivalTime");
TextWriter tw0 = new StreamWriter(outputDir + "resultArrivalTime.txt");
foreach (double time in ana.resultArrivalTimes)
{
tw0.WriteLine(time);
}
tw0.Close();
//Console.WriteLine("HITFinishTime");
TextWriter tw2 = new StreamWriter(outputDir + "HITFinishTime.txt");
foreach (double time in ana.finalHITEndTime.Values)
{
tw2.WriteLine(time);
}
tw2.Close();
//Console.WriteLine("SequenceNumberOfResultsPerTask");
TextWriter tw3 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerTask.txt");
foreach (int seq in ana.SequenceNumberOfResultPerTask)
{
tw3.WriteLine(seq);
}
tw3.Close();
//Console.WriteLine("SequenceNumberOfResultsPerWorker");
TextWriter tw4 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerWorker.txt");
foreach (int seq in ana.SequenceNumberOfResultPerWorker)
{
tw4.WriteLine(seq);
}
tw4.Close();
//Console.WriteLine("MoneyPaidPerResult");
TextWriter tw5 = new StreamWriter(outputDir + "MoneyPaidPerResult.txt");
foreach (int seq in ana.PricePerResult.Keys)
{
tw5.WriteLine(ana.PricePerResult[seq]);
}
tw5.Close();
TextWriter tw6 = new StreamWriter(outputDir + "acceptedTimePerResult.txt");
foreach (double time in ana.acceptedTimeTakenPerResult)
{
tw6.WriteLine(time);
}
tw6.Close();
///////////////////// aggregation time histogram, out of scope
//Console.WriteLine("resultArrivalTime",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//string filepath = String.Format(@"C:\research\MSRDVA\SatyamResearch\aggregationTimePerTask_MinNoResults_{0}_MaxNoResults_{1}.txt",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//TextWriter tw1 = new StreamWriter(filepath);
//foreach (double time in ana.timeTakenTillAggregationPerTask.Values)
//{
// tw1.WriteLine(time);
//}
//tw1.Close();
Console.WriteLine("Median Time For Accepted Task: {0}", ana.medianAcceptedTimeTakenPerResultInSeconds);
Console.WriteLine("Median Time For All Task: {0}", ana.medianTimeTakenPerResultInSeconds);
Console.WriteLine("Total Tasks: {0}", ana.noTasks);
Console.WriteLine("Total Accepted Results: {0}", ana.totalAcceptedResults);
Console.WriteLine("Total Results: {0}", ana.totalResults);
Console.WriteLine("Avg Paid Results Per Task: {0}", ana.avgNoPaidResultsPerTask);
return(ana);
}
private bool getNewRandomJob()
{
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Tracking);
if (entry != null)
{
taskTableDB.IncrementDoneScore(entry.ID);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI);
//string urls = "";
//for (int i=0;i<ImageURLs.Count;i++)
//{
// urls += ImageURLs[i];
// if (i == ImageURLs.Count - 1) break;
// urls += ',';
//}
Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ',');
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters);
Dictionary <string, List <string> > subcategories = job.Categories;
List <string> categories = subcategories.Keys.ToList();
//CategorySelection_RakdioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
//CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
//DescriptionPanel.Visible = true;
//DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
//Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
// pass parameters from old template
Slug_Hidden.Value = "null";
Start_Hidden.Value = "0";
Stop_Hidden.Value = (ImageURLs.Count - 1).ToString();
Skip_Hidden.Value = "0";
PerObject_Hidden.Value = "0.1";
Completion_Hidden.Value = "0.5";
BlowRadius_Hidden.Value = "0";
JobId_Hidden.Value = "1";
LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ',');
Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_');
Training_Hidden.Value = "0";
fps_Hidden.Value = job.FrameRate.ToString();
Hidden_ChunkDuration.Value = job.ChunkDuration.ToString();
var web = new WebClient();
System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0]));
ImageWidth_Hidden.Value = x.Width.ToString();
ImageHeight_Hidden.Value = x.Height.ToString();
// image boundary for now
//string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" };
string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" };
RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ',');
taskTableDB.close();
return(true);
}
else
{
taskTableDB.close();
return(false);
}
}
//public static void AnalyzeApprovalRateByGUID(List<string> guids)
//{
// Console.WriteLine("Analyzing Results Statistics of {0}", guids[0]);
// SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
// List<SatyamResultsTableEntry> entries = new List<SatyamResultsTableEntry>();
// foreach (string guid in guids)
// {
// entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
// }
// resultsDB.close();
//}
public static void AggregationAnalysis(List <SatyamAggregatedResultsTableEntry> aggEntries,
//List<SatyamResultsTableEntry> ResultEntries,
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime,
Dictionary <int, int> noResultsNeededForAggregation,
Dictionary <int, int> noResultsNeededForAggregation_new,
string guid, string configString,
int minChancesGiven = 3,
double anotherChanceProbablity = 0,
double approvalRatioThreshold = 0.5,
bool usePreApprovalResult = false)
{
if (noResultsNeededForAggregation == null)
{
noResultsNeededForAggregation = new Dictionary <int, int>();
}
if (noResultsNeededForAggregation_new == null)
{
noResultsNeededForAggregation_new = new Dictionary <int, int>();
}
string approvalString = configString +
"_Thresh_" + approvalRatioThreshold +
"_Explore_" + anotherChanceProbablity;
Dictionary <int, SatyamAggregatedResultsTableEntry> aggEntriesByTaskID = new Dictionary <int, SatyamAggregatedResultsTableEntry>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultEntriesByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
//Dictionary<DateTime, double> TotalApprovalRatioByTime = new Dictionary<DateTime, double>();
List <string> TotalApprovalRatioByTime = new List <string>();
List <string> TotalApprovalRatioByResult = new List <string>();
List <string> FilteredTotalApprovalRatioByTime = new List <string>();
List <string> FilteredTotalApprovalRatioByResult = new List <string>();
List <SatyamResultsTableEntry> approvedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> InconclusiveEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> approvedEntriesActuallyPaid = new List <SatyamResultsTableEntry>();
Dictionary <string, int> noTasksApprovedPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noTasksDonePerWorker = new Dictionary <string, int>();
Dictionary <string, double> ApprovalRatioPerWorker = new Dictionary <string, double>();
Dictionary <string, bool> FirstTimersWorkersRecord = new Dictionary <string, bool>();
List <SatyamResultsTableEntry> FilteredApprovedEntries = new List <SatyamResultsTableEntry>();
List <SatyamResultsTableEntry> filteredRejectedEntries = new List <SatyamResultsTableEntry>();
Dictionary <int, double> AggregationTimePerTask = new Dictionary <int, double>();
foreach (SatyamAggregatedResultsTableEntry aggEntry in aggEntries)
{
if (!aggEntriesByTaskID.ContainsKey(aggEntry.SatyamTaskTableEntryID))
{
aggEntriesByTaskID.Add(aggEntry.SatyamTaskTableEntryID, aggEntry);
}
else
{
// duplicate agg entry for a task
}
}
//Dictionary<int, int> noResultsNeededForAggregation = getNoResultsNeededForAggregationFromLog(approvalString, guid);
foreach (int k in noResultsNeededForAggregation_new.Keys)
{
noResultsNeededForAggregation.Add(k, noResultsNeededForAggregation_new[k]);
}
int count = 0;
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
Console.WriteLine("Judging result: {0}/{1}", count, entriesBySubmitTime.Count);
count++;
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry res in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(res.ResultString);
int taskID = res.SatyamTaskTableEntryID;
if (!ResultEntriesByTaskID.ContainsKey(taskID))
{
ResultEntriesByTaskID.Add(taskID, new List <SatyamResultsTableEntry>());
}
ResultEntriesByTaskID[taskID].Add(res);
if (!aggEntriesByTaskID.ContainsKey(taskID))
{
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//InconclusiveEntries.Add(res);
//}
InconclusiveEntries.Add(res);
continue;
}
SatyamAggregatedResultsTableEntry agg = aggEntriesByTaskID[taskID];
//foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
//{
//SatyamResult satyamResult = JSonUtils.ConvertJSonToObject<SatyamResult>(res.ResultString);
string workerId = satyamResult.amazonInfo.WorkerID;
// the worker pre-filtering step
bool filtered = false;
if (noTasksDonePerWorker.ContainsKey(workerId) && noTasksDonePerWorker[workerId] >= minChancesGiven)
{
double approvalRatio = (double)noTasksApprovedPerWorker[workerId] / (double)noTasksDonePerWorker[workerId];
ApprovalRatioPerWorker[workerId] = approvalRatio;
if (approvalRatio < approvalRatioThreshold)
{
// roll a dice to decide whether to skip this worker
int rnd = new Random().Next(100);
if (rnd > 100 * anotherChanceProbablity)
{
filtered = true;
}
else
{
// another chance granted
}
}
}
if (!noTasksDonePerWorker.ContainsKey(workerId))
{
noTasksDonePerWorker.Add(workerId, 0);
noTasksApprovedPerWorker.Add(workerId, 0);
ApprovalRatioPerWorker.Add(workerId, -1);
}
if (!filtered)
{
noTasksDonePerWorker[workerId]++;
}
bool acceptable = false;
if (usePreApprovalResult)
{
if (ResultStatus.acceptedStatusList.Contains(res.Status))
{
acceptable = true;
}
}
else
{
acceptable = AcceptanceCriterionChecker.IsAcceptable(agg, res);
}
if (acceptable)
{
approvedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, true);
}
if (!filtered)
{
noTasksApprovedPerWorker[workerId]++;
FilteredApprovedEntries.Add(res);
}
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count <= noResultsNeededForAggregation[taskID])
{
approvedEntriesActuallyPaid.Add(res);
}
}
else
{
rejectedEntries.Add(res);
if (!FirstTimersWorkersRecord.ContainsKey(workerId))
{
FirstTimersWorkersRecord.Add(workerId, false);
}
if (!filtered)
{
filteredRejectedEntries.Add(res);
}
}
//}
/// timetaken till aggregation
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
ResultEntriesByTaskID[taskID].Count == noResultsNeededForAggregation[taskID])
{
// aggregation happen
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (!AggregationTimePerTask.ContainsKey(taskID))
{
AggregationTimePerTask.Add(taskID, (res.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
}
string ApprovalString = approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByResult.Add(ApprovalString);
string filteredApprovalString = FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByResult.Add(filteredApprovalString);
}
string approvalStringByTime = t.ToString() + " " + approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
+ (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
TotalApprovalRatioByTime.Add(approvalStringByTime);
string filteredApprovalStringByTime = t.ToString() + " " + FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
+ (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
FilteredTotalApprovalRatioByTime.Add(filteredApprovalStringByTime);
}
// count first timer succ rate
int FirstTimeSucc = 0;
foreach (KeyValuePair <string, bool> wid in FirstTimersWorkersRecord)
{
if (wid.Value)
{
FirstTimeSucc++;
}
}
double FirstTimeSuccRate = (double)FirstTimeSucc / FirstTimersWorkersRecord.Count;
Console.WriteLine("Approval #, ratio: {0},{1}", approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count));
Console.WriteLine("Filtered Approval #, ratio: {0},{1}", FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count));
int approvePrevRejected = 0;
List <string> ApprovedAssignmentIDs = new List <string>();
foreach (SatyamResultsTableEntry approved in approvedEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(approved.ResultString);
ApprovedAssignmentIDs.Add(satyamResult.amazonInfo.AssignmentID);
if (approved.Status == ResultStatus.accepted_NotPaid || approved.Status == ResultStatus.rejected_NotPaid || approved.Status == ResultStatus.rejected)
{
approvePrevRejected++;
}
}
Console.WriteLine("Approving Previously Rejected: {0}, {1}", approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
string outputFileDir = DirectoryConstants.defaultTempDirectory + guid + "\\ApprovalAnalysis\\";
if (guid != null && approvalString != null)
{
if (!Directory.Exists(outputFileDir))
{
Directory.CreateDirectory(outputFileDir);
}
}
string summaryFile = outputFileDir + "resultSummary.txt";
string summary = String.Format("{0} {1} {2} {3} {4} {5} {6} {7}\n",
approvalString,
approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count),
approvedEntriesActuallyPaid.Count,
FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count),
approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);
File.AppendAllText(summaryFile, summary);
string outputFile = outputFileDir + approvalString + ".txt";
File.WriteAllLines(outputFile, ApprovedAssignmentIDs.ToArray());
//string outputFilePaid = outputFileDir + approvalString + "_ActuallyPaid.txt";
//File.WriteAllText(outputFilePaid, approvedEntriesActuallyPaid.Count.ToString());
List <string> approvalRates = new List <string>();
foreach (string worker in ApprovalRatioPerWorker.Keys)
{
approvalRates.Add(ApprovalRatioPerWorker[worker].ToString());
}
string approvalDistributionFile = outputFileDir + approvalString + "_dist.txt";
File.WriteAllLines(approvalDistributionFile, approvalRates.ToArray());
string approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime.txt";
string approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults.txt";
File.WriteAllLines(approvalByTimeFile, TotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, TotalApprovalRatioByResult.ToArray());
approvalByTimeFile = outputFileDir + approvalString + "_approvalRateByTime_filtered.txt";
approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults_filtered.txt";
File.WriteAllLines(approvalByTimeFile, FilteredTotalApprovalRatioByTime.ToArray());
File.WriteAllLines(approvalByResultFile, FilteredTotalApprovalRatioByResult.ToArray());
List <string> aggregationTime = new List <string>();
foreach (int task in AggregationTimePerTask.Keys)
{
aggregationTime.Add(task.ToString() + " " + AggregationTimePerTask[task].ToString());
}
string aggregationTimeFile = outputFileDir + approvalString + "_aggregationTime.txt";
File.WriteAllLines(aggregationTimeFile, aggregationTime.ToArray());
}
public static List <SatyamTask> generate(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
//BlobContainerManager bcm = new BlobContainerManager();
//string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
//if (status != "SUCCESS")
//{
// return null;
//}
////first copy all the images to satyam location
//List<string> FileTypes = ValidFileTypesByTemplate[job.JobTemplateType];
////Now get all the URIs at the remote location that are relevant
//List<string> FullURIList = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
//List<string> URIList = new List<string>();
//List<string> fileNameList = new List<string>();
//foreach(string uri in FullURIList)
//{
// string filename = URIUtilities.filenameFromURI(uri);
// if(CheckFileExtensions.IsAllowedType(filename,FileTypes))
// {
// URIList.Add(uri);
// fileNameList.Add(filename);
// }
//}
////Now create a dictionary mapping filename to URI for both the locations
//Dictionary<string, string> fileNameToURI = new Dictionary<string, string>();
//for (int i = 0; i < URIList.Count; i++)
//{
// string fileName = URIUtilities.filenameFromURI(URIList[i]);
// fileNameToURI.Add(fileName, URIList[i]);
//}
List <string> satyamURIList = new List <string>();
switch (job.JobTemplateType)
{
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
satyamURIList = satyamStorage.getImmediateNextLevelURLList(satyamContainerName, satyamDirectoryName); // list only immediate next level subblobs, directories / files
break;
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
satyamURIList = satyamStorage.getURLListOfSpecificExtension(satyamContainerName, satyamDirectoryName, new List <string>()
{
"txt"
}); // list only the annotation files per chunk per object
break;
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
default:
// all the images to satyam location
satyamURIList = satyamStorage.getURLList(satyamContainerName, satyamDirectoryName);
break;
}
//Dictionary<string, string> satyamFileNameToURI = new Dictionary<string, string>();
//for (int i = 0; i < satyamURIList.Count; i++)
//{
// string satyamFileName = URIUtilities.filenameFromURI(satyamURIList[i]);
// satyamFileNameToURI.Add(satyamFileName, satyamURIList[i]);
//}
//List<string> fileNameList = fileNameToURI.Keys.ToList();
//now create a task for each of these one by one
List <SatyamTask> tasks = new List <SatyamTask>();
foreach (string uri in satyamURIList)
{
SatyamTask task = new SatyamTask();
task.jobEntry = job;
//task.OriginalURI = fileNameToURI[filename];
task.SatyamURI = uri;
tasks.Add(task);
}
return(tasks);
}
/// <summary>
/// Input should be a folder of videos and corresponding annotation file with the same name
/// </summary>
/// <param name="jobEntry"></param>
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >();
foreach (string videolink in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(videolink);
if (!videos.ContainsKey(videoName))
{
videos.Add(videoName, new List <string>());
}
videos[videoName].Add(videolink);
}
foreach (string videoName in videos.Keys)
{
// filter out those that doesn't provide a annotation file with it....
if (videos[videoName].Count != 2)
{
Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName);
//Directory.Delete(guidFolder, true);
//return false;
continue;
}
string videoURL = "";
string annotationURL = "";
foreach (string fileLink in videos[videoName])
{
string extension = URIUtilities.fileExtensionFromURI(fileLink);
if (extension != "txt")
{
videoURL = fileLink;
}
else
{
annotationURL = fileLink;
}
}
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL);
client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
File.Delete(outputDirectory + "\\" + videoNameWithExtension);
List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk);
//parse VIRAT annotation file
string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL);
client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension);
parseAnnotationFileIntoChunkFolders(chunkFolders,
outputDirectory + "\\" + annotationNameWithExtension,
noFramePerChunk, outputFPS);
//upload
for (int i = 0; i < chunkFolders.Count; i++)
{
string subDir = chunkFolders[i];
satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
}
Directory.Delete(outputDirectory, true);
}
return(true);
}
protected void JobSubmitButton_Click(object sender, EventArgs e)
{
string AzureBlobStorageConnectionString = AzureBlobStorageConnectionStringTextBox.Text;
string AzureContainer = AzureBlobStorageContainerNameTextBox.Text;
string AzureContainerDirectory = AzureBlobStorageContainerDirectoryNameTextBox.Text;
string AmazonAccessKeyIDValue = AmazonAccessKeyID.Text;
string AmazonSecretAccessKeyIDValue = AmazonSecretAccessKeyID.Text;
string Description = CategoryDescription.Text;
double pricePerImage = 0;
double price = 0;
string jobGuid = NewJobGUID.Text;
string AmazonTaskTitle = AmazonTaskTitleTextBox.Text;
string AmazonTaskDescription = AmazonTaskDescriptionTextBox.Text;
string AmazonTaskKeywords = AmazonTaskKeywordsTextBox.Text;
if (AzureBlobStorageConnectionString == "" || AzureContainer == "")
{
Template_ErrorMessageLabel.Text = "Error : Azure Connection String and Container are mandatory fields.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AzureConnectionInfo connectionInfo = new AzureConnectionInfo(AzureBlobStorageConnectionString, AzureContainer, AzureContainerDirectory);
int noFiles = connectionInfo.getNoFiles();
if (noFiles == -1)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
else if (noFiles == 0)
{
Template_ErrorMessageLabel.Text = "Error : There are 0 files at the Azure Location";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (Hidden_MechanicalTurk.Value == "true")
{
bool success = Double.TryParse(PriceTextBox.Text, out pricePerImage);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Non-numerical price entered";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
pricePerImage = pricePerImage / 100; //convert to dollars
price = pricePerImage;
//need to round to cents
price = Math.Floor(price * 100) / 100;
if (price <= 0)
{
Template_ErrorMessageLabel.Text = "Error : The Price is Zero";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
AmazonMTurkHIT hit = new AmazonMTurkHIT();
success = hit.setAccount(AmazonAccessKeyIDValue, AmazonSecretAccessKeyIDValue, false);
if (!success)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double balance = hit.getAccountBalance();
if (balance < 0)
{
Template_ErrorMessageLabel.Text = "Error : Invalid Amazon Turk Account";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
double moneyNeeded = 4 * price * noFiles / TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_IMAGES_PER_TASK;
if (balance < moneyNeeded)
{
Template_ErrorMessageLabel.Text = "Error : Insufficient money in Amazon Turk Account. You will need at least " + moneyNeeded + "$.";
Template_ErrorMessageLabel.ForeColor = System.Drawing.Color.Red;
Template_ErrorMessageLabel.Font.Bold = true;
return;
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Image Classification Task";
}
if (AmazonTaskTitle == "")
{
Template_ErrorMessageLabel.Text = "Quickly earn money by simply selecting what an image looks like!";
}
if (AmazonTaskKeywords == "Pictures, Classify, Categorize")
{
Template_ErrorMessageLabel.Text = "";
}
}
AzureInformation AzureInfo = new AzureInformation();
AzureInfo.AzureBlobStorageConnectionString = AzureBlobStorageConnectionString;
AzureInfo.AzureBlobStorageContainerName = AzureContainer;
AzureInfo.AzureBlobStorageContainerDirectoryName = AzureContainerDirectory;
AmazonMTurkHITInformation AmazonInfo = new AmazonMTurkHITInformation();
if (Hidden_MechanicalTurk.Value == "true")
{
AmazonInfo.AmazonAccessKeyID = AmazonAccessKeyIDValue;
AmazonInfo.AmazonSecretAccessKeyID = AmazonSecretAccessKeyIDValue;
AmazonInfo.Price = price; //in dollars
AmazonInfo.AmazonMTurkTaskTitle = AmazonTaskTitle;
AmazonInfo.AmazonMTurkTaskDescription = AmazonTaskDescription;
AmazonInfo.AmazonMTurkTaskKeywords = AmazonTaskKeywords;
}
else
{
AmazonInfo.AmazonAccessKeyID = "";
AmazonInfo.AmazonSecretAccessKeyID = "";
AmazonInfo.Price = 0;
AmazonInfo.AmazonMTurkTaskTitle = "";
AmazonInfo.AmazonMTurkTaskDescription = "";
AmazonInfo.AmazonMTurkTaskKeywords = "";
}
SingleObjectLabelingSubmittedJob jobParameters = new SingleObjectLabelingSubmittedJob();
jobParameters.Description = Description;
SatyamJob job = new SatyamJob();
job.UserID = userName;
job.amazonHITInformation = AmazonInfo;
job.azureInformation = AzureInfo;
job.JobGUIDString = jobGuid;
if (Hidden_MechanicalTurk.Value == "true")
{
job.JobTemplateType = TaskConstants.CameraPoseEsitmation_MTurk;
job.TasksPerJob = 1;
}
else
{
job.JobTemplateType = TaskConstants.CameraPoseEsitmation;
job.TasksPerJob = 0; //does not matter
}
job.JobSubmitTime = DateTime.Now;
job.JobParameters = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingSubmittedJob>(jobParameters);
string jobDefinition = JSonUtils.ConvertObjectToJSon <SatyamJob>(job);
SatyamJobSubmissionsTableAccess dbAccess = new SatyamJobSubmissionsTableAccess();
dbAccess.AddEntry(job.JobTemplateType, job.UserID, job.JobGUIDString, jobDefinition, job.JobSubmitTime);
dbAccess.close();
Response.Redirect("NewJobSubmitted.aspx");
}
private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
//Thread.Sleep(5000);
if (SubmitButton.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice(TaskConstants.Detection_Image_MTurk, price);
//entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
// TaskConstants.Detection_Image_MTurk, price, doneScore);
entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
TaskConstants.Segmentation_Image_MTurk, price);
}
else
{
entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Segmentation_Image_MTurk);
}
taskTableDB.close();
if (entry == null)
{
return(false);
}
taskTableDB = new SatyamTaskTableAccess();
taskTableDB.IncrementDoneScore(entry.ID);
taskTableDB.close();
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string uri = task.SatyamURI;
TheImage.ImageUrl = uri;
SatyamJob jobDefinitionEntry = task.jobEntry;
ImageSegmentationSubmittedJob job = JSonUtils.ConvertJSonToObject <ImageSegmentationSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
CategorySelection_RadioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
DescriptionPanel.Visible = true;
DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
return(true);
}
