private void populateRows()
{
SatyamJobSubmissionsTableAccess dbaccess = new SatyamJobSubmissionsTableAccess();
List <SatyamJobSubmissionsTableAccessEntry> entries = dbaccess.getAllEntriesByUserID(UserId);
dbaccess.close();
List <TableRow> rows = new List <TableRow>();
int maxShow = 10;
for (int i = entries.Count - 1; i >= Math.Max(0, entries.Count - 1 - maxShow); i--)
{
TableRow row = new TableRow();
row.JobGUID = entries[i].JobGUID;
row.JobTemplateType = entries[i].JobTemplateType;
row.JobSubmitTime = entries[i].JobSubmitTime.ToLocalTime().ToString();
row.JobStatus = entries[i].JobStatus;
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
int remainingTasks = taskDB.getTasksCountByGUID(row.JobGUID);
taskDB.close();
row.TaskPending = remainingTasks.ToString();
if (entries[i].JobProgress == "")
{
row.JobProgress = "1";
}
else
{
row.JobProgress = (1 - (double)remainingTasks / Convert.ToDouble(entries[i].JobProgress)).ToString("0.000");
}
int totalResults, totalAggregated;
row.ApprovalRate = SatyamResultsAnalysis.getPaymentRateByGUID(row.JobGUID, out totalResults, out totalAggregated).ToString("0.000");
row.TotalResults = totalResults.ToString();
row.TotalAggregated = totalAggregated.ToString();
rows.Add(row);
}
rpt.DataSource = rows;
rpt.DataBind();
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
{
DateTime start = DateTime.Now;
bool logging = false;
if (logging)
{
log.Info($"DisposalDispatch executed at: {DateTime.Now}");
}
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
// first try and remove all the expired entries
List <SatyamAmazonHITTableAccessEntry> expiredHitEntries = hitDB.getAllEntriesByStatus(HitStatus.expired);
hitDB.close();
/// dispatch into the queue
SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess();
foreach (SatyamAmazonHITTableAccessEntry expiredEntry in expiredHitEntries)
{
if (logging)
{
log.Info($"Dispatching Disposal for {expiredEntry.HITID}");
}
string queueName = "hit-disposal";
string m = JSonUtils.ConvertObjectToJSon(expiredEntry);
satyamQueue.Enqueue(queueName, m);
if ((DateTime.Now - start).TotalSeconds > 280)
{
return;
}
}
hitDB = new SatyamAmazonHITTableAccess();
List <SatyamAmazonHITTableAccessEntry> hitEntriesPending = hitDB.getAllEntriesByStatus(HitStatus.pending);
List <SatyamAmazonHITTableAccessEntry> hitEntriesSubmitted = hitDB.getAllEntriesByStatus(HitStatus.submitted);
List <SatyamAmazonHITTableAccessEntry> hitEntriesAccepted = hitDB.getAllEntriesByStatus(HitStatus.accepted);
List <SatyamAmazonHITTableAccessEntry> hitEntriesRejected = hitDB.getAllEntriesByStatus(HitStatus.rejected);
List <SatyamAmazonHITTableAccessEntry> hitEntriesTaken = hitDB.getAllEntriesByStatus(HitStatus.taken);
//hitDB.close();
List <SatyamAmazonHITTableAccessEntry> hitEntries = new List <SatyamAmazonHITTableAccessEntry>();
hitEntries.AddRange(hitEntriesPending);
hitEntries.AddRange(hitEntriesSubmitted);
hitEntries.AddRange(hitEntriesAccepted);
hitEntries.AddRange(hitEntriesRejected);
hitEntries.AddRange(hitEntriesTaken);
if (hitEntries.Count == 0)
{
return;
}
//Group them by jobguids since each job can have many tasks
Dictionary <string, List <SatyamAmazonHITTableAccessEntry> > HITSByGUID = new Dictionary <string, List <SatyamAmazonHITTableAccessEntry> >();
foreach (SatyamAmazonHITTableAccessEntry hitEntry in hitEntries)
{
if (hitEntry.EntryExpired())
{
//hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatus(hitEntry.ID, HitStatus.expired);
//hitDB.close();
}
string guid = hitEntry.JobGUID;
if (!HITSByGUID.ContainsKey(hitEntry.JobGUID))
{
HITSByGUID.Add(hitEntry.JobGUID, new List <SatyamAmazonHITTableAccessEntry>());
}
HITSByGUID[guid].Add(hitEntry);
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
hitDB.close();
//now check if this guid has any entries in the task table
List <string> guids = HITSByGUID.Keys.ToList();
foreach (string guid in guids)
{
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
int noTasks = taskDB.getTasksCountByGUID(guid);
taskDB.close();
if (noTasks == 0)//remove these additional HITS from the system
{
List <SatyamAmazonHITTableAccessEntry> hitEntriesToBeRemoved = HITSByGUID[guid];
foreach (SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved in hitEntriesToBeRemoved)
{
if (logging)
{
log.Info($"Dispatching Disposal for {hitEntryToBeRemoved.HITID}");
}
string queueName = "hit-disposal";
string m = JSonUtils.ConvertObjectToJSon(hitEntryToBeRemoved);
satyamQueue.Enqueue(queueName, m);
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
}
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
if (logging)
{
log.Info($"DisposalDispatch finished at: {DateTime.Now}");
}
}
//makes sure that unneccesary HITS- those jobs that are completed are removed
//go to HIT Table check if there any pending HITS for which the jobGUID does exist in the taskTable
public static void ClearHITSFromAmazon()
{
int succCount = 0;
int failCount = 0;
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
//first try and remove all the expired entries
List <SatyamAmazonHITTableAccessEntry> expiredHitEntries = hitDB.getAllEntriesByStatus(HitStatus.expired);
AmazonMTurkHIT hit = new AmazonMTurkHIT();
foreach (SatyamAmazonHITTableAccessEntry expiredEntry in expiredHitEntries)
{
//hit.setAccount(expiredEntry.AmazonAccessKeyID, expiredEntry.AmazonSecretAccessKeyID, false);
//if (hit.DeleteHIT(expiredEntry.HITID))
//{
// hitDB.DeleteEntry(expiredEntry.ID);
//}
bool succ = SafeDeleteHit(expiredEntry);
if (succ)
{
succCount++;
}
else
{
failCount++;
}
Console.WriteLine("Succ: {0}, Fail: {1}", succCount, failCount);
}
List <SatyamAmazonHITTableAccessEntry> hitEntriesPending = hitDB.getAllEntriesByStatus(HitStatus.pending);
List <SatyamAmazonHITTableAccessEntry> hitEntriesSubmitted = hitDB.getAllEntriesByStatus(HitStatus.submitted);
List <SatyamAmazonHITTableAccessEntry> hitEntriesAccepted = hitDB.getAllEntriesByStatus(HitStatus.accepted);
List <SatyamAmazonHITTableAccessEntry> hitEntriesRejected = hitDB.getAllEntriesByStatus(HitStatus.rejected);
List <SatyamAmazonHITTableAccessEntry> hitEntriesTaken = hitDB.getAllEntriesByStatus(HitStatus.taken);
List <SatyamAmazonHITTableAccessEntry> hitEntries = new List <SatyamAmazonHITTableAccessEntry>();
hitEntries.AddRange(hitEntriesPending);
hitEntries.AddRange(hitEntriesSubmitted);
hitEntries.AddRange(hitEntriesAccepted);
hitEntries.AddRange(hitEntriesRejected);
hitEntries.AddRange(hitEntriesTaken);
hitDB.close();
if (hitEntries.Count == 0)
{
return;
}
//Group them by jobguids since each job can have many tasks
Dictionary <string, List <SatyamAmazonHITTableAccessEntry> > HITSByGUID = new Dictionary <string, List <SatyamAmazonHITTableAccessEntry> >();
foreach (SatyamAmazonHITTableAccessEntry hitEntry in hitEntries)
{
// if launched for too long, auto expire
if (hitEntry.EntryExpired())
{
hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatus(hitEntry.ID, HitStatus.expired);
hitDB.close();
}
// categorize by guid
string guid = hitEntry.JobGUID;
if (!HITSByGUID.ContainsKey(hitEntry.JobGUID))
{
HITSByGUID.Add(hitEntry.JobGUID, new List <SatyamAmazonHITTableAccessEntry>());
}
HITSByGUID[guid].Add(hitEntry);
}
//now check if this guid has any entries in the task table
List <string> guids = HITSByGUID.Keys.ToList();
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
foreach (string guid in guids)
{
int noTasks = taskDB.getTasksCountByGUID(guid);
if (noTasks == 0)//remove these additional HITS from the system
{
List <SatyamAmazonHITTableAccessEntry> hitEntriesToBeRemoved = HITSByGUID[guid];
hit = new AmazonMTurkHIT();
hit.setAccount(hitEntriesToBeRemoved[0].AmazonAccessKeyID, hitEntriesToBeRemoved[0].AmazonSecretAccessKeyID, false);
foreach (SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved in hitEntriesToBeRemoved)
{
//if (!hit.DeleteHIT(hitEntryToBeRemoved.HITID))
//{
// try
// {
// hit.expireHIT(hitEntryToBeRemoved.HITID);
// }
// catch
// {
// Console.WriteLine("HIT already expired?");
// }
// hitDB = new SatyamAmazonHITTableAccess();
// hitDB.UpdateStatus(hitEntryToBeRemoved.ID, HitStatus.expired);
// hitDB.close();
//}
//else
//{
// hitDB = new SatyamAmazonHITTableAccess();
// hitDB.DeleteEntry(hitEntryToBeRemoved.ID);
// hitDB.close();
//}
bool succ = SafeDeleteHit(hitEntryToBeRemoved);
if (succ)
{
succCount++;
}
else
{
failCount++;
//TODO: auto decide pay or reject
}
Console.WriteLine("Succ: {0}, Fail: {1}", succCount, failCount);
}
}
}
taskDB.close();
}