public override BatchQueueItem[] Process(BatchQueueItem[] items)
{
var ret = new List <BatchQueueItem>();
foreach (var item in items)
{
using (var context = new BatchContext())
{
var batch = context.Batches.Find(item.BatchId);
if (batch == null)
{
// This happens if the batch is deleted before extraction
ret.Add(item);
}
else
{
try
{
var parser = batch.CreateParser();
// Delete existing lines if needed
var oldLines = context.BatchLines.Where(bl => bl.Batch_BatchId == batch.BatchId);
context.BatchLines.RemoveRange(oldLines);
context.SaveChanges();
context.Entry <Batch>(batch).Reload();
var lines = parser.ToArray();
if (batch.Lines == null)
{
batch.Lines = new List <BatchLine>();
}
Array.ForEach <BatchLine>(lines, bl => batch.Lines.Add(bl));
batch.EnqueueAllAfterExtraction(context);
ret.Add(item);
}
catch (Exception ex)
{
Admin.LogException(ex);
batch.Status = BatchStatus.Error;
context.SaveChanges();
}
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
}
}
}
return(ret.ToArray());
}
public Batch AddBatch(Batch batch)
{
batch.BatchId = Guid.NewGuid();
_batchContext.Batches.Add(batch);
_batchContext.SaveChanges();
AzureBlob azureBlob = new AzureBlob();
azureBlob.InitializeBlobContainer(_storageAccountKey, _storageAccountName, _containerName);
return(batch);
}
protected override void PerformTimerAction()
{
int daysToKeepFiles = Properties.Settings.Default.DaysToKeepBatchFiles;
DateTime deleteOlderThan = DateTime.Now.AddDays(-daysToKeepFiles);
using (BatchContext context = new BatchContext())
{
IQueryable <Batch> batchesToDelete = context.Batches.Where(b => b.CompletedTS < deleteOlderThan);
foreach (Batch batch in batchesToDelete)
{
try
{
context.Batches.Remove(batch);
batch.SignalAllSemaphores();
Admin.LogSuccess("Mass PNR Lookup: Removed batch " + batch.BatchId);
} catch (Exception e)
{
Admin.LogException(e);
}
}
context.SaveChanges();
}
}
public override BatchQueueItem[] Process(BatchQueueItem[] items)
{
var ret = new List <BatchQueueItem>();
foreach (var item in items)
{
try
{
using (var context = new BatchContext())
{
var batch = context.Batches.Find(item.BatchId);
batch.GenerateOutput();
batch.Status = BatchStatus.Completed;
batch.CompletedTS = DateTime.Now;
context.SaveChanges();
// Signal the notification
batch.NotificationSemaphore().Signal();
}
ret.Add(item);
}
catch (Exception ex)
{
Logger.LogException(ex);
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
}
}
return(ret.ToArray());
}
User GetUser(BatchContext context, string userName)
{
var user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();
if (user == null)
{
lock ("User-adding")
{
user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();
if (user == null)
{
user = new Models.User()
{
Name = userName
};
context.Users.Add(user);
context.SaveChanges();
user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();
context.Entry <User>(user).Collection(u => u.Batches).Load();
}
}
}
return(user);
}
public ActionResult Retry(int id)
{
using (var context = new BatchContext())
{
var batch = LoadBatch(id, context);
if (batch != null && (batch.Status == BatchStatus.Completed || batch.Status == BatchStatus.Notified))
{
batch.Status = BatchStatus.Created;
batch.ResetCounters();
if (batch.Lines != null)
{
context.BatchLines.RemoveRange(batch.Lines);
}
context.SaveChanges();
var extractionQueue = CprBroker.Engine.Queues.Queue.GetQueues <Queues.ExtractionQueue>().Single();
extractionQueue.Enqueue(new Queues.BatchQueueItem()
{
BatchId = batch.BatchId
});
return(Json("Success.", JsonRequestBehavior.AllowGet));
}
}
return(Json("Unable to retry", JsonRequestBehavior.AllowGet));
}
public void EnqueueFile(System.IO.Stream stream, string name, int length, string userName)
{
using (var context = new BatchContext())
{
var user = GetUser(context, userName);
// Now we have a user object
var batch = new Batch()
{
Size = length,
SourceContents = new byte[length],
Status = BatchStatus.Created,
FileName = name,
SubmittedTS = DateTime.Now,
User = user
};
stream.Read(batch.SourceContents, 0, length);
context.Batches.Add(batch);
context.SaveChanges();
var extractionQueue = CprBroker.Engine.Queues.Queue.GetQueues <Queues.ExtractionQueue>().Single();
extractionQueue.Enqueue(new Queues.BatchQueueItem()
{
BatchId = batch.BatchId
});
}
}
public void Process_LargeBatch_OK(int batchSize)
{
var lines = Properties.Resources.Test_Opslag.Split(Environment.NewLine.ToCharArray(), StringSplitOptions.RemoveEmptyEntries).Take(2).ToList();
StringBuilder b = new StringBuilder((lines[1].Length + 2) * batchSize);
b.AppendLine(lines[0]);
for (int i = 0; i < batchSize; i++)
{
b.AppendLine(lines[1]);
}
var bytes = Commons.CsvEncoding.GetBytes(b.ToString());
var batch = new Batch()
{
FileName = "", Size = bytes.Length, SourceContents = bytes, SubmittedTS = DateTime.Now
};
using (var context = new BatchContext())
{
context.Batches.Add(batch);
context.SaveChanges();
}
var queueItem = new Queues.BatchQueueItem()
{
BatchId = batch.BatchId
};
var queue = new ExtractionQueue();
queue.Process(new BatchQueueItem[] { queueItem });
}
private void AddIfNotExists(Customer item)
{
if (Table.Any(x => x.FirstName == item.FirstName &&
x.LastName == item.LastName))
{
return;
}
Insert(item);
BatchContext.SaveChanges();
}
public ActionResult Resume(int id)
{
using (var context = new BatchContext())
{
var batch = LoadBatch(id, context);
if (batch != null && batch.Status == BatchStatus.Paused)
{
batch.Status = BatchStatus.Processing;
context.SaveChanges();
batch.SearchSemaphore().SignalAll();
return(Json("Success.", JsonRequestBehavior.AllowGet));
}
}
return(new HttpNotFoundResult());
}
public ActionResult Remove(int id)
{
try
{
using (var context = new BatchContext())
{
var batch = LoadBatch(id, context);
if (batch != null)
{
context.Batches.Remove(batch);
context.SaveChanges();
batch.SignalAllSemaphores();
return(Json("Success.", JsonRequestBehavior.AllowGet));
}
}
} catch (Exception)
{
return(Json("An Error occured.", JsonRequestBehavior.AllowGet));
// DEBUGGING return Json(ex.Message+"\r\n"+ex.StackTrace+"\r\n\r\n"+ex.InnerException, JsonRequestBehavior.AllowGet);
}
return(new HttpStatusCodeResult(500));
}
public override LineQueueItem[] Process(LineQueueItem[] items)
{
var ret = new List <LineQueueItem>();
using (var context = new BatchContext())
{
foreach (var item in items)
{
bool itemSucceeded = false;
var batchLine = context.BatchLines.Find(item.BatchLineId);
if (batchLine == null)
{
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max attempts reached - signal and remove anyway
ret.Add(item);
}
continue;
}
var partManager = new PartManager();
var soegObject = batchLine.ToSoegObject();
if (soegObject != null)
{
// Try to search locally first
// Since Mass PNR Lookup does some name-matching itself first, we need the local search.
// There could be a local person found by CPR Broker, that Mass PNR Lookup does not match, therefore using "SourceUsageOrder.LocalThenExternal" is not enough
if (search(partManager, soegObject, batchLine, SourceUsageOrder.LocalOnly))
{
itemSucceeded = true;
}
else
{
// If no local person was found, search Externally
if (search(partManager, soegObject, batchLine, SourceUsageOrder.ExternalOnly))
{
itemSucceeded = true;
}
}
}
else
{
batchLine.Error = "Invalid address";
}
lock ("BatchCounts")
{
context.Entry <Batch>(batchLine.Batch).Reload(); // Reload to avoid overwriting the counts
if (itemSucceeded)
{
batchLine.Batch.SucceededLines++;
}
// Queue management
if (itemSucceeded)
{
ret.Add(item);
// Decrement the wait count on the semaphore
batchLine.Batch.GenerationSemaphore().Signal();
}
else if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max attempts reached - signal and remove anyway
ret.Add(item);
batchLine.Batch.FailedLines++;
batchLine.Batch.GenerationSemaphore().Signal();
}
// Save the result at this point
context.SaveChanges();
}
}
}
return(ret.ToArray());
}
public void SaveChanges(BatchReport report = null)
{
_context.SaveChanges();
}
public void SaveChanges()
{
_context.SaveChanges();
}
public void AddBatchesToShiftLog(List <BatchReport> reports)
{
foreach (var report in reports.Where(x => x.IsValidBatch == true))
{
var shiftLog = GetOrCreateShiftLog(report.StartTime);
if (!DoesBatchExistInShift(shiftLog.OperatorShiftLogId, report.BatchReportId))
{
_batchContext.BatchesForShift.Add(new BatchesForShift
{
ShiftId = shiftLog.OperatorShiftLogId,
BatchId = report.BatchReportId,
});
}
}
_batchContext.SaveChanges();
}
public override BatchQueueItem[] Process(BatchQueueItem[] items)
{
var ret = new List <BatchQueueItem>();
foreach (var item in items)
{
try
{
using (var context = new BatchContext())
{
var batch = context.Batches.Find(item.BatchId);
// If the batch is deleted
if (batch == null)
{
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
continue;
}
// If this item is the latest assigned notification queue item
if (batch.NotificationSemaphore().Impl.SemaphoreId == item.Impl.SemaphoreId.Value)
{
var userPrincipal = batch?.User?.GetUserPrincipal(ContextType.Domain);
var email = userPrincipal?.EmailAddress;
if (!string.IsNullOrEmpty(email))
{
Admin.LogFormattedSuccess(
"Sending email for batch <{0}>, user <{1}>, principal <{2}>, email <{3}>",
batch.BatchId,
batch.User?.Name,
userPrincipal?.Name,
email
);
// Create client and message
var smtpClient = new SmtpClient();
MailMessage msg = new MailMessage()
{
//From = new MailAddress((smtpClient.Credentials as System.Net.NetworkCredential).UserName),
Subject = "Batch completed",
Body = string.Format("Your batch '{0}' has been completed at {1}", batch.FileName, batch.CompletedTS),
From = new MailAddress(email)
};
msg.To.Add(new MailAddress(email, userPrincipal.DisplayName));
// Send the message
smtpClient.Send(msg);
// Update the status
batch.Status = BatchStatus.Notified;
context.SaveChanges();
ret.Add(item);
}
else
{
Admin.LogFormattedError(
"Could not find email for batch <{0}>, user <{1}>, principal <{2}>, email <{3}>",
batch?.BatchId,
batch?.User?.Name,
userPrincipal?.Name,
email
);
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
}
}
else
{
Admin.LogFormattedError(
"Semaphore mismatch, skipping user notificatiuons. Batch <{0}> queue item <{1}>",
batch.BatchId,
item.Impl.QueueItemId);
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
}
}
}
catch (Exception ex)
{
Admin.LogException(ex, string.Format("queue item <{0}>", item.Impl.QueueItemId));
if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
{
// Max retry reached, clean up queueItem
ret.Add(item);
}
}
}
return(ret.ToArray());
}