Beispiel #1
0
        public async Task <IEnumerable <Batch> > GetCurrentState(int?groupId = null)
        {
            groupId = groupId == null ? GroupId : groupId;

            using var batchContext = new BatchContext();
            return(await batchContext.Batches.Where(batch => batch.GroupId.Equals(groupId)).OrderBy(ord => ord.BatchId).ToListAsync());
        }
Beispiel #2
0
 public ValueTask OnBatchRunCompleteAsync(BatchContext context, string errorMessageIfFailed, Exception exceptionIfExists)
 {
     this.CompleteSuccessfully = (errorMessageIfFailed == null && exceptionIfExists == null);
     this.ErrorMessage         = errorMessageIfFailed;
     this.Exception            = exceptionIfExists;
     return(innerInterceptor.OnBatchRunCompleteAsync(context, errorMessageIfFailed, exceptionIfExists));
 }
Beispiel #3
0
        public override BatchQueueItem[] Process(BatchQueueItem[] items)
        {
            var ret = new List <BatchQueueItem>();

            foreach (var item in items)
            {
                try
                {
                    using (var context = new BatchContext())
                    {
                        var batch = context.Batches.Find(item.BatchId);
                        batch.GenerateOutput();
                        batch.Status      = BatchStatus.Completed;
                        batch.CompletedTS = DateTime.Now;
                        context.SaveChanges();

                        // Signal the notification
                        batch.NotificationSemaphore().Signal();
                    }
                    ret.Add(item);
                }
                catch (Exception ex)
                {
                    Logger.LogException(ex);
                    if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
                    {
                        // Max retry reached, clean up queueItem
                        ret.Add(item);
                    }
                }
            }
            return(ret.ToArray());
        }
        public static void ProcessFiles(
            [ActivityTrigger]
            BatchContext batchContext,
            TraceWriter log)
        {
            log.Info($"*** PROCESSFILES {batchContext.FolderName} - starting...");

            // TODO - insert real processing here
            System.Threading.Thread.Sleep(2500);

            log.Info($"*** PROCESSFILES {batchContext.FolderName} - done, cleaning up....");

            foreach (var filename in batchContext.RequiredFiles)
            {
                // TODO - replace local file access with blob access
                var path = Path.Combine(batchContext.FolderName, $"{batchContext.BatchId}_{filename}");
                if (File.Exists(path))
                {
                    log.Verbose($"*** Deleting {path}");
                    File.Delete(path);
                }
                else
                {
                    log.Error($"*** Missing file {path}"); // shouldn't hit this!
                }
            }

            log.Info($"*** PROCESSFILES {batchContext.FolderName} - done");
        }
        public ActionResult Retry(int id)
        {
            using (var context = new BatchContext())
            {
                var batch = LoadBatch(id, context);

                if (batch != null && (batch.Status == BatchStatus.Completed || batch.Status == BatchStatus.Notified))
                {
                    batch.Status = BatchStatus.Created;
                    batch.ResetCounters();
                    if (batch.Lines != null)
                    {
                        context.BatchLines.RemoveRange(batch.Lines);
                    }
                    context.SaveChanges();

                    var extractionQueue = CprBroker.Engine.Queues.Queue.GetQueues <Queues.ExtractionQueue>().Single();
                    extractionQueue.Enqueue(new Queues.BatchQueueItem()
                    {
                        BatchId = batch.BatchId
                    });

                    return(Json("Success.", JsonRequestBehavior.AllowGet));
                }
            }
            return(Json("Unable to retry", JsonRequestBehavior.AllowGet));
        }
        User GetUser(BatchContext context, string userName)
        {
            var user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();

            if (user == null)
            {
                lock ("User-adding")
                {
                    user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();
                    if (user == null)
                    {
                        user = new Models.User()
                        {
                            Name = userName
                        };
                        context.Users.Add(user);
                        context.SaveChanges();

                        user = context.Users.Where(u => u.Name.Equals(userName)).FirstOrDefault();
                        context.Entry <User>(user).Collection(u => u.Batches).Load();
                    }
                }
            }
            return(user);
        }
        public void EnqueueFile(System.IO.Stream stream, string name, int length, string userName)
        {
            using (var context = new BatchContext())
            {
                var user = GetUser(context, userName);

                // Now we have a user object
                var batch = new Batch()
                {
                    Size           = length,
                    SourceContents = new byte[length],
                    Status         = BatchStatus.Created,
                    FileName       = name,
                    SubmittedTS    = DateTime.Now,
                    User           = user
                };
                stream.Read(batch.SourceContents, 0, length);
                context.Batches.Add(batch);
                context.SaveChanges();

                var extractionQueue = CprBroker.Engine.Queues.Queue.GetQueues <Queues.ExtractionQueue>().Single();
                extractionQueue.Enqueue(new Queues.BatchQueueItem()
                {
                    BatchId = batch.BatchId
                });
            }
        }
Beispiel #8
0
        protected override void PerformTimerAction()
        {
            int daysToKeepFiles = Properties.Settings.Default.DaysToKeepBatchFiles;

            DateTime deleteOlderThan = DateTime.Now.AddDays(-daysToKeepFiles);

            using (BatchContext context = new BatchContext())
            {
                IQueryable <Batch> batchesToDelete = context.Batches.Where(b => b.CompletedTS < deleteOlderThan);

                foreach (Batch batch in batchesToDelete)
                {
                    try
                    {
                        context.Batches.Remove(batch);
                        batch.SignalAllSemaphores();
                        Admin.LogSuccess("Mass PNR Lookup: Removed batch " + batch.BatchId);
                    } catch (Exception e)
                    {
                        Admin.LogException(e);
                    }
                }
                context.SaveChanges();
            }
        }
 public async Task TestCircuit(Func <BatchContext, Task> next, BatchContext context)
 {
     OnCircuitTest(EventArgs.Empty);
     _log.LogInformation($"Subscriber circuit breaker: testing circuit for subscriber '{_config.SubscriberName}'");
     State = CircuitState.HalfOpen;
     await next(context).ConfigureAwait(false);
 }
        public void Process_LargeBatch_OK(int batchSize)
        {
            var           lines = Properties.Resources.Test_Opslag.Split(Environment.NewLine.ToCharArray(), StringSplitOptions.RemoveEmptyEntries).Take(2).ToList();
            StringBuilder b     = new StringBuilder((lines[1].Length + 2) * batchSize);

            b.AppendLine(lines[0]);
            for (int i = 0; i < batchSize; i++)
            {
                b.AppendLine(lines[1]);
            }
            var bytes = Commons.CsvEncoding.GetBytes(b.ToString());
            var batch = new Batch()
            {
                FileName = "", Size = bytes.Length, SourceContents = bytes, SubmittedTS = DateTime.Now
            };

            using (var context = new BatchContext())
            {
                context.Batches.Add(batch);
                context.SaveChanges();
            }

            var queueItem = new Queues.BatchQueueItem()
            {
                BatchId = batch.BatchId
            };
            var queue = new ExtractionQueue();

            queue.Process(new BatchQueueItem[] { queueItem });
        }
Beispiel #11
0
        public void SetFileTest()
        {
            string       script  = "scriptname";
            BatchContext context = new BatchContext(1, script, 2);

            context.File = "test";
            Assert.AreEqual("test", context.File);
        }
Beispiel #12
0
        public void SetLineNumberTest()
        {
            string       script  = "scriptname";
            BatchContext context = new BatchContext(1, script, 2);

            context.LineNumber = 10;
            Assert.AreEqual(10, context.LineNumber);
        }
Beispiel #13
0
        public void ConstructorTest()
        {
            string       script  = "scriptname";
            BatchContext context = new BatchContext(1, script, 2);

            Assert.AreEqual(script, context.File);
            Assert.AreEqual(1, context.BatchOffset);
            Assert.AreEqual(2, context.LineNumber);
        }
Beispiel #14
0
        public override BatchQueueItem[] Process(BatchQueueItem[] items)
        {
            var ret = new List <BatchQueueItem>();

            foreach (var item in items)
            {
                using (var context = new BatchContext())
                {
                    var batch = context.Batches.Find(item.BatchId);

                    if (batch == null)
                    {
                        // This happens if the batch is deleted before extraction
                        ret.Add(item);
                    }
                    else
                    {
                        try
                        {
                            var parser = batch.CreateParser();
                            // Delete existing lines if needed
                            var oldLines = context.BatchLines.Where(bl => bl.Batch_BatchId == batch.BatchId);
                            context.BatchLines.RemoveRange(oldLines);
                            context.SaveChanges();

                            context.Entry <Batch>(batch).Reload();

                            var lines = parser.ToArray();
                            if (batch.Lines == null)
                            {
                                batch.Lines = new List <BatchLine>();
                            }

                            Array.ForEach <BatchLine>(lines, bl => batch.Lines.Add(bl));

                            batch.EnqueueAllAfterExtraction(context);
                            ret.Add(item);
                        }
                        catch (Exception ex)
                        {
                            Admin.LogException(ex);

                            batch.Status = BatchStatus.Error;
                            context.SaveChanges();
                        }
                        if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
                        {
                            // Max retry reached, clean up queueItem
                            ret.Add(item);
                        }
                    }
                }
            }
            return(ret.ToArray());
        }
Beispiel #15
0
        private void AddIfNotExists(Customer item)
        {
            if (Table.Any(x => x.FirstName == item.FirstName &&
                          x.LastName == item.LastName))
            {
                return;
            }

            Insert(item);
            BatchContext.SaveChanges();
        }
 public async Task Invoke(Func <BatchContext, Task> next, BatchContext context)
 {
     if (State == CircuitState.Closed)
     {
         await next(context).ConfigureAwait(false);
     }
     else
     {
         await Task.Delay(TimeSpan.FromSeconds(_circuitTestIntervalInSeconds), context.TokenSource.Token).ConfigureAwait(false);
         await TestCircuit(next, context).ConfigureAwait(false);
     }
 }
 public async Task Invoke(Func <BatchContext, Task> next, BatchContext context)
 {
     try
     {
         await next(context).ConfigureAwait(false);
     }
     catch (Exception ex)
     {
         _statisticsReporter.Increment("ProcessBatchInfrastructureException",
                                       $"Client=PatLite.{_config.SubscriberName}," +
                                       $"ExceptionType={ex.GetType()}");
         throw;
     }
 }
 public ActionResult Resume(int id)
 {
     using (var context = new BatchContext())
     {
         var batch = LoadBatch(id, context);
         if (batch != null && batch.Status == BatchStatus.Paused)
         {
             batch.Status = BatchStatus.Processing;
             context.SaveChanges();
             batch.SearchSemaphore().SignalAll();
             return(Json("Success.", JsonRequestBehavior.AllowGet));
         }
     }
     return(new HttpNotFoundResult());
 }
        public ActionResult Result(int id)
        {
            using (var context = new BatchContext())
            {
                var batch = LoadBatch(id, context);

                if (batch != null && batch.GeneratedContents != null)
                {
                    var ext = batch.FileName.Split('.').Skip(1).LastOrDefault();
                    ext = string.IsNullOrEmpty(ext) ? "" : "." + ext;
                    return(new FileContentResult(batch.GeneratedContents, "application/unspecified")
                    {
                        FileDownloadName = string.Format("{0}-result{1}", batch.FileName, ext)
                    });
                }
            }
            return(new HttpNotFoundResult());
        }
Beispiel #20
0
        public override void Build()
        {
            // init fields

            m_InputDepth  = Net.InputDepth;
            m_InputHeight = Net.InputHeight;
            m_InputWidth  = Net.InputWidth;
            m_OutputDepth = Net[Net.LayerCount - 1].OutputDepth;

            m_Gradient = new double[Net.LayerCount][];
            m_Values   = new double[Net.LayerCount][][, ];
            m_Errors   = new double[Net.LayerCount][][, ];
            for (int l = 0; l < Net.LayerCount; l++)
            {
                var layer = Net[l];

                m_Gradient[l] = new double[layer.ParamCount];
                m_Values[l]   = new double[layer.OutputDepth][, ];
                m_Errors[l]   = new double[layer.OutputDepth][, ];
                for (int p = 0; p < layer.OutputDepth; p++)
                {
                    m_Values[l][p] = new double[layer.OutputHeight, layer.OutputWidth];
                    m_Errors[l][p] = new double[layer.OutputHeight, layer.OutputWidth];
                }
            }

            // init optimizer
            if (m_Optimizer == null)
            {
                m_Optimizer = Registry.Optimizer.SGD;
            }

            // init scheduler
            if (m_LearningRateScheduler == null)
            {
                m_LearningRateScheduler = Registry.LearningRateScheduler.Constant(m_LearningRate);
            }

            // init batch context
            if (m_UseBatchParallelization)
            {
                m_BatchContext = new BatchContext(this, m_MaxBatchThreadCount);
            }
        }
 public ActionResult Remove(int id)
 {
     try
     {
         using (var context = new BatchContext())
         {
             var batch = LoadBatch(id, context);
             if (batch != null)
             {
                 context.Batches.Remove(batch);
                 context.SaveChanges();
                 batch.SignalAllSemaphores();
                 return(Json("Success.", JsonRequestBehavior.AllowGet));
             }
         }
     } catch (Exception)
     {
         return(Json("An Error occured.", JsonRequestBehavior.AllowGet));
         // DEBUGGING return Json(ex.Message+"\r\n"+ex.StackTrace+"\r\n\r\n"+ex.InnerException, JsonRequestBehavior.AllowGet);
     }
     return(new HttpStatusCodeResult(500));
 }
Beispiel #22
0
        public async Task WriteBatchAndNumberToDatabase(BatchAndNumberFullInfo batchAndNumberFullInfo)
        {
            //I am given to understand that this is an antequated way to do this. I agree. I am however struggling to utilise the dependency injection native to EF Core, as at the top of this page.
            //Right now, I just want to focus on functionality, but please be aware that I know this is not ideal.
            var optionsBuilder = new DbContextOptionsBuilder <BatchContext>();

            optionsBuilder.UseSqlServer("Server=(local)\\sqlexpress;Database=RuanGatesBDAssessmentDB;Trusted_Connection=True;MultipleActiveResultSets=True;");

            var batch = new Batch();

            batch.BatchElements               = new List <BatchElement>();
            batch.BatchAndNumberInput         = new BatchAndNumberInput();
            batch.BatchAndNumberInput.Batches = batchAndNumberFullInfo.BatchAndNumberInputDetails.Batches;
            batch.BatchAndNumberInput.Numbers = batchAndNumberFullInfo.BatchAndNumberInputDetails.Numbers;
            batch.CollectionId = batchAndNumberFullInfo.BatchAndNumberInputDetails.RequestId;

            BatchElement batchElement = new BatchElement();

            batchElement.BatchNumber      = batchAndNumberFullInfo.BatchAndNumber.Batch;
            batchElement.NumbersRemaining = 4;
            batchElement.Aggregate        = 12;
            batchElement.NumbersInBatch   = new List <NumberInBatch>();

            NumberInBatch number = new NumberInBatch();

            number.Number = batchAndNumberFullInfo.BatchAndNumber.Number;

            batchElement.NumbersInBatch.Add(number);
            batch.BatchElements.Add(batchElement);

            batch.GrandTotal = 99;

            using (var ctx = new BatchContext(optionsBuilder.Options))
            {
                ctx.Batches.Add(batch);
                await ctx.SaveChangesAsync();
            }
        }
        public async Task Invoke(Func <BatchContext, Task> next, BatchContext context)
        {
            _timer.Start();

            await next(context).ConfigureAwait(false);

            var currentInterval = _timer.ElapsedMilliseconds / _intervalInMilliSeconds;

            var startInterval = (currentInterval - _rollingIntervals > 0) ? currentInterval - _rollingIntervals : 0;
            var messagesProcessedInLastInterval = 0;

            for (long i = startInterval; i <= currentInterval; i++)
            {
                var intervalFound = _messagesProcessed.FirstOrDefault(x => x.IntervalNumber == i);
                if (intervalFound != null)
                {
                    messagesProcessedInLastInterval += intervalFound.MessagesProcessed;
                }
            }

            var elapsedSinceLastIntervalBegan = _timer.ElapsedMilliseconds - currentInterval * _intervalInMilliSeconds;
            var wholeIntervalsElapsed         = currentInterval - startInterval;

            var    elapsedTime      = wholeIntervalsElapsed * _intervalInMilliSeconds + elapsedSinceLastIntervalBegan;
            double elapsedIntervals = (double)elapsedTime / (_intervalInMilliSeconds);
            var    processingRate   = _intervalsPerMinute * messagesProcessedInLastInterval / elapsedIntervals;

            if (processingRate > _rateLimit)
            {
                var targetTimePerMessage = 1000 * 60 / _rateLimit;
                var targetTime           = messagesProcessedInLastInterval * targetTimePerMessage;
                var delay = targetTime - elapsedTime;
                OnThrottling(new ThrottlingEventArgs(delay));
                await _throttler.Delay(delay).ConfigureAwait(false);
            }
        }
Beispiel #24
0
        public override LineQueueItem[] Process(LineQueueItem[] items)
        {
            var ret = new List <LineQueueItem>();

            using (var context = new BatchContext())
            {
                foreach (var item in items)
                {
                    bool itemSucceeded = false;

                    var batchLine = context.BatchLines.Find(item.BatchLineId);

                    if (batchLine == null)
                    {
                        if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
                        {
                            // Max attempts reached - signal and remove anyway
                            ret.Add(item);
                        }
                        continue;
                    }

                    var partManager = new PartManager();
                    var soegObject  = batchLine.ToSoegObject();


                    if (soegObject != null)
                    {
                        // Try to search locally first
                        // Since Mass PNR Lookup does some name-matching itself first, we need the local search.
                        // There could be a local person found by CPR Broker, that Mass PNR Lookup does not match, therefore using "SourceUsageOrder.LocalThenExternal" is not enough
                        if (search(partManager, soegObject, batchLine, SourceUsageOrder.LocalOnly))
                        {
                            itemSucceeded = true;
                        }
                        else
                        {
                            // If no local person was found, search Externally
                            if (search(partManager, soegObject, batchLine, SourceUsageOrder.ExternalOnly))
                            {
                                itemSucceeded = true;
                            }
                        }
                    }
                    else
                    {
                        batchLine.Error = "Invalid address";
                    }

                    lock ("BatchCounts")
                    {
                        context.Entry <Batch>(batchLine.Batch).Reload(); // Reload to avoid overwriting the counts

                        if (itemSucceeded)
                        {
                            batchLine.Batch.SucceededLines++;
                        }


                        // Queue management
                        if (itemSucceeded)
                        {
                            ret.Add(item);
                            // Decrement the wait count on the semaphore
                            batchLine.Batch.GenerationSemaphore().Signal();
                        }
                        else if (item.Impl.AttemptCount >= this.Impl.MaxRetry - 1)
                        {
                            // Max attempts reached - signal and remove anyway
                            ret.Add(item);
                            batchLine.Batch.FailedLines++;

                            batchLine.Batch.GenerationSemaphore().Signal();
                        }
                        // Save the result at this point
                        context.SaveChanges();
                    }
                }
            }
            return(ret.ToArray());
        }
Beispiel #25
0
 public NumbersAndBatchesData(BatchContext context)
 {
     _context = context;
 }
 public BatchesController(BatchContext context)
 {
     _context = context;
 }
 public BatchReportRepository(BatchContext context)
 {
     _context = context;
 }
Beispiel #28
0
 public PcsActiveTempParametersRepository(BatchContext batchContext)
 {
     _batchContext = batchContext;
 }
Beispiel #29
0
 public PcsScoringSettingsController(BatchContext context)
 {
     _context = context;
 }
Beispiel #30
0
 protected override IQueryable <Batch> LoadBatches(BatchContext context)
 {
     return(context.Batches.Include("User"));
 }