/// <summary>
/// Add a new simulation to be performed.
/// </summary>
public void AddSimulation(GPProjectProfile Profile, int TrainingDataID, int ProjectID)
{
BatchProcess NewProcess = new BatchProcess(Profile, TrainingDataID, ProjectID);
m_Queue.Add(NewProcess);
//
// Inform all clients a new process was just added
foreach (KeyValuePair <IBatchClient, IBatchClient> Client in m_Clients)
{
Client.Value.AddProcess(
NewProcess,
GPDatabaseUtils.FieldValue(NewProcess.ProjectID, "tblProject", "Name"),
NewProcess.Profile.Name,
NewProcess.TimeAdded,
NewProcess.TimeStarted,
NewProcess.IsStarted);
}
//
// Indicate to the thread a new model has been requested. This is done by sending
// a "Default" command. What the effectively does is to make execing of simulations
// the lowest priority event. We want register client events to take priority
// over exec simulations, so they start getting updated as soon as possible.
m_CommandQueue.Enqueue(Command.Default);
wh_CommandEvent.Set();
}
/// <summary>
/// Sends notification to all clients a batch process has just started, indicating which one it is.
/// </summary>
/// <param name="Process"></param>
private void BroadcastModelStarted(BatchProcess Process)
{
foreach (KeyValuePair <IBatchClient, IBatchClient> Client in m_Clients)
{
Client.Value.ProcessStarted(Process, Process.TimeStarted, Process.Profile.m_maxNumber);
}
}
private void process_MajorProgressChanged(object sender, EventArgs e)
{
if (!IsHandleCreated)
{
return; // to avoid exception when this method is called after Cancel();Close();
}
BeginInvoke(new MethodInvoker(delegate() {
BatchProcess proc = sender as BatchProcess;
String message = proc.MajorProgressMessage;
Int32 percent = proc.MajorProgressPercentage;
if (percent == -1)
{
__progOverall.Style = ProgressBarStyle.Marquee; // the setter checks for equality, so don't worry about performance
__progOverallLbl.Text = message;
return;
}
__progOverall.Style = ProgressBarStyle.Blocks;
__progOverall.Value = percent;
__progOverallLbl.Text = String.Format("{0}% - {1} - {2}/{3} files", percent, message, proc.FilesDone, proc.FilesCount);
}));
}
public void RunOperations()
{
var items = new List <Dictionary <string, AttributeValue> >(Manager.SourceTable.Scan.Items);
Operations.ForEach(op => op.Process(ref items));
BatchProcess.WriteItemsAsync(items);
}
public BatchProcessDTO(BatchProcess source)
{
BatchProcessID = source.BatchProcessID;
EventTypeID = source.EventTypeID;
SORID = source.SORID;
ContentType = source.ContentType;
Body = source.Body;
}
private void SendData(BatchProcess batchProcess, string scenarioId)
{
var body = JsonConvert.SerializeObject(batchProcess);
var content = new StringContent(body, Encoding.UTF8, "application/json");
content.Headers.Add("flowId", scenarioId);
var result = httpClient.PostAsync("batch", content).Result;
}
public async Task Process(BatchProcess data)
{
var flowId = HttpContext.Items["flowId"].ToString();
var messages = data.Messages.Select(m => mapper.Map <Message>(m)).ToArray();
var transfers = data.Transfers.Select(t => mapper.Map <Transfer>(t)).ToArray();;
var instalments = data.RepaidInstalmentsIds;
var parallelTasks = new List <Task>();
if (messages.Length > 0)
{
parallelTasks.Add(Task.Run(async() =>
{
await usersClient.BatchAddMessagesAsync(new BatchAddMessagesRequest {
Messages = { messages }
}, HttpContext.CreateHeadersWithFlowId());
}));
}
if (transfers.Length > 0)
{
parallelTasks.Add(Task.Run(async() =>
{
await accountsWriteClient.BatchTransferAsync(new BatchTransferRequest {
Transfers = { transfers }
}, HttpContext.CreateHeadersWithFlowId());
}));
}
if (data.ProcessedPaymentsIds.Length > 0)
{
parallelTasks.Add(Task.Run(async() =>
{
var request = new UpdateLatestProcessingTimestampRequest {
Ids = { data.ProcessedPaymentsIds }, LatestProcessingTimestamp = data.ProcessingTimestamp.ToNullableTimestamp()
};
await paymentsWriteClient.UpdateLatestProcessingTimestampAsync(request, HttpContext.CreateHeadersWithFlowId());
}));
}
if (instalments.Length > 0)
{
parallelTasks.Add(Task.Run(async() =>
{
await loansWriteClient.BatchRepayInstalmentsAsync(new BatchRepayInstalmentsRequest {
Ids = { instalments }
}, HttpContext.CreateHeadersWithFlowId());
}));
}
if (parallelTasks.Count > 0)
{
await Task.WhenAll(parallelTasks);
}
}
/// <summary>
/// Manages the modeling for a batch process.
/// </summary>
/// <param name="Process"></param>
/// <returns>true if model was created, false otherwise</returns>
private bool BatchModel(BatchProcess Process)
{
#if GPLOG
GPLog.ReportLine("Batch Processing: Requesting a program.", true);
#endif
//
// Create the modeler object - the parameter to this thread is
// the model profile.
m_Modeler = new GPModeler(
Process.Profile,
Process.TrainingDataID,
null, //new GPModeler.DEL_ValidatedServer(AddValidatedServer),
new GPModeler.DEL_ReportStatus(ReceiveStatus),
new GPModeler.DEL_ReportFitness(ReceiveFitness),
null); //new GPModeler.DEL_GenerationComplete(GenerationComplete));
//
// Make an asynchronous call that gets the modeling started
MethodInvoker miModeler = new MethodInvoker(m_Modeler.RequestProgram);
IAsyncResult ar = miModeler.BeginInvoke(null, null);
//
// Wait for the modeling to complete
miModeler.EndInvoke(ar);
//
// Record the time of completion
Process.TimeCompleted = DateTime.Now;
//
// Check to see if we were canceled
if (m_CancelSimulation)
{
#if GPLOG
GPLog.ReportLine("Batch Processing: Simulation Canceled", true);
#endif
Process.Canceled = true;
//
// Reset the cancel flag
m_CancelSimulation = false;
return(false);
}
#if GPLOG
GPLog.ReportLine("Batch Processing: Program request complete.", true);
#endif
//
// Save the best program to the database
int ProgramID = 0; // A dummy variable
m_Modeler.SaveBestToDB(Process.ProjectID, ref ProgramID);
return(true);
}
/// <summary>
/// Sends notification to any clients a batch process has finished modeling
/// </summary>
/// <param name="Process"></param>
private void BroadcastModelComplete(BatchProcess Process)
{
UpdatePendingRegistrations();
foreach (KeyValuePair <IBatchClient, IBatchClient> Client in m_Clients)
{
Client.Value.ProcessComplete(
Process,
Process.TimeCompleted,
Process.Canceled,
Process.Fitness,
Process.Hits,
Process.Complexity);
}
}
public async Task Process(BatchProcess data)
{
var messages = data.Messages.Select(m => mapper.Map <Message>(m));
var transfers = data.Transfers.Select(t => mapper.Map <Transfer>(t));
var request = new ProcessBatchRequest
{
ProcessingTimestamp = data.ProcessingTimestamp.ToNullableTimestamp(),
ProcessedPaymentsIds = { data.ProcessedPaymentsIds },
Transfers = { transfers },
Messages = { messages },
RepaidInstalmentsIds = { data.RepaidInstalmentsIds }
};
await batchesBranchClient.ProcessAsync(request, HttpContext.CreateHeadersWithFlowId());
}
public Task Process(BatchProcess data)
{
var flowId = HttpContext.Items["flowId"].ToString();
var messages = data.Messages.Select(m => mapper.Map <UserMessage>(m)).ToArray();
if (messages.Length > 0)
{
var messagesEvent = new BatchAddMessagesEvent {
Messages = messages
};
publishingRouter.Publish(Queues.Users, messagesEvent, flowId);
}
if (data.Transfers.Length > 0)
{
var transfersEvent = new BatchTransferEvent {
Transfers = data.Transfers
};
publishingRouter.Publish(Queues.Accounts, transfersEvent, flowId);
}
if (data.ProcessedPaymentsIds.Length > 0)
{
var processingTimestampEvent = new UpdateLatestProcessingTimestampEvent {
Ids = data.ProcessedPaymentsIds, Timestamp = data.ProcessingTimestamp
};
publishingRouter.Publish(Queues.Payments, processingTimestampEvent, flowId);
}
var instalments = data.RepaidInstalmentsIds;
if (instalments.Length > 0)
{
var transfersEvent = new BatchRepayInstalmentsEvent {
Ids = instalments
};
publishingRouter.Publish(Queues.Loans, transfersEvent, flowId);
}
return(Task.CompletedTask);
}
private void tmrFileLookUp_Tick(object sender, ElapsedEventArgs e)
{
//WriteLog("Tick1");
try
{
this.timer.Stop();
//WriteLog("Tick2");
BatchProcess bObj = new BatchProcess();
bObj.beginProcess();
WriteLog("Batch Process service started");
BatchProcessRawClaim objBatchProcessRawClaim = new BatchProcessRawClaim();
objBatchProcessRawClaim.BeginProcess();
WriteLog("Batch Process Raw Claim service started");
BatchProcessReimbursementClaim objBatchProcessReimbursementClaim = new BatchProcessReimbursementClaim();
objBatchProcessReimbursementClaim.beginProcess();
WriteLog("Batch Process Reimbursement Claim service started");
BatchProcessNonEBClaim objBatchProcessNonEBClaim = new BatchProcessNonEBClaim();
objBatchProcessNonEBClaim.beginProcess();
WriteLog("Batch Process Non EB Claim service started");
BatchProcessVessel objVessel = new BatchProcessVessel();
objVessel.beginProcess();
WriteLog("Batch Process Vessel Upload service started");
this.timer.Start();
}
catch (Exception ex)
{
WriteLog(ex.Message);
EventLogs.Publish(ex.Message, System.Diagnostics.EventLogEntryType.Information);
this.timer.Start();
//EventLogs.WriteLogInFile(ex.Message, System.Diagnostics.EventLogEntryType.Information);
}
}
public void PublishAsync(string topic, byte[] content, int maxNum = 500, int maxTime = 500)
{
var pInfo = new PublishInfo()
{
Name = _name,
Topic = topic,
Data = content
};
var cdata = pInfo.ToBytes();
lock (_locker)
{
if (_batchProcess == null)
{
_batchProcess = new BatchProcess <byte[]>((data) =>
{
SendBase(QueueSocketMsgType.PublishForBatch, data.ToBytes());
}, maxNum, maxTime);
}
}
_batchProcess.Package(cdata);
}
public void SetBatch(BatchProcess.BatchProcess batchProcess)
{
if(_batchProcess==null)
{
System.Diagnostics.Debug.WriteLine("Error in GrandScheduler.SetBatch() old batchProcess is still running.");
}
_batchProcess = batchProcess;
}
private void Perform(int index)
{
var currentTime = automatSettings.StartTime;
while (currentTime < automatSettings.EndTime)
{
var scenarioTimer = Stopwatch.StartNew();
var scenarioId = Guid.NewGuid().ToString();
var scenarioPartTimer = Stopwatch.StartNew();
var token = sessionRequester.GetToken("automat", scenarioId);
logger.Information($"Service='Automat' ScenarioId='{scenarioId}' Method='automat token' Processing='{scenarioPartTimer.ElapsedMilliseconds}'");
scenarioPartTimer.Restart();
var data = GetData(index, scenarioId, currentTime);
logger.Information($"Service='Automat' ScenarioId='{scenarioId}' Method='automat getbatch' Processing='{scenarioPartTimer.ElapsedMilliseconds}'");
var toPay = data.Payments.Where(p => p.LatestProcessingTimestamp + p.Interval < DateTime.UtcNow);
var balancesDict = data.Balances.ToDictionary(k => k.Id, v => v);
var loansDict = data.Loans.ToDictionary(k => k.PaymentId, v => v);
var withSufficientBalance = toPay.Where(p =>
{
if (p.Amount <= balancesDict[p.AccountId].Amount)
{
balancesDict[p.AccountId].Amount -= p.Amount;
return(true);
}
else
{
return(false);
};
}).ToArray();
var withInsufficientBalance = toPay.Except(withSufficientBalance);
var paidIds = withSufficientBalance.Select(p => p.Id).ToHashSet();
var transfers = withSufficientBalance.Select(p => CreateTransfer(p, loansDict.ContainsKey(p.Id) ? loansDict[p.Id] : null)).ToArray();
var messages = withInsufficientBalance.Select(p => CreateMessage(p, balancesDict[p.AccountId])).ToArray();
var repaidInstalments = data.Loans.Where(l => paidIds.Contains(l.PaymentId)).Select(l => l.Id).ToArray();
var processedIds = toPay.Select(p => p.Id).ToArray();
var batchProcess = new BatchProcess {
ProcessingTimestamp = currentTime, Transfers = transfers, Messages = messages, RepaidInstalmentsIds = repaidInstalments, ProcessedPaymentsIds = processedIds
};
scenarioPartTimer.Restart();
SendData(batchProcess, scenarioId);
logger.Information($"Service='Automat' ScenarioId='{scenarioId}' Method='automat processbatch' Processing='{scenarioPartTimer.ElapsedMilliseconds}'");
scenarioPartTimer.Restart();
sessionRequester.Logout(token, scenarioId);
logger.Information($"Service='Automat' ScenarioId='{scenarioId}' Method='automat logout' Processing='{scenarioPartTimer.ElapsedMilliseconds}'");
logger.Information($"Service='Automat' ScenarioId='{scenarioId}' Method='automat scenario' Processing='{scenarioTimer.ElapsedMilliseconds}'");
//Thread.Sleep(automatSettings.SleepTime);
//currentTime += scenarioTimer.Elapsed;
currentTime += TimeSpan.FromMinutes(5);
}
}
private void buttonBatch_Click(object sender, EventArgs e)
{
Form bpForm = new BatchProcess();
bpForm.Show();
}
