/// <summary>
/// Tests if step execution should start.
/// </summary>
/// <param name="allowStartIfComplete"></param>
/// <param name="stepExecution"></param>
/// <param name="lastStepExecution"></param>
/// <returns></returns>
/// <exception cref="JobExecutionException"></exception>
private bool ShouldStart(bool allowStartIfComplete, StepExecution stepExecution, StepExecution lastStepExecution)
{
if (lastStepExecution == null)
{
return(true);
}
BatchStatus stepStatus = lastStepExecution.BatchStatus;
//unknown status handling
HandleUnknownStatus(stepStatus);
//Completed status handling
if (stepStatus == BatchStatus.Completed)
{
return(allowStartIfComplete || IsSameJobExecution(stepExecution, lastStepExecution));
}
//Stopped or failed permits restart
if (HandleStoppedOrFailedStatus(stepStatus))
{
return(true);
}
//other statuses should not allow restart.
PreventFromRestarting(stepStatus);
throw new JobExecutionException(
string.Format("Cannot restart step from {0} status. "
+ "We believe the old execution was abandoned and therefore has been marked as un-restartable.",
stepStatus));
}
/// <summary>
/// @see IJobOperator#Stop .
/// </summary>
/// <param name="executionId"></param>
/// <returns></returns>
/// <exception cref="NoSuchJobException"> </exception>
/// <exception cref="JobExecutionNotRunningException"> </exception>
public bool Stop(long executionId)
{
JobExecution jobExecution = FindExecutionById(executionId);
// Indicate the execution should be stopped by setting it's status to
// 'STOPPING'. It is assumed that
// the step implementation will check this status at chunk boundaries.
BatchStatus status = jobExecution.Status;
if (!(status == BatchStatus.Started || status == BatchStatus.Starting))
{
throw new JobExecutionNotRunningException(
string.Format("JobExecution must be running so that it can be stopped: {0}", jobExecution));
}
jobExecution.Status = BatchStatus.Stopping;
JobRepository.Update(jobExecution);
try
{
IJob job = JobRegistry.GetJob(jobExecution.JobInstance.JobName);
var locator = job as IStepLocator;
if (locator != null)
{
//can only process as StepLocator is the only way to get the step object
//get the current stepExecution
foreach (StepExecution stepExecution in jobExecution.StepExecutions)
{
if (stepExecution.BatchStatus.IsRunning())
{
try
{
//have the step execution that's running -> need to 'stop' it
IStep step = locator.GetStep(stepExecution.StepName);
var taskletStep = step as TaskletStep;
if (taskletStep != null)
{
ITasklet tasklet = taskletStep.Tasklet;
var stoppableTasklet = tasklet as IStoppableTasklet;
if (stoppableTasklet != null)
{
StepSynchronizationManager.Register(stepExecution);
stoppableTasklet.Stop();
StepSynchronizationManager.Release();
}
}
}
catch (NoSuchStepException e)
{
_logger.Warn("Step not found {0}", e.Message);
}
}
}
}
}
catch (NoSuchJobException e)
{
_logger.Warn("Cannot find Job object {0}", e.Message);
}
return(true);
}
/// <summary>
/// Sets the editable for batch status.
/// </summary>
/// <param name="batch">The batch.</param>
private void SetEditableForBatchStatus(BatchStatus batchStatus)
{
if (batchStatus == BatchStatus.Closed)
{
tbName.ReadOnly = true;
dtpStart.Enabled = false;
dtpEnd.Enabled = false;
tbControlAmount.ReadOnly = true;
nbControlItemCount.ReadOnly = true;
campCampus.Enabled = false;
tbAccountingCode.ReadOnly = true;
tbNote.ReadOnly = true;
}
else
{
tbName.ReadOnly = false;
dtpStart.Enabled = true;
dtpEnd.Enabled = true;
tbControlAmount.ReadOnly = false;
nbControlItemCount.ReadOnly = false;
campCampus.Enabled = true;
tbAccountingCode.ReadOnly = false;
tbNote.ReadOnly = false;
}
}
/// <summary>
/// Creates a row mapper that for step executions.
/// </summary>
/// <param name="jobExecution">the job execution to use when creating the step executions</param>
/// <returns>a row mapper</returns>
private static RowMapper <StepExecution> GetStepExecutionRowMapper(JobExecution jobExecution)
{
return((dataRecord, i) =>
{
var wrapper = new DataRecordWrapper(dataRecord);
var stepExecution = new StepExecution(wrapper.Get <string>((1)), jobExecution, wrapper.Get <long>((0)))
{
StartTime = wrapper.Get <DateTime>((2)),
EndTime = wrapper.Get <DateTime>((3)),
BatchStatus = BatchStatus.ValueOf(wrapper.Get <string>((4))),
CommitCount = wrapper.Get <int>((5)),
ReadCount = wrapper.Get <int>((6)),
FilterCount = wrapper.Get <int>((7)),
WriteCount = wrapper.Get <int>((8)),
ExitStatus = new ExitStatus(wrapper.Get <string>((9)), wrapper.Get <string>(10)),
ReadSkipCount = wrapper.Get <int>((11)),
WriteSkipCount = wrapper.Get <int>((12)),
ProcessSkipCount = wrapper.Get <int>((13)),
RollbackCount = wrapper.Get <int>((14)),
LastUpdated = wrapper.Get <DateTime?>(15),
Version = wrapper.Get <int?>((16))
};
return stepExecution;
});
}
private void OnUpdateBatch(BatchStatus response, Dictionary <string, object> customData)
{
Log.Debug("TestCompareComplyV1.OnUpdateBatch()", "OnUpdateBatch Response: {0}", customData["json"].ToString());
Test(response != null);
Test(!string.IsNullOrEmpty(response.BatchId));
updateBatchTested = true;
}
public void UploadKeysToTheGateway(int fromLastNumberOfDays, int batchSize, int?batchCountLimit = null)
{
ValidateParameters(fromLastNumberOfDays, batchSize, batchCountLimit);
var stats = new BatchUploadStats();
BatchStatus lastStatus = new BatchStatus();
do
{
if (IsLimitReached(currentValue: stats.CurrentBatchNumber, limitValue: batchCountLimit))
{
_logger.LogInformation($"Limit of {batchCountLimit} batches has been reached. Stopping upload process");
break;
}
++stats.CurrentBatchNumber;
lastStatus = UploadNextBatch(batchSize, fromLastNumberOfDays);
stats.TotalKeysProcessed += lastStatus.KeysProcessed;
stats.TotalKeysSent += lastStatus.KeysSent;
}while (lastStatus.ProcessedSuccessfully && lastStatus.NextBatchExists);
if (lastStatus.ProcessedSuccessfully)
{
_logger.LogInformation($"Upload ended. Last batch processed successfully. Upload status: {stats.TotalKeysSent} keys ({ stats.CurrentBatchNumber} batches) has been sent from {stats.TotalKeysProcessed} records processed.");
}
else
{
_logger.LogError($"Upload interrupted. Last batch processed with error. Upload status: {stats.TotalKeysSent} keys ({ stats.CurrentBatchNumber - 1} batches) has been sent from {stats.TotalKeysProcessed} records processed.");
throw new InvalidOperationException($"Upload interrupted! Error occurred while sending batch {stats.CurrentBatchNumber}.");
}
}
protected virtual BatchStatus GetBatchStatusFromReader(IDataReader reader)
{
EntityConverter <BatchStatus> batchStatusEntity = new EntityConverter <BatchStatus>();
BatchStatus codeMasterBatchStatus = batchStatusEntity.Convert(reader);
return(codeMasterBatchStatus);
}
public void IsLessThanTest()
{
BatchStatus stopped = BatchStatus.Starting;
Assert.IsTrue(stopped.IsLessThan(BatchStatus.Stopped));
Assert.IsFalse(stopped.IsLessThan(BatchStatus.Completed));
}
public void MatchTest()
{
string started = "STARTED";
BatchStatus startedStatus = BatchStatus.Match(started);
Assert.AreEqual(BatchStatus.Started, startedStatus);
}
public CheckScannerBatch UpdateBatchStatus(string batchName, BatchStatus newStatus)
{
WithDbCommand(dbCommand =>
{
dbCommand.CommandType = CommandType.Text;
dbCommand.CommandText = "UPDATE Batches SET Exported = @BatchStatus WHERE IDBatch = @IDBatch";
var batchStatusParam = dbCommand.CreateParameter();
batchStatusParam.ParameterName = "BatchStatus";
batchStatusParam.DbType = DbType.Int16;
batchStatusParam.Value = newStatus == BatchStatus.Exported ? 1 : 0;
dbCommand.Parameters.Add(batchStatusParam);
var idBatchParam = dbCommand.CreateParameter();
idBatchParam.ParameterName = "IDBatch";
idBatchParam.DbType = DbType.String;
idBatchParam.Value = batchName;
idBatchParam.Size = batchName.Length;
dbCommand.Parameters.Add(idBatchParam);
dbCommand.Prepare();
dbCommand.ExecuteNonQuery();
return(true);
});
return(new CheckScannerBatch
{
Name = batchName,
Status = newStatus
});
}
public void IsGreaterThanTest()
{
BatchStatus stopped = BatchStatus.Stopped;
Assert.IsTrue(stopped.IsGreaterThan(BatchStatus.Starting));
Assert.IsFalse(stopped.IsGreaterThan(BatchStatus.Abandoned));
}
public void ValueOfTest()
{
BatchStatus st1 = BatchStatus.ValueOf("COMPLETED");
Assert.AreEqual(BatchStatus.Completed, st1);
BatchStatus st2 = BatchStatus.ValueOf("STARTING");
Assert.AreEqual(BatchStatus.Starting, st2);
BatchStatus st3 = BatchStatus.ValueOf("STARTED");
Assert.AreEqual(BatchStatus.Started, st3);
BatchStatus st4 = BatchStatus.ValueOf("STOPPING");
Assert.AreEqual(BatchStatus.Stopping, st4);
BatchStatus st5 = BatchStatus.ValueOf("STOPPED");
Assert.AreEqual(BatchStatus.Stopped, st5);
BatchStatus st6 = BatchStatus.ValueOf("FAILED");
Assert.AreEqual(BatchStatus.Failed, st6);
BatchStatus st7 = BatchStatus.ValueOf("ABANDONED");
Assert.AreEqual(BatchStatus.Abandoned, st7);
BatchStatus st8 = BatchStatus.ValueOf("UNKNOWN");
Assert.AreEqual(BatchStatus.Unknown, st8);
}
/// <summary>
/// Creates a row mapper for job executions.
/// </summary>
/// <param name="jobInstance">the job instance of the executions to create (optional)</param>
/// <returns>a row mapper for job executions</returns>
private RowMapper <JobExecution> GetJobExecutionRowMapper(JobInstance jobInstance = null)
{
JobParameters jobParameters = null;
return((dataRecord, i) =>
{
var wrapper = new DataRecordWrapper(dataRecord);
var id = wrapper.Get <long>(0);
var jobConfigurationLocation = wrapper.Get <string>(9);
if (jobParameters == null)
{
jobParameters = GetJobParameters(id);
}
var jobExecution = jobInstance == null ? new JobExecution(id, jobParameters, jobConfigurationLocation)
: new JobExecution(jobInstance, id, jobParameters, jobConfigurationLocation);
jobExecution.StartTime = wrapper.Get <DateTime?>(1);
jobExecution.EndTime = wrapper.Get <DateTime?>(2);
jobExecution.Status = dataRecord.IsDBNull(3) ? null : BatchStatus.ValueOf(dataRecord.GetString(3));
jobExecution.ExitStatus = new ExitStatus(wrapper.Get <string>(4), wrapper.Get <string>(5));
jobExecution.CreateTime = wrapper.Get <DateTime>(6);
jobExecution.LastUpdated = wrapper.Get <DateTime?>(7);
jobExecution.Version = wrapper.Get <int?>(8);
return jobExecution;
});
}
public void IsUnsuccessfulTest()
{
BatchStatus shouldbesuccessful = BatchStatus.Starting;
BatchStatus shouldbeunsuccessful = BatchStatus.Failed;
Assert.IsFalse(shouldbesuccessful.IsUnsuccessful());
Assert.IsTrue(shouldbeunsuccessful.IsUnsuccessful());
}
public void IsRunningTest()
{
BatchStatus shouldberunning = BatchStatus.Started;
BatchStatus shouldnotberunning = BatchStatus.Stopped;
Assert.IsTrue(shouldberunning.IsRunning());
Assert.IsFalse(shouldnotberunning.IsRunning());
}
public void MaxTest()
{
BatchStatus low = BatchStatus.Completed;
BatchStatus hi = BatchStatus.Failed;
BatchStatus max = BatchStatus.Max(low, hi);
Assert.AreEqual(hi, max);
}
private static bool HandleStoppedOrFailedStatus(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Stopped || stepStatus == BatchStatus.Failed)
{
return(true);
}
return(false);
}
/// <summary>
/// Job interruption handling.
/// </summary>
/// <param name="execution"></param>
/// <param name="e"></param>
private void HandleJobInterruptedException(JobExecution execution, JobInterruptedException e)
{
Logger.Info("Encountered interruption executing job: " + e.Message);
Logger.Debug(e, "Full exception");
execution.ExitStatus = GetDefaultExitStatusForFailure(e, execution);
execution.Status = BatchStatus.Max(BatchStatus.Stopped, e.Status);
execution.AddFailureException(e);
}
public void UpgradeToTest()
{
BatchStatus started = BatchStatus.Started;
// ! reentrant
started = started.UpgradeTo(BatchStatus.Stopped);
//should not be running, as upgraded to stopped
Assert.IsFalse(started.IsRunning());
}
/// <summary>
/// Update a pending or active batch-processing request.
///
/// Update a pending or active batch-processing request. You can rescan the input bucket to check for new
/// documents or cancel a request.
/// </summary>
/// <param name="batchId">The ID of the batch-processing request you want to update.</param>
/// <param name="action">The action you want to perform on the specified batch-processing request.</param>
/// <param name="modelId">The analysis model to be used by the service. For the `/v1/element_classification` and
/// `/v1/comparison` methods, the default is `contracts`. For the `/v1/tables` method, the default is `tables`.
/// These defaults apply to the standalone methods as well as to the methods' use in batch-processing requests.
/// (optional)</param>
/// <param name="customData">Custom data object to pass data including custom request headers.</param>
/// <returns><see cref="BatchStatus" />BatchStatus</returns>
public BatchStatus UpdateBatch(string batchId, string action, string modelId = null, Dictionary <string, object> customData = null)
{
if (string.IsNullOrEmpty(batchId))
{
throw new ArgumentNullException(nameof(batchId));
}
if (string.IsNullOrEmpty(action))
{
throw new ArgumentNullException(nameof(action));
}
if (string.IsNullOrEmpty(VersionDate))
{
throw new ArgumentNullException("versionDate cannot be null.");
}
BatchStatus result = null;
try
{
IClient client = this.Client;
if (_tokenManager != null)
{
client = this.Client.WithAuthentication(_tokenManager.GetToken());
}
var restRequest = client.PutAsync($"{this.Endpoint}/v1/batches/{batchId}");
restRequest.WithArgument("version", VersionDate);
if (!string.IsNullOrEmpty(action))
{
restRequest.WithArgument("action", action);
}
if (!string.IsNullOrEmpty(modelId))
{
restRequest.WithArgument("model_id", modelId);
}
if (customData != null)
{
restRequest.WithCustomData(customData);
}
restRequest.WithHeader("X-IBMCloud-SDK-Analytics", "service_name=compare-comply;service_version=v1;operation_id=UpdateBatch");
result = restRequest.As <BatchStatus>().Result;
if (result == null)
{
result = new BatchStatus();
}
result.CustomData = restRequest.CustomData;
}
catch (AggregateException ae)
{
throw ae.Flatten();
}
return(result);
}
private static void HandleUnknownStatus(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Unknown)
{
throw new JobExecutionException("Cannot restart step from UNKNOWN status. "
+ "The last execution ended with a failure that could not be rolled back, "
+ "so it may be dangerous to proceed. " +
"Manual intervention is probably necessary.");
}
}
private static void PreventFromRestarting(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Started || stepStatus == BatchStatus.Starting ||
stepStatus == BatchStatus.Stopping)
{
throw new JobExecutionException(string.Format(
"Cannot restart step from {0} status. "
+ "The old execution may still be executing, so you may need to verify manually that this is the case.", stepStatus));
}
}
public List <BatchStatus> ListBatchStatuses()
{
BatchStatus emptyBatchStatus = new BatchStatus();
emptyBatchStatus.BatchStatusId = 0;
emptyBatchStatus.BatchStatusName = "Select Batch Status";
List <BatchStatus> listBatchStatus = repository.BatchStatuses.OrderBy(x => x.BatchStatusName).ToList();
listBatchStatus.Insert(0, emptyBatchStatus);
return(listBatchStatus);
}
public BulkImportJobStatus Map(BatchStatus batchStatus)
{
var status = new BulkImportJobStatus();
status.Id = batchStatus.batchId;
status.ImportId = batchStatus.importId;
status.Status = batchStatus.status;
status.Message = batchStatus.message;
status.NumOfLeadsProcessed = batchStatus.numOfLeadsProcessed;
status.NumOfRowsWithWarning = batchStatus.numOfRowsWithWarning;
status.NumOfRowsFailed = batchStatus.numOfRowsFailed;
return(status);
}
/// <summary>
/// Determines whether [is valid batch status change] [the specified original status].
/// </summary>
/// <param name="origStatus">The original status.</param>
/// <param name="newStatus">The new status.</param>
/// <param name="currentPerson">The current person.</param>
/// <param name="errorMessage">The error message.</param>
/// <returns>
/// <c>true</c> if [is valid batch status change] [the specified original status]; otherwise, <c>false</c>.
/// </returns>
public bool IsValidBatchStatusChange(BatchStatus origStatus, BatchStatus newStatus, Person currentPerson, out string errorMessage)
{
errorMessage = string.Empty;
if (origStatus == BatchStatus.Closed && newStatus != BatchStatus.Closed)
{
if (!this.IsAuthorized("ReopenBatch", currentPerson))
{
errorMessage = "User is not authorized to reopen a closed batch";
return(false);
}
}
return(true);
}
private EnterpriseEventCacheDataRequest GetInvalidCacheRequestData()
{
BatchDetails linkBatchDetails = new BatchDetails()
{
Href = @"http://tempuri.org.uk/batch/7b4cdb10-ddfd-4ed6-b2be-d1543d8b7272"
};
BatchStatus linkBatchStatus = new BatchStatus()
{
Href = @"http://tempuri.org.uk/batch/7b4cdb10-ddfd-4ed6-b2be-d1543d8b7272/status"
};
Get linkGet = new Get()
{
Href = @"http://tempuri.org.uk/batch/7b4cdb10-ddfd-4ed6-b2be-d1543d8b7272/files/exchangeset123.zip",
};
CacheLinks links = new CacheLinks()
{
BatchDetails = linkBatchDetails,
BatchStatus = linkBatchStatus,
Get = linkGet
};
return(new EnterpriseEventCacheDataRequest
{
Links = links,
BusinessUnit = "ABC",
Attributes = new List <Attribute> {
new Attribute {
Key = "Agency", Value = "DE"
},
new Attribute {
Key = "CellName", Value = "DE416050"
},
new Attribute {
Key = "EditionNumber", Value = "2"
},
new Attribute {
Key = "UpdateNumber", Value = "0"
},
new Attribute {
Key = "ProductCode", Value = "DEF"
}
},
BatchId = "d6cd4d37-4d89-470d-9a33-82b3d7f54b6e",
BatchPublishedDate = DateTime.UtcNow
});
}
/// <summary>
/// Sets the heading batch status.
/// </summary>
/// <param name="batchStatus">The batch status.</param>
private void SetHeadingBatchStatus(BatchStatus batchStatus)
{
hlStatus.Text = batchStatus.ConvertToString();
switch (batchStatus)
{
case BatchStatus.Pending:
hlStatus.LabelType = LabelType.Danger;
break;
case BatchStatus.Open:
hlStatus.LabelType = LabelType.Warning;
break;
case BatchStatus.Closed:
hlStatus.LabelType = LabelType.Default;
break;
}
}
public ICommandProcessorOptions <TKey> GetImportOptions <TKey>(
ImportOptions options,
ICommandProcessorBatchConfiguration <TKey> configuration)
{
var batchStatus = new BatchStatus
{
From = FromInit,
Until = UntilInit,
Completed = false
};
if (!Directory.Exists(ImportFolder))
{
Directory.CreateDirectory(ImportFolder);
}
return(options.CreateProcessorOptions(batchStatus, configuration));
}
public IEnumerator TestCreateBatch()
{
Log.Debug("CompareComplyServiceV1IntegrationTests", "Attempting to CreateBatch...");
BatchStatus createBatchResponse = null;
using (FileStream fsInput = File.OpenRead(objectStorageCredentialsInputFilepath))
{
using (FileStream fsOutput = File.OpenRead(objectStorageCredentialsOutputFilepath))
{
using (MemoryStream msInput = new MemoryStream())
{
using (MemoryStream msOutput = new MemoryStream())
{
fsInput.CopyTo(msInput);
fsOutput.CopyTo(msOutput);
service.CreateBatch(
callback: (DetailedResponse <BatchStatus> response, IBMError error) =>
{
Log.Debug("CompareComplyServiceV1IntegrationTests", "CreateBatch result: {0}", response.Response);
createBatchResponse = response.Result;
createdBatchId = createBatchResponse.BatchId;
Assert.IsNotNull(createBatchResponse);
Assert.IsNotNull(createdBatchId);
Assert.IsNull(error);
},
function: "html_conversion",
inputCredentialsFile: msInput,
inputBucketLocation: "us-south",
inputBucketName: "compare-comply-integration-test-bucket-input",
outputCredentialsFile: msOutput,
outputBucketLocation: "us-south",
outputBucketName: "compare-comply-integration-test-bucket-output",
model: "contracts"
);
}
}
}
}
while (createBatchResponse == null)
{
yield return(null);
}
}
private static BatchStatus GetBatchStatus(xml.erp.BatchStatus batchStatus)
{
BatchStatus _status = default(BatchStatus);
switch (batchStatus)
{
case xml.erp.BatchStatus.Final:
_status = BatchStatus.Final;
break;
case xml.erp.BatchStatus.Intermediate:
_status = BatchStatus.Intermediate;
break;
case xml.erp.BatchStatus.Progress:
_status = BatchStatus.Progress;
break;
}
return(_status);
}
public CheckScannerBatch UpdateBatchStatus(string batchName, BatchStatus newStatus)
{
return (_checkScannerDao.UpdateBatchStatus(batchName, newStatus));
}
private void UpdateStatus(JobExecution jobExecution, BatchStatus status)
{
jobExecution.Status = status;
JobRepository.Update(jobExecution);
}
private static void PreventFromRestarting(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Started || stepStatus == BatchStatus.Starting
|| stepStatus == BatchStatus.Stopping)
{
throw new JobExecutionException(string.Format(
"Cannot restart step from {0} status. "
+ "The old execution may still be executing, so you may need to verify manually that this is the case.",stepStatus));
}
}
private static void HandleUnknownStatus(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Unknown)
{
throw new JobExecutionException("Cannot restart step from UNKNOWN status. "
+ "The last execution ended with a failure that could not be rolled back, "
+ "so it may be dangerous to proceed. " +
"Manual intervention is probably necessary.");
}
}
private static bool HandleStoppedOrFailedStatus(BatchStatus stepStatus)
{
if (stepStatus == BatchStatus.Stopped || stepStatus == BatchStatus.Failed)
{
return true;
}
return false;
}
