protected void CheckForDuplicate(BatchJob batchJob, DataRow targetRow)
{
var keyFields = batchJob
.SourceMapping
.Where(b => (b.Index.HasValue || b.Default.HasValue) && b.IsIncluded && b.IsKey)
.Select(b => b.Name)
.ToList();
var keyValues = keyFields
.Select(k => targetRow[k])
.ToList();
// use a hash code of all the key values to check duplicate
var hashCode = keyValues
.Aggregate(HashCode.Seed, (hash, value) => hash.Combine(value));
bool added = _duplicateHash.Add(hashCode);
if (added)
{
return;
}
string message = "Duplicate key found. Field: {0}, Value: {1}"
.FormatWith(keyFields.ToDelimitedString(), keyValues.ToDelimitedString());
throw new DuplicateException(message);
}
private void QueueDwhDiff(SendManifestPackageDTO package)
{
var extracts = _extractRepository.GetAllRelated(package.ExtractId).ToList();
if (extracts.Any())
{
package.Extracts = extracts.Select(x => new ExtractDto()
{
Id = x.Id, Name = x.Name
}).ToList();
}
_ctSendService.NotifyPreSending();
var job1 =
BatchJob.StartNew(x => { SendDiffJobBaselines(package); });
var job2 =
BatchJob.ContinueBatchWith(job1, x => { SendDiffJobProfiles(package); });
var job3 =
BatchJob.ContinueBatchWith(job2, x => { SendDiffNewJobProfiles(package); });
var job4 =
BatchJob.ContinueBatchWith(job3, x => { SendDiffNewOtherJobProfiles(package); });
var job5 =
BatchJob.ContinueBatchWith(job4, x => { SendDiffCovidJobProfiles(package); });
var jobEnd =
BatchJob.ContinueBatchWith(job5, x => { _ctSendService.NotifyPostSending(package, _version); });
}
public void StartBatchJob_WithInfiniteItems_RunsMoreThanArbitraryTimes(int arbitraryRuns)
{
var batchSize = 1;
BatchJobOptions options = new BatchJobOptions()
{
MaxItems = -1,
BatchSize = batchSize,
DelayInSecs = 0
};
BatchJob job = new BatchJob(options, Mock.Of <ILogger>());
IOrderHandler handler = Mock.Of <IOrderHandler>();
Mock.Get(handler).Setup(h => h.HandleBatch(batchSize)).Verifiable();
// fire and forget since it is infinite thread
job.Start(handler.HandleBatch).Wait(1000);
// give time for iterations
job.Stop();
Mock.Get(handler).Verify((m) => m.HandleBatch(batchSize), Times.AtLeast(arbitraryRuns));
}
protected override void RowConverter(BatchJob r, DataRow record)
{
try
{
//CastDbValueRow(record, "BEGIN_ON", true, "DateTime");
r.BATCH_JOB_ID = CastDbValueRow(record, "BATCH_JOB_ID", true, "decimal");
r.BATCH_JOB_TYPE = CastDbValueRow(record, "BATCH_JOB_TYPE");
r.BATCH_STATUS = CastDbValueRow(record, "BATCH_STATUS");
r.COMMENTS = CastDbValueRow(record, "COMMENTS");
r.COMPLETED_ON = CastDbValueRow(record, "COMPLETED_ON", true, "DateTime");
r.ERR_MSG = CastDbValueRow(record, "ERR_MSG");
r.EXT_SYS = CastDbValueRow(record, "EXT_SYS");
r.NBR_DEL = CastDbValueRowDecimalOrZero(record, "NBR_DEL");
r.NBR_INS = CastDbValueRowDecimalOrZero(record, "NBR_INS");
r.NBR_REJECT = CastDbValueRowDecimalOrZero(record, "NBR_REJECT");
r.NBR_SEL = CastDbValueRowDecimalOrZero(record, "NBR_SEL");
r.NBR_UPD = CastDbValueRowDecimalOrZero(record, "NBR_UPD");
r.SUBMISSION_TYPE = CastDbValueRow(record, "SUBMISSION_TYPE");
r.SUBMITTED_BY = CastDbValueRow(record, "SUBMITTED_BY");
r.SUBMITTED_ON = CastDbValueRow(record, "SUBMITTED_ON", true, "DateTime");
r.TIMESPAN_GE = CastDbValueRow(record, "TIMESPAN_GE", true, "DateTime");
r.TIMESPAN_LT = CastDbValueRow(record, "TIMESPAN_LT", true, "DateTime");
}
catch (Exception ex)
{
throw ex;
}
}
public AjaxReturn BatchStatus(int id)
{
AjaxReturn result = new AjaxReturn();
BatchJob batch = AppModule.GetBatchJob(id);
if (batch == null)
{
Log("Invalid batch id");
result.error = "Invalid batch id";
}
else
{
if (batch == null)
{
Log("Invalid batch id");
result.error = "Invalid batch id";
}
else
{
Log("Batch {0}:{1}%:{2}", batch.Id, batch.PercentComplete, batch.Status);
result.data = batch;
if (batch.Finished)
{
result.error = batch.Error;
result.redirect = batch.Redirect;
Log("Batch finished - redirecting to {0}", batch.Redirect);
}
}
}
return(result);
}
public RunOptions()
{
Job = new BatchJob();
//TODO: make this a parameter once more providers are implemented
Job.ApplicationId = Plus.CadApplicationIds.SolidWorks;
}
public override List <BatchJob> ConvertDataReaderToList(IDataReader reader)
{
decimal val;
List <BatchJob> list = new List <BatchJob>();
while (reader.Read())
{
BatchJob item = new BatchJob();
item.COMPLETED_ON = DateTime.Parse(reader["completed_on"].ToString());
item.BATCH_STATUS = reader["batch_status"].ToString();
item.SUBMISSION_TYPE = reader["submission_type"].ToString();
item.SUBMITTED_BY = reader["submitted_by"].ToString();
item.ERR_MSG = reader["err_msg"].ToString();
item.COMMENTS = reader["comments"].ToString();
item.BATCH_JOB_TYPE = reader["batch_job_type"].ToString();
if (!String.IsNullOrEmpty(reader["timespan_ge"].ToString()))
{
item.TIMESPAN_GE = DateTime.Parse(reader["timespan_ge"].ToString());
}
if (!String.IsNullOrEmpty(reader["timespan_lt"].ToString()))
{
item.TIMESPAN_LT = DateTime.Parse(reader["timespan_lt"].ToString());
}
item.NBR_SEL = CastDbValueDecimalOrZero(reader, "NBR_SEL");
item.NBR_INS = CastDbValueDecimalOrZero(reader, "NBR_INS");
item.NBR_UPD = CastDbValueDecimalOrZero(reader, "NBR_UPD");
item.NBR_DEL = CastDbValueDecimalOrZero(reader, "NBR_DEL");
item.NBR_REJECT = CastDbValueDecimalOrZero(reader, "NBR_REJECT");
item.EXT_SYS = reader["ext_sys"].ToString();
item.BATCH_JOB_ID = int.Parse(reader["batch_job_id"].ToString());
list.Add(item);
}
return(list);
}
public void BatchRunnerOptionsTimeoutTest()
{
var mock = new Mock <IBatchRunnerModel>();
BatchJob opts = null;
mock.Setup(m => m.CreateExecutor(It.IsAny <BatchJob>())).Callback <BatchJob>(e => opts = e).Returns(new Mock <IBatchRunJobExecutor>().Object);
mock.Setup(m => m.InstalledVersions).Returns(new AppVersionInfo[] { new SwAppVersionInfo(SwVersion_e.Sw2019), new SwAppVersionInfo(SwVersion_e.Sw2020) });
var modelMock = mock.Object;
var msgSvcMock = new Mock <IMessageService>().Object;
var vm = new BatchManagerVM(modelMock, msgSvcMock);
vm.Document = new BatchDocumentVM("", new BatchJob(), modelMock, msgSvcMock);
vm.Document.Input.Add("abc");
vm.Document.Macros.Add("xyz");
vm.Document.Settings.Version = new SwAppVersionInfo(SwVersion_e.Sw2019);
vm.Document.Settings.Timeout = 300;
vm.Document.Settings.IsTimeoutEnabled = false;
vm.Document.Settings.IsTimeoutEnabled = true;
vm.Document.RunJobCommand.Execute(null);
Assert.AreEqual(300, opts.Timeout);
}
public void SaveBatchJobs(BatchJob model)
{
var queryString = @"
MERGE INTO [dbo].[BatchJobs] [target]
USING(
VALUES
(@id,@agent,@createdDateTime,@attachInfo,@metadata)
)[source]([Id],[Agent],[CreatedDateTime],[AttachInfo],[Metadata])
ON [source].[Id] = [target].[Id]
WHEN MATCHED THEN UPDATE SET
[target].[Agent] = [source].[Agent],
[target].[Metadata] = [source].[Metadata],
[target].[AttachInfo] = [source].[AttachInfo],
[target].[CreatedDateTime] = [source].[CreatedDateTime]
WHEN NOT MATCHED BY TARGET THEN
INSERT ([Id],[Agent],[CreatedDateTime],[AttachInfo],[Metadata])
VALUES([source].[Id],[source].[Agent],[source].[CreatedDateTime],[source].[AttachInfo],[source].[Metadata]);
";
using (var database = DatabaseFactory.GenerateDatabase())
{
database.Execute(queryString, new
{
@id = model.Id,
@agent = model.Agent,
@attachInfo = model.AttachInfo,
@metadata = model.Metadata,
@createdDateTime = model.CreatedDateTime
});
}
}
public async Task <BatchJob> Post([FromBody] BatchJob job)
{
if (job == null)
{
throw new BrowseCloudValidationException("You must include a valid BatchJob in the body of the request.");
}
if (job.DocumentId == null)
{
throw new BrowseCloudValidationException("You must include a valid BatchJob.DocumentId property");
}
if (job.JobType != JobType.CountingGridGeneration && job.TargetId == null)
{
throw new BrowseCloudValidationException("A job target id must be included for this type of job.");
}
var doc = await this.documentDbService.GetDocument <Document>(job.DocumentId.ToString());
if (doc == null)
{
throw new BrowseCloudValidationException($"There is no document {job.DocumentId}");
}
var idsToSearch = await this.graphService.GetAllCurrentUserIdentities();
if (!doc.UserCanModify(idsToSearch))
{
throw new BrowseCloudValidationException($"You are unauthorized to add a job to document {job.DocumentId}. No write access.");
}
return(await JobHelper.MakeBatchJob(doc, this.config.CommandPrefix ?? string.Empty, this.documentDbService, this.batchService, this.logger, job));
}
/// <summary>
/// Waits while the batch job is pending.
/// </summary>
/// <param name="batchJobService">The batch job service.</param>
/// <param name="batchJob">The batch job.</param>
/// <param name="isPending">True, if the job status is pending, false
/// otherwise.</param>
/// <param name="pollAttempts">The poll attempts to make while waiting for
/// batchjob completion or cancellation.</param>
private BatchJob WaitWhileJobIsPending(BatchJobService batchJobService, BatchJob batchJob,
out bool isPending, out long pollAttempts)
{
pollAttempts = 0;
isPending = true;
do
{
int sleepMillis = (int)Math.Pow(2, pollAttempts) *
POLL_INTERVAL_SECONDS_BASE * 1000;
Console.WriteLine("Sleeping {0} millis...", sleepMillis);
Thread.Sleep(sleepMillis);
Selector selector = new Selector()
{
fields = new string[] { BatchJob.Fields.Id, BatchJob.Fields.Status,
BatchJob.Fields.DownloadUrl, BatchJob.Fields.ProcessingErrors,
BatchJob.Fields.ProgressStats },
predicates = new Predicate[] {
Predicate.Equals(BatchJob.Fields.Id, batchJob.id)
}
};
batchJob = batchJobService.get(selector).entries[0];
Console.WriteLine("Batch job ID {0} has status '{1}'.", batchJob.id, batchJob.status);
isPending = PENDING_STATUSES.Contains(batchJob.status);
} while (isPending && ++pollAttempts <= MAX_RETRIES);
return(batchJob);
}
private void ReadTeamLevelDatasource(Action <ExcelSignleRow> pass,
BatchJob job,
FileEntity item,
DataSourceNames source)
{
try
{
var startRuning = DateTime.UtcNow;
var startRowIndex = source == DataSourceNames.TeamLevelReport ? 19 : 1;
var startColumnIndex = source == DataSourceNames.TeamLevelReport ? 2 : 1;
foreach (var jMetadata in ExcelHelper.ReadExcel <JObject>(item.FileName, 0, startRowIndex, startColumnIndex))
{
pass(new ExcelSignleRow()
{
BatchJob = job,
DataSourceName = source,
DateTime = startRuning,
JMetadata = jMetadata,
Properties = null,
});
}
}
catch (IOException ex)
{
Logger.Error($"Excel file can't be read;{ex.Message}", ex);
return;
}
}
private void ReadIndividualDatasource(Action <ExcelSignleRow> pass,
BatchJob jobs,
FileEntity item,
DataSourceNames source)
{
try
{
var startRuning = DateTime.UtcNow;
var count = 0;
foreach (var row in ExcelHelper.ReadExcel <ExcelSignleRow>(item.FileName, Convert, 0))
{
row.DataSourceName = source;
row.BatchJob = jobs;
var descriptors = Unity.GetDescriptor(source);
for (var i = 0; i < row.Properties.Length; i++)
{
var header = row.Properties[i].Descriptor.Header;
row.Properties[i].Descriptor = descriptors.TryFirst(o => o.Header.Equals(header, StringComparison.OrdinalIgnoreCase));
}
row.Properties = row.Properties.Where(o => o.Descriptor != null).ToArray();
row.Fix(source, this.directly.GetMappings());
row.DateTime = startRuning;
count++;
pass(row);
}
Logger.Warn($"Total Rows {count};{item.FileName}");
}
catch (IOException ex)
{
Logger.Error($"Excel file can't be read;{ex.Message}", ex);
return;
}
}
protected void CheckForDuplicate(BatchJob batchJob, DataRow targetRow)
{
var keyFields = batchJob
.SourceMapping
.Where(b => (b.Index.HasValue || b.Default.HasValue) && b.IsIncluded && b.IsKey)
.Select(b => b.Name)
.ToList();
var keyValues = keyFields
.Select(k => targetRow[k])
.ToList();
// use a hash code of all the key values to check duplicate
var hashCode = keyValues
.Aggregate(HashCode.Seed, (hash, value) => hash.Combine(value));
bool added = _duplicateHash.Add(hashCode);
if (added)
return;
string message = "Duplicate key found. Field: {0}, Value: {1}"
.FormatWith(keyFields.ToDelimitedString(), keyValues.ToDelimitedString());
throw new DuplicateException(message);
}
public void BatchRunnerOptionsTimeoutTest()
{
var mock = new Mock <IBatchRunnerModel>();
BatchJob opts = null;
mock.Setup(m => m.CreateExecutor(It.IsAny <BatchJob>())).Callback <BatchJob>(e => opts = e).Returns(new Mock <IBatchRunJobExecutor>().Object);
mock.Setup(m => m.InstalledVersions).Returns(new ISwVersion[] { SwApplicationFactory.CreateVersion(SwVersion_e.Sw2019), SwApplicationFactory.CreateVersion(SwVersion_e.Sw2020) });
mock.Setup(m => m.GetVersionId(It.IsAny <IXVersion>())).Returns("Sw2020");
mock.Setup(m => m.ParseVersion(It.IsAny <string>())).Returns(new Mock <IXVersion>().Object);
var modelMock = mock.Object;
var msgSvcMock = new Mock <IMessageService>().Object;
var vm = new BatchManagerVM(modelMock, msgSvcMock);
vm.Document = new BatchDocumentVM("", new BatchJob(), modelMock, msgSvcMock);
vm.Document.Input.Add("abc");
vm.Document.Macros.Add(new MacroData()
{
FilePath = "xyz"
});
vm.Document.Settings.Version = SwApplicationFactory.CreateVersion(SwVersion_e.Sw2019);
vm.Document.Settings.Timeout = 300;
vm.Document.Settings.IsTimeoutEnabled = false;
vm.Document.Settings.IsTimeoutEnabled = true;
vm.Document.RunJobCommand.Execute(null);
Assert.AreEqual(300, opts.Timeout);
}
public virtual void ValidateRow(BatchJob batchJob, DataRow targetRow)
{
// check for DBNull or null
CheckForNull(batchJob, targetRow);
// look for duplicate key values
CheckForDuplicate(batchJob, targetRow);
}
public BatchDocumentMockVM(string name, BatchJob job, ICadApplicationInstanceProvider[] appProviders,
IJournalExporter[] journalExporters, IResultsSummaryExcelExporter[] resultsExporters,
IMessageService msgSvc, IXLogger logger, Func <BatchJob, IBatchRunJobExecutor> execFact,
IBatchApplicationProxy batchAppProxy, MainWindow parentWnd, IRibbonButtonCommand[] backstageCmds)
: base(name, job, appProviders, journalExporters, resultsExporters,
msgSvc, logger, execFact, batchAppProxy, parentWnd, backstageCmds)
{
}
public virtual void ValidateRow(BatchJob batchJob, DataRow targetRow)
{
// check for DBNull or null
CheckForNull(batchJob, targetRow);
// look for duplicate key values
CheckForDuplicate(batchJob, targetRow);
}
public static BatchJob Build(Action <BatchBuilder> builder)
{
var batchJob = new BatchJob();
var batchBuilder = new BatchBuilder(batchJob);
builder(batchBuilder);
return(batchJob);
}
public void SaveJobToFile(BatchJob job, string filePath)
{
var svc = new UserSettingsService();
svc.StoreSettings(job, filePath);
AppendRecentFiles(filePath);
}
public TasklistViewItem(BatchJob batchjob) : base(new string[6])
{
_batchjob = batchjob;
UpdateListItem();
UpdateStatus();
AttachEvents();
ResetProgress();
}
/// <summary>Snippet for GetBatchJob</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void GetBatchJobResourceNames()
{
// Create client
BatchJobServiceClient batchJobServiceClient = BatchJobServiceClient.Create();
// Initialize request argument(s)
BatchJobName resourceName = BatchJobName.FromCustomerBatchJob("[CUSTOMER]", "[BATCH_JOB]");
// Make the request
BatchJob response = batchJobServiceClient.GetBatchJob(resourceName);
}
/// <summary>Snippet for GetBatchJob</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public void GetBatchJob()
{
// Create client
BatchJobServiceClient batchJobServiceClient = BatchJobServiceClient.Create();
// Initialize request argument(s)
string resourceName = "customers/[CUSTOMER_ID]/batchJobs/[BATCH_JOB_ID]";
// Make the request
BatchJob response = batchJobServiceClient.GetBatchJob(resourceName);
}
/// <summary>Snippet for GetBatchJobAsync</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public async Task GetBatchJobResourceNamesAsync()
{
// Create client
BatchJobServiceClient batchJobServiceClient = await BatchJobServiceClient.CreateAsync();
// Initialize request argument(s)
BatchJobName resourceName = BatchJobName.FromCustomerBatchJob("[CUSTOMER]", "[BATCH_JOB]");
// Make the request
BatchJob response = await batchJobServiceClient.GetBatchJobAsync(resourceName);
}
public BulkImportJob Map(BatchJob batchJob)
{
var job = new BulkImportJob();
job.ImportId = batchJob.ImportId;
job.batchId = batchJob.batchId;
job.Status = batchJob.Status;
return(job);
}
void MatchItemToType(BatchJob bj)
{
foreach (BatchJobType bjt in BatchJobTypes)
{
if (bjt.BATCH_JOB_TYPE == bj.BATCH_JOB_TYPE)
{
bj.BatchJobTypeDescription = bjt.DESCR;
}
}
}
/// <summary>Snippet for GetBatchJobAsync</summary>
/// <remarks>
/// This snippet has been automatically generated for illustrative purposes only.
/// It may require modifications to work in your environment.
/// </remarks>
public async Task GetBatchJobAsync()
{
// Create client
BatchJobServiceClient batchJobServiceClient = await BatchJobServiceClient.CreateAsync();
// Initialize request argument(s)
string resourceName = "customers/[CUSTOMER]/batchJobs/[BATCH_JOB]";
// Make the request
BatchJob response = await batchJobServiceClient.GetBatchJobAsync(resourceName);
}
public BatchRunJobExecutor(BatchJob job, IApplicationProvider appProvider)
{
m_Job = job;
m_AppProvider = appProvider;
m_LogWriter = new LogWriter();
m_PrgHander = new ProgressHandler();
m_IsExecuting = false;
}
public Task <BatchJobExecutionResult> Execute(BatchJob batchJob)
{
var message = string.Format("There is no executor for this job. To create one, implement {0}<{1}>",
typeof(IBatchJobExecutor).FullName,
batchJob.GetType().FullName);
var batchJobExecutionResult = BatchJobExecutionResult.Failure(message);
_setBatchJobExecutionStatus.Complete(batchJob, batchJobExecutionResult);
return(Task.FromResult(batchJobExecutionResult));
}
public async Task <BatchJobExecutionResult> Execute(BatchJob batchJob)
{
return(await BatchJobExecutionResult.TryAsync(() =>
{
T job = batchJob as T;
return job == null
? Task.FromResult(BatchJobExecutionResult.Failure("Batch job is not of the correct type"))
: Execute(job);
}));
}
/// <summary>Snippet for GetBatchJob</summary>
public void GetBatchJob()
{
// Snippet: GetBatchJob(string, CallSettings)
// Create client
BatchJobServiceClient batchJobServiceClient = BatchJobServiceClient.Create();
// Initialize request argument(s)
string resourceName = "customers/[CUSTOMER]/batchJobs/[BATCH_JOB]";
// Make the request
BatchJob response = batchJobServiceClient.GetBatchJob(resourceName);
// End snippet
}
/// <summary>Snippet for GetBatchJob</summary>
public void GetBatchJobResourceNames()
{
// Snippet: GetBatchJob(BatchJobName, CallSettings)
// Create client
BatchJobServiceClient batchJobServiceClient = BatchJobServiceClient.Create();
// Initialize request argument(s)
BatchJobName resourceName = BatchJobName.FromCustomerBatchJob("[CUSTOMER_ID]", "[BATCH_JOB_ID]");
// Make the request
BatchJob response = batchJobServiceClient.GetBatchJob(resourceName);
// End snippet
}
protected IList<string> SelectFields(BatchJob batchJob, IEnumerable<string> fields)
{
var selectFields = batchJob
.SourceMapping
.Where(b => (b.Index.HasValue || b.Default.HasValue) && b.IsIncluded)
.Select(b => b.Name)
.Intersect(fields)
.ToList();
return selectFields;
}
protected void CheckForNull(BatchJob batchJob, DataRow targetRow)
{
var requiredFields = batchJob
.SourceMapping
.Where(b => (b.Index.HasValue || b.Default.HasValue) && b.IsIncluded && b.CanBeNull == false)
.Select(b => b.Name)
.ToList();
// first null
var invalidField = requiredFields
.FirstOrDefault(targetRow.IsNull);
if (!invalidField.HasValue())
return;
string message = "Field '{0}' can not be null. {1}"
.FormatWith(invalidField, targetRow.ItemArray.ToDelimitedString());
throw new ValidationException(message);
}
