// [END sync_query]
/// <summary>Inserts an asynchronous query Job for a particular query.</summary>
/// <param name="bigquery">an authorized BigQuery client</param>
/// <param name="projectId">a string containing the project ID</param>
/// <param name="querySql">the actual query string</param>
/// <param name="batch">true if you want to run the query as BATCH</param>
/// <returns>a reference to the inserted query job</returns>
// [START async_query]
public static Job AsyncQuery(BigqueryService bigquery, string projectId,
string querySql, bool batch)
{
JobConfigurationQuery queryConfig = new JobConfigurationQuery
{
Query = querySql,
Priority = batch ? "BATCH" : null
};
JobConfiguration config = new JobConfiguration { Query = queryConfig };
Job job = new Job { Configuration = config };
return bigquery.Jobs.Insert(job, projectId).Execute();
}
// [END build_service]
// [START run_query]
/// <summary>
/// Executes the given query synchronously.
/// </summary>
/// <param name="querySql">the query to execute.</param>
/// <param name="bigquery">the BigquerService object.</param>
/// <param name="projectId">the id of the project under which to run the query.</param>
/// <returns>a list of the results of the query.</returns>
private static async Task<IList<TableRow>> ExecuteQueryAsync(
string querySql, BigqueryService bigquery, string projectId)
{
JobConfigurationQuery queryConfig = new JobConfigurationQuery { Query = querySql };
JobConfiguration config = new JobConfiguration { Query = queryConfig };
Job job = new Job { Configuration = config };
JobsResource.InsertRequest insert = bigquery.Jobs.Insert(job, projectId);
JobReference jobRef = (await insert.ExecuteAsync()).JobReference;
GetQueryResultsResponse queryResult = await
bigquery.Jobs.GetQueryResults(projectId, jobRef.JobId).ExecuteAsync();
return queryResult.Rows;
}
public override bool ShouldExecute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
logger.Trace("LicensedReports.UsersGroupsRolesPermissions={0}", programOptions.LicensedReports.UsersGroupsRolesPermissions);
loggerConsole.Trace("LicensedReports.UsersGroupsRolesPermissions={0}", programOptions.LicensedReports.UsersGroupsRolesPermissions);
if (programOptions.LicensedReports.UsersGroupsRolesPermissions == false)
{
loggerConsole.Warn("Not licensed for users, groups, roles and permissions");
return(false);
}
logger.Trace("Input.UsersGroupsRolesPermissions={0}", jobConfiguration.Input.UsersGroupsRolesPermissions);
loggerConsole.Trace("Input.UsersGroupsRolesPermissions={0}", jobConfiguration.Input.UsersGroupsRolesPermissions);
logger.Trace("Output.UsersGroupsRolesPermissions={0}", jobConfiguration.Output.UsersGroupsRolesPermissions);
loggerConsole.Trace("Output.UsersGroupsRolesPermissions={0}", jobConfiguration.Output.UsersGroupsRolesPermissions);
if (jobConfiguration.Input.UsersGroupsRolesPermissions == false || jobConfiguration.Output.UsersGroupsRolesPermissions == false)
{
loggerConsole.Trace("Skipping users, groups, roles and permissions");
}
return(jobConfiguration.Input.UsersGroupsRolesPermissions == true && jobConfiguration.Output.UsersGroupsRolesPermissions == true);
}
public async Task <Result> RunTask(string groupId, string taskId)
{
JobConfiguration jobConfiguration = TaskConfigurationService.GetTaskConfiguration(groupId, taskId);
try
{
await EMR.RunJob(EMRClient, jobConfiguration);
return(new Result()
{
ErrorCode = Constants.ErrorCodes.OK
});
}
catch (Exception exception) { }
return(new Result()
{
ErrorCode = Constants.ErrorCodes.FAILURE
});
}
public IActionResult AddDelayedJobs([FromBody] JobConfiguration jobConfiguration, CancellationToken cancellationToken)
{
try
{
_logger.LogInformation($"Initializing delayed jobs. Configuration:{JsonConvert.SerializeObject(jobConfiguration)}");
jobConfiguration?.HttpJobs.Select(job => _jobFactory.GenerateDelayedJob(job, cancellationToken)).ToArray();
jobConfiguration?.StoredProcedureJobs.Select(job => _jobFactory.GenerateDelayedJob(job, cancellationToken)).ToArray();
_logger.LogInformation($"Initializing delayed jobs successful.");
return(Ok());
}
catch (System.Exception e)
{
_logger.LogError($"Initializing delayed jobs unsuccessful. Exception:{e}");
throw e;
}
}
public override bool ShouldExecute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
logger.Trace("LicensedReports.MetricsList={0}", programOptions.LicensedReports.MetricsList);
loggerConsole.Trace("LicensedReports.MetricsList={0}", programOptions.LicensedReports.MetricsList);
if (programOptions.LicensedReports.MetricsList == false)
{
loggerConsole.Warn("Not licensed for list of metrics");
return(false);
}
logger.Trace("Input.MetricsList={0}", jobConfiguration.Input.MetricsList);
loggerConsole.Trace("Input.MetricsList={0}", jobConfiguration.Input.MetricsList);
logger.Trace("Output.MetricsList={0}", jobConfiguration.Output.MetricsList);
loggerConsole.Trace("Output.MetricsList={0}", jobConfiguration.Output.MetricsList);
if (jobConfiguration.Input.MetricsList == false && jobConfiguration.Output.MetricsList == false)
{
loggerConsole.Trace("Skipping report of list of metrics");
}
return(jobConfiguration.Input.MetricsList == true && jobConfiguration.Output.MetricsList == true);
}
public override bool ShouldExecute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
logger.Trace("LicensedReports.DetectedEntities={0}", programOptions.LicensedReports.DetectedEntities);
loggerConsole.Trace("LicensedReports.DetectedEntities={0}", programOptions.LicensedReports.DetectedEntities);
if (programOptions.LicensedReports.DetectedEntities == false)
{
loggerConsole.Warn("Not licensed for detected entities");
return(false);
}
logger.Trace("Input.DetectedEntities={0}", jobConfiguration.Input.DetectedEntities);
loggerConsole.Trace("Input.DetectedEntities={0}", jobConfiguration.Input.DetectedEntities);
logger.Trace("Output.DetectedEntities={0}", jobConfiguration.Output.DetectedEntities);
loggerConsole.Trace("Output.DetectedEntities={0}", jobConfiguration.Output.DetectedEntities);
if (jobConfiguration.Input.DetectedEntities == false || jobConfiguration.Output.DetectedEntities == false)
{
loggerConsole.Trace("Skipping report of detected entities");
}
return(jobConfiguration.Input.DetectedEntities == true && jobConfiguration.Output.DetectedEntities == true);
}
public override bool ShouldExecute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
logger.Trace("LicensedReports.Snapshots={0}", programOptions.LicensedReports.Snapshots);
loggerConsole.Trace("LicensedReports.Snapshots={0}", programOptions.LicensedReports.Snapshots);
if (programOptions.LicensedReports.Snapshots == false)
{
loggerConsole.Warn("Not licensed for snapshots");
return(false);
}
logger.Trace("Input.Snapshots={0}", jobConfiguration.Input.Snapshots);
loggerConsole.Trace("Input.Snapshots={0}", jobConfiguration.Input.Snapshots);
logger.Trace("Output.Snapshots={0}", jobConfiguration.Output.Snapshots);
loggerConsole.Trace("Output.Snapshots={0}", jobConfiguration.Output.Snapshots);
if (jobConfiguration.Input.Snapshots == false || jobConfiguration.Output.Snapshots == false)
{
loggerConsole.Trace("Skipping report of snapshot method call lines");
}
return(jobConfiguration.Input.Snapshots == true && jobConfiguration.Output.Snapshots == true);
}
public override bool ShouldExecute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
logger.Trace("LicensedReports.Flowmaps={0}", programOptions.LicensedReports.Flowmaps);
loggerConsole.Trace("LicensedReports.Flowmaps={0}", programOptions.LicensedReports.Flowmaps);
if (programOptions.LicensedReports.EntityMetrics == false)
{
loggerConsole.Warn("Not licensed for entity flowmaps");
return(false);
}
logger.Trace("Input.Flowmaps={0}", jobConfiguration.Input.Flowmaps);
loggerConsole.Trace("Input.Flowmaps={0}", jobConfiguration.Input.Flowmaps);
logger.Trace("Output.Flowmaps={0}", jobConfiguration.Output.Flowmaps);
loggerConsole.Trace("Output.Flowmaps={0}", jobConfiguration.Output.Flowmaps);
if (jobConfiguration.Input.Flowmaps == false && jobConfiguration.Output.Flowmaps == false)
{
loggerConsole.Trace("Skipping report of entity flowmaps");
}
return(jobConfiguration.Input.Flowmaps == true && jobConfiguration.Output.Flowmaps == true);
}
private int extractEvents(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi, string eventType)
{
JArray listOfEvents = new JArray();
if (File.Exists(FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType)) == false)
{
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
try
{
string eventsJSON = controllerApi.GetApplicationEvents(jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix);
if (eventsJSON != String.Empty)
{
// Load events
JArray eventsInHourArray = JArray.Parse(eventsJSON);
foreach (JObject eventObject in eventsInHourArray)
{
listOfEvents.Add(eventObject);
}
}
}
catch (Exception)
{
logger.Warn("Unable to parse JSON for Events Application={0}, EventType={1}, From={2}, To={3}", jobTarget.ApplicationID, eventType, fromTimeUnix, toTimeUnix);
}
}
if (listOfEvents.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfEvents, FilePathMap.ApplicationEventsDataFilePath(jobTarget, eventType));
logger.Info("{0} {1} events from {2:o} to {3:o}", eventType, listOfEvents.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} {1} events", eventType, listOfEvents.Count);
}
}
return(listOfEvents.Count);
}
private int extractHealthRuleViolations(JobConfiguration jobConfiguration, JobTarget jobTarget, ControllerApi controllerApi)
{
JArray listOfHealthRuleViolations = new JArray();
if (File.Exists(FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget)) == false)
{
foreach (JobTimeRange jobTimeRange in jobConfiguration.Input.HourlyTimeRanges)
{
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobTimeRange.To);
string healthRuleViolationsJSON = controllerApi.GetApplicationHealthRuleViolations(jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
if (healthRuleViolationsJSON != String.Empty)
{
try
{
// Load health rule violations
JArray healthRuleViolationsInHourArray = JArray.Parse(healthRuleViolationsJSON);
foreach (JObject healthRuleViolationObject in healthRuleViolationsInHourArray)
{
listOfHealthRuleViolations.Add(healthRuleViolationObject);
}
}
catch (Exception)
{
logger.Warn("Unable to parse JSON for HR Violations Application={0}, From={1}, To={2}", jobTarget.ApplicationID, fromTimeUnix, toTimeUnix);
}
}
}
if (listOfHealthRuleViolations.Count > 0)
{
FileIOHelper.WriteJArrayToFile(listOfHealthRuleViolations, FilePathMap.ApplicationHealthRuleViolationsDataFilePath(jobTarget));
logger.Info("{0} health rule violations from {1:o} to {2:o}", listOfHealthRuleViolations.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
loggerConsole.Info("{0} health rule violations", listOfHealthRuleViolations.Count);
}
}
return(listOfHealthRuleViolations.Count);
}
internal static IEnumerable <ResolvedRssItem> ResolveRssItems(JobConfiguration jobConfig, IEnumerable <RssItem> rssItems)
{
var resolvedMatches = new List <ResolvedRssItem>();
var jobRules = jobConfig.Rules.OrderBy(x => x.Priority).ToList();
foreach (var item in rssItems)
{
var resolvedMatch = new ResolvedRssItem()
{
Title = item.Title,
Description = item.Description,
Link = item.Link
};
foreach (var rule in jobRules)
{
try
{
Regex regex = new Regex(rule.Regex);
var match = regex.Match(item.Title);
if (match.Success)
{
resolvedMatch.DownloadPath = CreateDownloadPath(jobConfig, rule, match);
}
}
catch (ArgumentException)
{
}
}
if (String.IsNullOrEmpty(resolvedMatch.DownloadPath))
{
resolvedMatch.DownloadPath = jobConfig.DownloadPath;
}
if (resolvedMatch.DownloadPath.Last() != '/')
{
resolvedMatch.DownloadPath += "/";
}
resolvedMatches.Add(resolvedMatch);
}
return(resolvedMatches);
}
/// <summary>
/// Creates a <see cref="Job"/> with a client-generated <see cref="JobReference"/> based on
/// <paramref name="options"/> and containing the given job configuration.
/// </summary>
/// <remarks>This method is internal for test purposes.</remarks>
/// <param name="configuration">Configuration for the job. Must not be null.</param>
/// <param name="options">Options for job creation. May be null.</param>
internal Job CreateJob(JobConfiguration configuration, JobCreationOptions options)
{
GaxPreconditions.CheckNotNull(configuration, nameof(configuration));
string projectId = options?.ProjectId ?? ProjectId;
string jobId = options?.JobId;
string jobIdPrefix = options?.JobIdPrefix;
if (jobId != null && jobIdPrefix != null)
{
throw new ArgumentException("Only one of JobId or JobIdPrefix can be specified for a single operation.");
}
if (jobId == null)
{
var prefix = jobIdPrefix ?? DefaultJobIdPrefix;
jobId = prefix + Guid.NewGuid().ToString().Replace("-", "_");
}
return(new Job {
Configuration = configuration, JobReference = GetJobReference(projectId, jobId)
});
}
public bool StopJob(Job job, JobConfiguration config)
{
try
{
_mainManager.SetStatusBarInfoText($"Wait while stopping Job");
if (jobTasks.Any(x => x.Key.Id == job.Id))
{
duplicates.Cancel();
var key = jobTasks.First(x => x.Key.Id == job.Id).Key;
jobTasks[key]?.Cancel();
jobTasks.Remove(key);
}
_mainManager.SetStatusBarInfoText(null);
return(true);
}
catch (Exception ex)
{
_logger.Error(ex);
}
return(false);
}
public static Job Create(OutputStream stream, Watermarks watermarks, string jobName)
{
Logger.LogInformationMessage("Create certificate logging job.", new object[0]);
if (stream == null)
{
throw new ArgumentNullException("stream");
}
if (watermarks == null)
{
throw new ArgumentNullException("watermarks");
}
JobConfiguration jobConfiguration = new JobConfiguration(jobName);
Watermark watermark = watermarks.Get(jobName);
string text = Path.Combine(jobConfiguration.DiagnosticsRootDirectory, "CertificateLogs");
LogDirectorySource logDirectorySource = new LogDirectorySource(jobName, new CertificateLogSchema(), null, text, "*.log", new Comparison <LogFileInfo>(LogFileInfo.LastWriteTimeComparer), new DateTime?(watermark.Timestamp), null, 6, null);
SingleStreamJob singleStreamJob = new SingleStreamJob(jobName, logDirectorySource, "CertificateLog", stream, Watermark.LatestWatermark, watermarks.Directory);
singleStreamJob.Extension = new CertificateLogExtension(singleStreamJob);
Logger.LogInformationMessage("Created certificate logging job", new object[0]);
return(singleStreamJob);
}
public async Task <bool> AddConfigDataAsync(CheckConfig config)
{
try
{
JobConfiguration jobConfiguration = EntityAutoMapper.Instance.GetMapperResult <JobConfiguration>(_mapperCfg, config);
if (null == jobConfiguration)
{
return(false);
}
using (HealthManjuuCoreContext context = new HealthManjuuCoreContext())
{
return(await AddConfigDataAsync(context, jobConfiguration));
}
}
catch (System.Exception ex)
{
NLogMgr.ErrorExLog(_errorLog, "添加检测工具配置异常", ex);
return(false);
}
}
private void extractListOfMetrics(
JobTarget jobTarget,
JobConfiguration jobConfiguration,
string metricPathPrefix,
string fileNamePrefix,
int maxDepth)
{
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
DateTime startTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From;
DateTime endTime = jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To;
StringBuilder sbMetricPath = new StringBuilder(128);
sbMetricPath.Append(metricPathPrefix);
for (int currentDepth = 0; currentDepth <= maxDepth; currentDepth++)
{
sbMetricPath.Append("|*");
string metricPath = sbMetricPath.ToString();
loggerConsole.Trace("Depth {0:00}/{1:00}, {2}", currentDepth, maxDepth, metricPath);
logger.Info("Retrieving metric lists for Application {0}({1}), Metric='{2}', From {3:o}, To {4:o}", jobTarget.Application, jobTarget.ApplicationID, metricPath, startTime, endTime);
string metricsJson = String.Empty;
string metricsDataFilePath = FilePathMap.MetricsListDataFilePath(jobTarget, fileNamePrefix, currentDepth);
if (File.Exists(metricsDataFilePath) == false)
{
metricsJson = controllerApi.GetMetricData(
jobTarget.ApplicationID,
metricPath,
UnixTimeHelper.ConvertToUnixTimestamp(startTime),
UnixTimeHelper.ConvertToUnixTimestamp(endTime),
true);
FileIOHelper.SaveFileToPath(metricsJson, metricsDataFilePath);
}
}
}
}
private async Task OnAfterCollect(Job job, JobConfiguration config)
{
_mainManager.SetStatusBarInfoText($"Job find duplicates");
int maxPara = config.MaxParallelism;
if (maxPara < 1)
{
maxPara = Environment.ProcessorCount;
}
duplicates.ProcessFileProgressWithItems += (object sender, Compare.Duplicates.ProcessFileProgressItem e) =>
{
_mainManager.SetStatusBarInfoText($"Job compare files ({e.Progress}%)");
Task.Run(async() =>
{
await OnSaveDuplicateResultProgress(job, e);
});
};
var cache = await OnGetDuplicateResultProgress(job);
if (cache != null)
{
var cacheIndex = await _jobServiceRepository.GetDuplicateResultIndexCache(job.Id);
if (cacheIndex != null && cacheIndex.Count > 0)
{
foreach (var item in cacheIndex)
{
cache.IndexCollection.Add(item.Value.ToUpper());
}
}
}
duplicates.SetCache(cache);
var result = await duplicates.Find(maxPara);
_mainManager.SetStatusBarInfoText($"Finish job");
await OnComplete(result, job, config);
_mainManager.SetStatusBarInfoText(null);
}
public DeploymentResult Execute(JobConfiguration jobConfiguration)
{
DeploymentResult result;
foreach (var stepConfiguration in jobConfiguration.Steps.Where(stepConfiguration => stepConfiguration.SubSystem == SsisSubSystem.IntegrationServices))
{
try
{
_ioWrapper.CreateDirectoryIfNotExists(stepConfiguration.DtsxFileDestination);
_ioWrapper.CopyFile(stepConfiguration.DtsxFile, stepConfiguration.DtsxFileDestination);
_ioWrapper.CopyFile(stepConfiguration.DtsxConfigurationFile, stepConfiguration.DtsxFileDestination);
}
catch (Exception ex)
{
result = new DeploymentResult(ex, false, GetType());
return(result);
}
}
result = new DeploymentResult(null, true, GetType());
return(result);
}
private static JobConfiguration <TJobData> LoadConfigurationForJob <TJobData>(JobTypeEnum jobType)
where TJobData : IJobDataConfiguration <TJobData>
{
var jobConfiguration = new JobConfiguration <TJobData>();
var configuration = _configuration.GetSection($"{jobType}");
configuration.Bind(jobConfiguration);
if (jobConfiguration.JobInfo == null)
{
throw new Exception($"JobInfo binding failed for type '{jobType}'.");
}
if (jobConfiguration.Scheduling == null)
{
throw new Exception($"Scheduling binding failed for type '{jobType}'.");
}
if (jobConfiguration.JobData == null)
{
throw new Exception($"JobData binding failed for type '{jobType}'.");
}
return(jobConfiguration);
}
internal List <MetricExtractMapping> getMetricsExtractMappingList(JobConfiguration jobConfiguration)
{
List <MetricExtractMapping> entityMetricExtractMappingList = new List <MetricExtractMapping>();
if (jobConfiguration.Output.BSG)
{
entityMetricExtractMappingList = FileIOHelper.ReadListFromCSVFile(FilePathMap.EntityMetricExtractMappingFilePathBSG(), new MetricExtractMappingReportMap());
loggerConsole.Info("Loaded BSG Metric Set");
logger.Info("Loaded BSG Metric Set");
}
else
{
entityMetricExtractMappingList = FileIOHelper.ReadListFromCSVFile(FilePathMap.EntityMetricExtractMappingFilePath(), new MetricExtractMappingReportMap());
loggerConsole.Info("Loaded Default Metric Set");
logger.Info("Loaded Default Metric Set");
}
List <MetricExtractMapping> entityMetricExtractMappingListFiltered = new List <MetricExtractMapping>(entityMetricExtractMappingList.Count);
foreach (string metricSet in jobConfiguration.Input.MetricsSelectionCriteria.MetricSets)
{
List <MetricExtractMapping> entityMetricExtractMappingListForMetricSet = entityMetricExtractMappingList.Where(m => m.MetricSet == metricSet).ToList();
if (entityMetricExtractMappingListForMetricSet != null)
{
logger.Info("Input job specified {0} metric set, resulted in {1} metrics from mapping file", metricSet, entityMetricExtractMappingListForMetricSet.Count);
loggerConsole.Info("Input job specified {0} metric set, resulted in {1} metrics from mapping file", metricSet, entityMetricExtractMappingListForMetricSet.Count);
entityMetricExtractMappingListFiltered.AddRange(entityMetricExtractMappingListForMetricSet);
foreach (MetricExtractMapping mem in entityMetricExtractMappingListForMetricSet)
{
logger.Trace("{0}, path={1}", mem, mem.MetricPath);
}
}
}
logger.Info("Selected {0} metrics from mapping file", entityMetricExtractMappingListFiltered.Count);
loggerConsole.Info("Selected {0} metrics from mapping file", entityMetricExtractMappingListFiltered.Count);
return(entityMetricExtractMappingListFiltered);
}
public CreateNotification(int order, JobConfiguration configuration)
{
Order = order;
Name = "Create Notification";
Contents = @"
USE [msdb]
GO
IF NOT EXISTS (SELECT name FROM msdb.dbo.sysoperators WHERE name = N'@@OperatorName@@')
EXEC msdb.dbo.sp_add_operator
@name=N'@@OperatorName@@',
@enabled=1,
@weekday_pager_start_time=0,
@weekday_pager_end_time=235959,
@saturday_pager_start_time=0,
@saturday_pager_end_time=235959,
@sunday_pager_start_time=0,
@sunday_pager_end_time=235959,
@pager_days=127,
@email_address=N'@@OperatorEmail@@',
@pager_address=N'@@OperatorEmail@@'
GO
EXEC msdb.dbo.sp_update_job
@job_name=N'@@JobName@@',
@notify_level_email=2,
@notify_level_netsend=2,
@notify_level_page=3,
@notify_email_operator_name=N'@@OperatorName@@',
@notify_page_operator_name=N'@@OperatorName@@'
GO
"
.Replace("@@JobName@@", configuration.JobName)
.Replace("@@OperatorName@@", configuration.Notification.Operator)
.Replace("@@OperatorEmail@@", configuration.Notification.OperatorEmail);
}
public void StartMultiThreaded()
{
Type jobType = typeof(SleepJob);
JobLockProvider jobLockProvider = new StaticLockProvider();
var jobConfiguration = new JobConfiguration
{
Name = "SleepJob",
Description = "Test Sleep Job with xml history",
Interval = TimeSpan.FromSeconds(5),
Type = typeof(SleepJob).AssemblyQualifiedName,
JobLockProvider = typeof(StaticLockProvider).AssemblyQualifiedName,
};
jobConfiguration.Arguments.Add("sleep", 10);
var log = new StringBuilder();
var j = new JobRunner(jobConfiguration, jobType, jobLockProvider, null);
JobManager.Current.JobStarting += (t, e) => log.AppendLine(e.ToString() + ":" + Thread.CurrentThread.ManagedThreadId);
Thread t1 = new Thread(j.Start);
Thread t2 = new Thread(j.Start);
Thread t3 = new Thread(j.Start);
Thread t4 = new Thread(j.Start);
t1.Start();
t2.Start();
Thread.Sleep(TimeSpan.FromSeconds(5));
t3.Start();
t4.Start();
// wait for job
Thread.Sleep(TimeSpan.FromSeconds(45));
string l = log.ToString();
Assert.IsNotNull(l);
}
private static JobConfiguration CreateJobConfiguration()
{
var jobConfiguration =
JobConfiguration.Create()
.WithName("Test Name")
.WithDescription("Test Description")
.WithSsisOwner("Test SSIS Owner")
.WithSsisServer("Test SSIS Server")
.WithStep(StepConfiguration.Create()
.WithName("Test Step Name")
.WithId(1)
.WithSubSystem(SsisSubSystem.TransactStructuredQueryLanguage)
.ExecuteCommand("Test Command")
.OnFailure(JobAction.QuitWithFailure)
.OnSuccess(JobAction.QuitWithSuccess))
.WithSchedule(ScheduleConfiguration.Create()
.WithName("Test Schedule Configuration")
.RunOn(FrequencyDay.Monday)
.ActivatedFrom(DateTime.Now)
.ActivatedUntil(DateTime.Now.AddDays(1))
.StartingAt(TimeSpan.FromSeconds(6)));
return jobConfiguration;
}
public void InitializeCheckJob_Should_CreatePingCheckJobWithTrigger()
{
// Arrange
var config = new JobConfiguration
{
Type = "Ping"
};
var xml = new XElement("Trigger");
xml.SetAttributeValue("Type", "simple");
xml.SetAttributeValue("Repeat", "10");
config.Triggers.Add(xml);
var job = new HealthCheckJob
{
JobConfiguration = config
};
var mock = new Mock <IComponentFactory>();
_ = mock.Setup(f => f.GetPlugin("Ping")).Returns(new Mock <IHealthCheckPlugin>().Object);
_ = mock.Setup(f => f.GetTrigger(It.IsAny <XElement>())).Returns(new Mock <ITrigger>().Object);
var manager = new PluginManager(mock.Object);
var group = new HealthCheckGroup
{
Name = "PROD"
};
// Act
_ = manager.InitializeCheckJob(job, group);
// Assert
_ = Assert.Single(job.Triggers);
}
public void StartWithTimeOfDayLaterThenStart()
{
Type jobType = typeof(SleepJob);
JobLockProvider jobLockProvider = new StaticLockProvider();
JobHistoryProvider jobHistoryProvider = new StaticHistoryProvider
{
LastResult = string.Empty,
LastRunTime = DateTime.Now,
LastStatus = JobStatus.None
};
// run time 1 hour ago
DateTime runTime = DateTime.Now.Subtract(TimeSpan.FromHours(1));
var jobConfiguration = new JobConfiguration
{
Name = "SleepJob",
Description = "Test Sleep Job with xml history",
Interval = runTime.TimeOfDay,
IsTimeOfDay = true,
Type = typeof(SleepJob).AssemblyQualifiedName,
JobLockProvider = typeof(StaticLockProvider).AssemblyQualifiedName,
JobHistoryProvider = typeof(StaticHistoryProvider).AssemblyQualifiedName
};
var j = new JobRunner(jobConfiguration, jobType, jobLockProvider, jobHistoryProvider);
Assert.IsNotNull(j);
j.Start();
DateTime nextRun = runTime.AddDays(1);
Assert.AreEqual(nextRun.Date, j.NextRunTime.Date);
Assert.AreEqual(nextRun.Hour, j.NextRunTime.Hour);
Assert.AreEqual(nextRun.Minute, j.NextRunTime.Minute);
}
/// <summary>
/// Entry point for the execution of <see cref="FileTransformController"/>.
/// </summary>
public override void Run()
{
ValidateInputParameters();
JobConfiguration jobConf = new JobConfiguration
{
MapperType = typeof(FileTransformMapper),
CombinerType = typeof(FileTransformReducer),
ReducerType = typeof(FileTransformReducer),
MapOutputStorage = MapOutputStoreType.Local,
KeyPartitioner = typeof(HashModuloKeyPartitioner <int, string>),
ExceptionPolicy = new TerminateOnFirstException(),
JobTimeout = TimeSpan.FromMinutes(JobTimeoutInMinutes)
};
Job job = new Job(jobConf, this);
job.DataPartitioner = new BlobTextPartitioner(this.CloudClient, this.
InputDataLocation, this.RequestedNumberOfMappers);
job.RecordWriter = new FileTransformWriter(this.CloudClient, this.OutputDataLocation);
job.NoOfReduceTasks = this.RequestedNumberOfReducers;
job.Run();
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_WEB) == 0)
{
return(true);
}
try
{
loggerConsole.Info("Prepare Detected WEB Entities Report File");
#region Prepare the report package
// Prepare package
ExcelPackage excelReport = new ExcelPackage();
excelReport.Workbook.Properties.Author = String.Format("AppDynamics DEXTER {0}", Assembly.GetEntryAssembly().GetName().Version);
excelReport.Workbook.Properties.Title = "AppDynamics DEXTER Detected WEB Entities Report";
excelReport.Workbook.Properties.Subject = programOptions.JobName;
excelReport.Workbook.Properties.Comments = String.Format("Targets={0}\nFrom={1:o}\nTo={2:o}", jobConfiguration.Target.Count, jobConfiguration.Input.TimeRange.From, jobConfiguration.Input.TimeRange.To);
#endregion
#region Parameters sheet
// Parameters sheet
ExcelWorksheet sheet = excelReport.Workbook.Worksheets.Add(SHEET_PARAMETERS);
var hyperLinkStyle = sheet.Workbook.Styles.CreateNamedStyle("HyperLinkStyle");
hyperLinkStyle.Style.Font.UnderLineType = ExcelUnderLineType.Single;
hyperLinkStyle.Style.Font.Color.SetColor(colorBlueForHyperlinks);
fillReportParametersSheet(sheet, jobConfiguration, "AppDynamics DEXTER Detected WEB Entities Report");
#endregion
#region TOC sheet
// Navigation sheet with link to other sheets
sheet = excelReport.Workbook.Worksheets.Add(SHEET_TOC);
#endregion
#region Entity sheets and their associated pivots
// Entity sheets
sheet = excelReport.Workbook.Worksheets.Add(SHEET_CONTROLLERS_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_APPLICATIONS_ALL_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_APPLICATIONS_WEB_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_WEB_PAGES_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Pivot";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_WEB_PAGES_TYPE_PIVOT);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_WEB_PAGES_TYPE_PIVOT);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Table";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_WEB_PAGES_LIST);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT + 3, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_PAGE_RESOURCES_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Pivot";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_PAGE_RESOURCES_TYPE_PIVOT);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_PAGE_RESOURCES_TYPE_PIVOT);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Table";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_PAGE_RESOURCES_LIST);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT + 2, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_PAGE_BUSINESS_TRANSACTIONS_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Pivot";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_PAGE_BUSINESS_TRANSACTIONS_TYPE_PIVOT);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_PAGE_BUSINESS_TRANSACTIONS_TYPE_PIVOT);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Table";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_PAGE_BUSINESS_TRANSACTIONS_LIST);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT + 2, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_GEO_LOCATIONS_LIST);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Pivot";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_GEO_LOCATIONS_TYPE_PIVOT);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(LIST_SHEET_START_TABLE_AT + 1, 1);
sheet = excelReport.Workbook.Worksheets.Add(SHEET_GEO_LOCATIONS_TYPE_PIVOT);
sheet.Cells[1, 1].Value = "Table of Contents";
sheet.Cells[1, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_TOC);
sheet.Cells[1, 2].StyleName = "HyperLinkStyle";
sheet.Cells[2, 1].Value = "See Table";
sheet.Cells[2, 2].Formula = String.Format(@"=HYPERLINK(""#'{0}'!A1"", ""<Go>"")", SHEET_GEO_LOCATIONS_LIST);
sheet.Cells[2, 2].StyleName = "HyperLinkStyle";
sheet.View.FreezePanes(PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT + 2, 1);
#endregion
loggerConsole.Info("Fill Detected WEB Entities Report File");
#region Report file variables
ExcelRangeBase range = null;
ExcelTable table = null;
#endregion
#region Controllers
loggerConsole.Info("List of Controllers");
sheet = excelReport.Workbook.Worksheets[SHEET_CONTROLLERS_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.ControllerSummaryReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Applications - All
loggerConsole.Info("List of Applications - All");
sheet = excelReport.Workbook.Worksheets[SHEET_APPLICATIONS_ALL_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.ControllerApplicationsReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Applications
loggerConsole.Info("List of Applications");
sheet = excelReport.Workbook.Worksheets[SHEET_APPLICATIONS_WEB_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.WEBApplicationsReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Web Pages
loggerConsole.Info("List of Web Pages");
sheet = excelReport.Workbook.Worksheets[SHEET_WEB_PAGES_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.WEBPagesReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Web Page Resources
loggerConsole.Info("List of Web Page Resources");
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_RESOURCES_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.WEBPageResourcesReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Web Page Business Transactions
loggerConsole.Info("List of Web Page Business Transactions");
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_BUSINESS_TRANSACTIONS_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.WEBPageBusinessTransactionsReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
#region Geo Locations
loggerConsole.Info("List of Geo Locations");
sheet = excelReport.Workbook.Worksheets[SHEET_GEO_LOCATIONS_LIST];
EPPlusCSVHelper.ReadCSVFileIntoExcelRange(FilePathMap.WEBGeoLocationsReportFilePath(), 0, sheet, LIST_SHEET_START_TABLE_AT, 1);
#endregion
loggerConsole.Info("Finalize Detected WEB Entities Report File");
#region Controllers sheet
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_CONTROLLERS_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_CONTROLLERS);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 25;
sheet.Column(table.Columns["Version"].Position + 1).Width = 15;
}
#endregion
#region Applications - All
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_APPLICATIONS_ALL_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_APPLICATIONS_ALL);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
sheet.Column(table.Columns["Description"].Position + 1).Width = 15;
sheet.Column(table.Columns["CreatedBy"].Position + 1).Width = 15;
sheet.Column(table.Columns["UpdatedBy"].Position + 1).Width = 15;
sheet.Column(table.Columns["CreatedOn"].Position + 1).Width = 20;
sheet.Column(table.Columns["UpdatedOn"].Position + 1).Width = 20;
sheet.Column(table.Columns["CreatedOnUtc"].Position + 1).Width = 20;
sheet.Column(table.Columns["UpdatedOnUtc"].Position + 1).Width = 20;
}
#endregion
#region Applications
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_APPLICATIONS_WEB_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_APPLICATIONS_WEB);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
ExcelAddress cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumPages"].Position + 1, sheet.Dimension.Rows, table.Columns["NumPages"].Position + 1);
var cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumAJAXRequests"].Position + 1, sheet.Dimension.Rows, table.Columns["NumAJAXRequests"].Position + 1);
cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumVirtualPages"].Position + 1, sheet.Dimension.Rows, table.Columns["NumVirtualPages"].Position + 1);
cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumIFrames"].Position + 1, sheet.Dimension.Rows, table.Columns["NumIFrames"].Position + 1);
cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumActivity"].Position + 1, sheet.Dimension.Rows, table.Columns["NumActivity"].Position + 1);
cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
cfAddressNum = new ExcelAddress(LIST_SHEET_START_TABLE_AT + 1, table.Columns["NumNoActivity"].Position + 1, sheet.Dimension.Rows, table.Columns["NumNoActivity"].Position + 1);
cfNum = sheet.ConditionalFormatting.AddDatabar(cfAddressNum, colorLightBlueForDatabars);
}
#endregion
#region Web Pages
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_WEB_PAGES_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_WEB_PAGES);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
sheet.Column(table.Columns["PageType"].Position + 1).Width = 10;
sheet.Column(table.Columns["PageName"].Position + 1).Width = 20;
sheet.Column(table.Columns["FirstSegment"].Position + 1).Width = 20;
sheet.Column(table.Columns["From"].Position + 1).Width = 20;
sheet.Column(table.Columns["To"].Position + 1).Width = 20;
sheet.Column(table.Columns["FromUtc"].Position + 1).Width = 20;
sheet.Column(table.Columns["ToUtc"].Position + 1).Width = 20;
// Make pivot
sheet = excelReport.Workbook.Worksheets[SHEET_WEB_PAGES_TYPE_PIVOT];
ExcelPivotTable pivot = sheet.PivotTables.Add(sheet.Cells[PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT + 1, 1], range, PIVOT_WEB_PAGES_TYPE);
setDefaultPivotTableSettings(pivot);
addFilterFieldToPivot(pivot, "HasActivity");
addFilterFieldToPivot(pivot, "ARTRange", eSortType.Ascending);
addFilterFieldToPivot(pivot, "NumNameSegments", eSortType.Ascending);
addRowFieldToPivot(pivot, "Controller");
addRowFieldToPivot(pivot, "ApplicationName");
addRowFieldToPivot(pivot, "PageType");
addRowFieldToPivot(pivot, "FirstSegment");
addRowFieldToPivot(pivot, "PageName");
addDataFieldToPivot(pivot, "PageID", DataFieldFunctions.Count, "NumPages");
addDataFieldToPivot(pivot, "ART", DataFieldFunctions.Average, "ART");
addDataFieldToPivot(pivot, "TimeTotal", DataFieldFunctions.Sum, "Time");
addDataFieldToPivot(pivot, "Calls", DataFieldFunctions.Sum, "Calls");
addDataFieldToPivot(pivot, "CPM", DataFieldFunctions.Average, "CPM");
ExcelChart chart = sheet.Drawings.AddChart(GRAPH_WEB_PAGES_TYPE, eChartType.ColumnClustered, pivot);
chart.SetPosition(2, 0, 0, 0);
chart.SetSize(800, 300);
sheet.Column(1).Width = 20;
sheet.Column(2).Width = 20;
sheet.Column(3).Width = 20;
sheet.Column(4).Width = 20;
sheet.Column(5).Width = 20;
}
#endregion
#region Web Page Resources
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_RESOURCES_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_PAGE_RESOURCES);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
sheet.Column(table.Columns["PageType"].Position + 1).Width = 10;
sheet.Column(table.Columns["PageName"].Position + 1).Width = 20;
sheet.Column(table.Columns["ChildPageType"].Position + 1).Width = 10;
sheet.Column(table.Columns["ChildPageName"].Position + 1).Width = 20;
sheet.Column(table.Columns["From"].Position + 1).Width = 20;
sheet.Column(table.Columns["To"].Position + 1).Width = 20;
sheet.Column(table.Columns["FromUtc"].Position + 1).Width = 20;
sheet.Column(table.Columns["ToUtc"].Position + 1).Width = 20;
// Make pivot
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_RESOURCES_TYPE_PIVOT];
ExcelPivotTable pivot = sheet.PivotTables.Add(sheet.Cells[PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT, 1], range, PIVOT_PAGE_RESOURCES_TYPE);
setDefaultPivotTableSettings(pivot);
addFilterFieldToPivot(pivot, "HasActivity");
addFilterFieldToPivot(pivot, "ARTRange", eSortType.Ascending);
addRowFieldToPivot(pivot, "Controller");
addRowFieldToPivot(pivot, "ApplicationName");
addRowFieldToPivot(pivot, "PageName");
addRowFieldToPivot(pivot, "ChildPageType");
addRowFieldToPivot(pivot, "ChildPageName");
addDataFieldToPivot(pivot, "ChildPageID", DataFieldFunctions.Count, "NumPages");
addDataFieldToPivot(pivot, "ART", DataFieldFunctions.Average, "ART");
addDataFieldToPivot(pivot, "Calls", DataFieldFunctions.Sum, "Calls");
addDataFieldToPivot(pivot, "CPM", DataFieldFunctions.Average, "CPM");
ExcelChart chart = sheet.Drawings.AddChart(GRAPH_PAGE_RESOURCES_TYPE, eChartType.ColumnClustered, pivot);
chart.SetPosition(2, 0, 0, 0);
chart.SetSize(800, 300);
sheet.Column(1).Width = 20;
sheet.Column(2).Width = 20;
sheet.Column(3).Width = 20;
sheet.Column(4).Width = 20;
sheet.Column(5).Width = 20;
}
#endregion
#region Web Page Business Transactions
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_BUSINESS_TRANSACTIONS_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_PAGE_BUSINESS_TRANSACTIONS);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
sheet.Column(table.Columns["PageType"].Position + 1).Width = 10;
sheet.Column(table.Columns["PageName"].Position + 1).Width = 20;
sheet.Column(table.Columns["TierName"].Position + 1).Width = 20;
sheet.Column(table.Columns["BTName"].Position + 1).Width = 20;
sheet.Column(table.Columns["BTType"].Position + 1).Width = 15;
sheet.Column(table.Columns["From"].Position + 1).Width = 20;
sheet.Column(table.Columns["To"].Position + 1).Width = 20;
sheet.Column(table.Columns["FromUtc"].Position + 1).Width = 20;
sheet.Column(table.Columns["ToUtc"].Position + 1).Width = 20;
// Make pivot
sheet = excelReport.Workbook.Worksheets[SHEET_PAGE_BUSINESS_TRANSACTIONS_TYPE_PIVOT];
ExcelPivotTable pivot = sheet.PivotTables.Add(sheet.Cells[PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT, 1], range, PIVOT_PAGE_RESOURCES_TYPE);
setDefaultPivotTableSettings(pivot);
addFilterFieldToPivot(pivot, "HasActivity");
addFilterFieldToPivot(pivot, "ARTRange", eSortType.Ascending);
addRowFieldToPivot(pivot, "Controller");
addRowFieldToPivot(pivot, "ApplicationName");
addRowFieldToPivot(pivot, "PageName");
addRowFieldToPivot(pivot, "TierName");
addRowFieldToPivot(pivot, "BTName");
addDataFieldToPivot(pivot, "BTID", DataFieldFunctions.Count, "NumBTs");
addDataFieldToPivot(pivot, "ART", DataFieldFunctions.Average, "ART");
addDataFieldToPivot(pivot, "Calls", DataFieldFunctions.Sum, "Calls");
addDataFieldToPivot(pivot, "CPM", DataFieldFunctions.Average, "CPM");
ExcelChart chart = sheet.Drawings.AddChart(GRAPH_PAGE_BUSINESS_TRANSACTIONS_TYPE, eChartType.ColumnClustered, pivot);
chart.SetPosition(2, 0, 0, 0);
chart.SetSize(800, 300);
sheet.Column(1).Width = 20;
sheet.Column(2).Width = 20;
sheet.Column(3).Width = 20;
sheet.Column(4).Width = 20;
sheet.Column(5).Width = 20;
}
#endregion
#region Geo Locations
// Make table
sheet = excelReport.Workbook.Worksheets[SHEET_GEO_LOCATIONS_LIST];
logger.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
loggerConsole.Info("{0} Sheet ({1} rows)", sheet.Name, sheet.Dimension.Rows);
if (sheet.Dimension.Rows > LIST_SHEET_START_TABLE_AT)
{
range = sheet.Cells[LIST_SHEET_START_TABLE_AT, 1, sheet.Dimension.Rows, sheet.Dimension.Columns];
table = sheet.Tables.Add(range, TABLE_GEO_LOCATIONS);
table.ShowHeader = true;
table.TableStyle = TableStyles.Medium2;
table.ShowFilter = true;
table.ShowTotal = false;
sheet.Column(table.Columns["Controller"].Position + 1).Width = 15;
sheet.Column(table.Columns["ApplicationName"].Position + 1).Width = 20;
sheet.Column(table.Columns["LocationName"].Position + 1).Width = 15;
sheet.Column(table.Columns["Country"].Position + 1).Width = 15;
sheet.Column(table.Columns["Region"].Position + 1).Width = 15;
sheet.Column(table.Columns["GeoCode"].Position + 1).Width = 15;
sheet.Column(table.Columns["From"].Position + 1).Width = 20;
sheet.Column(table.Columns["To"].Position + 1).Width = 20;
sheet.Column(table.Columns["FromUtc"].Position + 1).Width = 20;
sheet.Column(table.Columns["ToUtc"].Position + 1).Width = 20;
// Make pivot
sheet = excelReport.Workbook.Worksheets[SHEET_GEO_LOCATIONS_TYPE_PIVOT];
ExcelPivotTable pivot = sheet.PivotTables.Add(sheet.Cells[PIVOT_SHEET_START_PIVOT_AT + PIVOT_SHEET_CHART_HEIGHT, 1], range, PIVOT_PAGE_RESOURCES_TYPE);
setDefaultPivotTableSettings(pivot);
addFilterFieldToPivot(pivot, "HasActivity");
addFilterFieldToPivot(pivot, "ARTRange", eSortType.Ascending);
addRowFieldToPivot(pivot, "Controller");
addRowFieldToPivot(pivot, "ApplicationName");
addRowFieldToPivot(pivot, "LocationType");
addRowFieldToPivot(pivot, "Country");
addRowFieldToPivot(pivot, "Region");
addRowFieldToPivot(pivot, "LocationName");
addDataFieldToPivot(pivot, "ART", DataFieldFunctions.Average, "ART");
addDataFieldToPivot(pivot, "Calls", DataFieldFunctions.Sum, "Calls");
addDataFieldToPivot(pivot, "CPM", DataFieldFunctions.Average, "CPM");
ExcelChart chart = sheet.Drawings.AddChart(GRAPH_GEO_LOCATIONS_TYPE, eChartType.ColumnClustered, pivot);
chart.SetPosition(2, 0, 0, 0);
chart.SetSize(800, 300);
sheet.Column(1).Width = 20;
sheet.Column(2).Width = 20;
sheet.Column(3).Width = 20;
sheet.Column(4).Width = 20;
sheet.Column(5).Width = 20;
sheet.Column(6).Width = 20;
}
#endregion
#region TOC sheet
// TOC sheet again
sheet = excelReport.Workbook.Worksheets[SHEET_TOC];
fillTableOfContentsSheet(sheet, excelReport);
#endregion
#region Save file
if (Directory.Exists(FilePathMap.ReportFolderPath()) == false)
{
Directory.CreateDirectory(FilePathMap.ReportFolderPath());
}
string reportFilePath = FilePathMap.WEBEntitiesExcelReportFilePath(jobConfiguration.Input.TimeRange);
logger.Info("Saving Excel report {0}", reportFilePath);
loggerConsole.Info("Saving Excel report {0}", reportFilePath);
try
{
// Save full report Excel files
excelReport.SaveAs(new FileInfo(reportFilePath));
}
catch (InvalidOperationException ex)
{
logger.Warn("Unable to save Excel file {0}", reportFilePath);
logger.Warn(ex);
loggerConsole.Warn("Unable to save Excel file {0}", reportFilePath);
}
#endregion
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
internal SingularBuilder(JobConfiguration config)
: base(config)
{
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(jobConfiguration) == false)
{
return(true);
}
bool haveProcessedAtLeastOneDBCollector = false;
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
if (jobTarget.Type == APPLICATION_TYPE_DB)
{
if (haveProcessedAtLeastOneDBCollector == false)
{
haveProcessedAtLeastOneDBCollector = true;
}
else
{
continue;
}
}
int numEventsTotal = 0;
#region Health Rule violations
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
loggerConsole.Info("Extract List of Health Rule Violations ({0} time ranges)", jobConfiguration.Input.HourlyTimeRanges.Count);
numEventsTotal = numEventsTotal + extractHealthRuleViolations(jobConfiguration, jobTarget, controllerApi);
}
#endregion
#region Events
loggerConsole.Info("Extract {0} event types ({1} time ranges)", EVENT_TYPES.Count, jobConfiguration.Input.HourlyTimeRanges.Count);
Parallel.ForEach(
EVENT_TYPES,
new ParallelOptions {
MaxDegreeOfParallelism = EVENTS_EXTRACT_NUMBER_OF_THREADS
},
() => 0,
(eventType, loop, subtotal) =>
{
using (ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
int numEventsInType = extractEvents(jobConfiguration, jobTarget, controllerApiParallel, eventType);
subtotal = subtotal + numEventsInType;
return(subtotal);
}
},
(finalResult) =>
{
Interlocked.Add(ref numEventsTotal, finalResult);
}
);
loggerConsole.Info("{0} events total", numEventsTotal);
#endregion
stepTimingTarget.NumEntities = numEventsTotal;
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public override bool Execute(ProgramOptions programOptions, JobConfiguration jobConfiguration)
{
Stopwatch stopWatch = new Stopwatch();
stopWatch.Start();
StepTiming stepTimingFunction = new StepTiming();
stepTimingFunction.JobFileName = programOptions.OutputJobFilePath;
stepTimingFunction.StepName = jobConfiguration.Status.ToString();
stepTimingFunction.StepID = (int)jobConfiguration.Status;
stepTimingFunction.StartTime = DateTime.Now;
stepTimingFunction.NumEntities = jobConfiguration.Target.Count;
this.DisplayJobStepStartingStatus(jobConfiguration);
FilePathMap = new FilePathMap(programOptions, jobConfiguration);
try
{
if (this.ShouldExecute(programOptions, jobConfiguration) == false)
{
return(true);
}
if (jobConfiguration.Target.Count(t => t.Type == APPLICATION_TYPE_APM) == 0)
{
logger.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
loggerConsole.Warn("No {0} targets to process", APPLICATION_TYPE_APM);
return(true);
}
// Process each target
for (int i = 0; i < jobConfiguration.Target.Count; i++)
{
Stopwatch stopWatchTarget = new Stopwatch();
stopWatchTarget.Start();
JobTarget jobTarget = jobConfiguration.Target[i];
if (jobTarget.Type != null && jobTarget.Type.Length > 0 && jobTarget.Type != APPLICATION_TYPE_APM)
{
continue;
}
StepTiming stepTimingTarget = new StepTiming();
stepTimingTarget.Controller = jobTarget.Controller;
stepTimingTarget.ApplicationName = jobTarget.Application;
stepTimingTarget.ApplicationID = jobTarget.ApplicationID;
stepTimingTarget.JobFileName = programOptions.OutputJobFilePath;
stepTimingTarget.StepName = jobConfiguration.Status.ToString();
stepTimingTarget.StepID = (int)jobConfiguration.Status;
stepTimingTarget.StartTime = DateTime.Now;
stepTimingTarget.NumEntities = 9;
try
{
this.DisplayJobTargetStartingStatus(jobConfiguration, jobTarget, i + 1);
#region Prepare time range
long fromTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].From);
long toTimeUnix = UnixTimeHelper.ConvertToUnixTimestamp(jobConfiguration.Input.HourlyTimeRanges[jobConfiguration.Input.HourlyTimeRanges.Count - 1].To);
long differenceInMinutes = (toTimeUnix - fromTimeUnix) / (60000);
#endregion
// Set up controller access
using (ControllerApi controllerApi = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)))
{
#region Application
loggerConsole.Info("Application Name");
string applicationJSON = controllerApi.GetAPMApplication(jobTarget.ApplicationID);
if (applicationJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(applicationJSON, FilePathMap.APMApplicationDataFilePath(jobTarget));
}
#endregion
#region Tiers
loggerConsole.Info("List of Tiers");
string tiersJSON = controllerApi.GetAPMTiers(jobTarget.ApplicationID);
if (tiersJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(tiersJSON, FilePathMap.APMTiersDataFilePath(jobTarget));
}
#endregion
#region Nodes
loggerConsole.Info("List of Nodes");
string nodesJSON = controllerApi.GetAPMNodes(jobTarget.ApplicationID);
if (nodesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodesJSON, FilePathMap.APMNodesDataFilePath(jobTarget));
}
#endregion
#region Backends
loggerConsole.Info("List of Backends");
string backendsJSON = controllerApi.GetAPMBackends(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
backendsJSON = controllerApi.GetAPMBackendsAdditionalDetail(jobTarget.ApplicationID);
if (backendsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendsJSON, FilePathMap.APMBackendsDetailDataFilePath(jobTarget));
}
List <AppDRESTBackend> backendsList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTBackend>(FilePathMap.APMBackendsDataFilePath(jobTarget));
if (backendsList != null)
{
loggerConsole.Info("DBMon Mappings for Backends ({0} entities)", backendsList.Count);
int j = 0;
var listOfBackendsInHourChunks = backendsList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_BACKENDS_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = BACKEND_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTBackend>, int>(
listOfBackendsInHourChunks,
parallelOptions,
() => 0,
(listOfBackendsInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTBackend backend in listOfBackendsInHourChunk)
{
if (File.Exists(FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend)) == false)
{
string backendToDBMonMappingJSON = controllerApi.GetAPMBackendToDBMonMapping(backend.id);
if (backendToDBMonMappingJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(backendToDBMonMappingJSON, FilePathMap.APMBackendToDBMonMappingDataFilePath(jobTarget, backend));
}
}
}
return(listOfBackendsInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Backends", backendsList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + backendsList.Count;
}
#endregion
#region Business Transactions
loggerConsole.Info("List of Business Transactions");
string businessTransactionsJSON = controllerApi.GetAPMBusinessTransactions(jobTarget.ApplicationID);
if (businessTransactionsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(businessTransactionsJSON, FilePathMap.APMBusinessTransactionsDataFilePath(jobTarget));
}
#endregion
#region Service Endpoints
loggerConsole.Info("List of Service Endpoints");
string serviceEndPointsJSON = controllerApi.GetAPMServiceEndpoints(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
serviceEndPointsJSON = controllerApi.GetAPMServiceEndpointsAdditionalDetail(jobTarget.ApplicationID);
if (serviceEndPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(serviceEndPointsJSON, FilePathMap.APMServiceEndpointsDetailDataFilePath(jobTarget));
}
#endregion
#region Errors
loggerConsole.Info("List of Errors");
string errorsJSON = controllerApi.GetAPMErrors(jobTarget.ApplicationID);
if (errorsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(errorsJSON, FilePathMap.APMErrorsDataFilePath(jobTarget));
}
#endregion
#region Information Points
loggerConsole.Info("List of Information Points");
string informationPointsJSON = controllerApi.GetAPMInformationPoints(jobTarget.ApplicationID);
if (informationPointsJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsJSON, FilePathMap.APMInformationPointsDataFilePath(jobTarget));
}
controllerApi.PrivateApiLogin();
string informationPointsDetailJSON = controllerApi.GetAPMInformationPointsAdditionalDetail(jobTarget.ApplicationID);
if (informationPointsDetailJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(informationPointsDetailJSON, FilePathMap.APMInformationPointsDetailDataFilePath(jobTarget));
}
#endregion
#region Node Properties
List <AppDRESTNode> nodesList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTNode>(FilePathMap.APMNodesDataFilePath(jobTarget));
if (nodesList != null)
{
loggerConsole.Info("Node Properties for Nodes ({0} entities)", nodesList.Count);
int j = 0;
var listOfNodesInHourChunks = nodesList.BreakListIntoChunks(ENTITIES_EXTRACT_NUMBER_OF_NODES_TO_PROCESS_PER_THREAD);
ParallelOptions parallelOptions = new ParallelOptions();
if (programOptions.ProcessSequentially == true)
{
parallelOptions.MaxDegreeOfParallelism = 1;
}
else
{
parallelOptions.MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS;
}
Parallel.ForEach <List <AppDRESTNode>, int>(
listOfNodesInHourChunks,
parallelOptions,
() => 0,
(listOfNodesInHourChunk, loop, subtotal) =>
{
// Set up controller access
ControllerApi controllerApiParallel = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword));
// Login into private API
controllerApiParallel.PrivateApiLogin();
foreach (AppDRESTNode node in listOfNodesInHourChunk)
{
if (File.Exists(FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node)) == false)
{
string nodePropertiesJSON = controllerApi.GetAPMNodeProperties(node.id);
if (nodePropertiesJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.APMNodeRuntimePropertiesDataFilePath(jobTarget, node));
}
}
if (File.Exists(FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node)) == false)
{
string nodeMetadataJSON = controllerApi.GetAPMNodeMetadata(jobTarget.ApplicationID, node.id);
if (nodeMetadataJSON != String.Empty)
{
FileIOHelper.SaveFileToPath(nodeMetadataJSON, FilePathMap.APMNodeMetadataDataFilePath(jobTarget, node));
}
}
}
return(listOfNodesInHourChunk.Count);
},
(finalResult) =>
{
Interlocked.Add(ref j, finalResult);
Console.Write("[{0}].", j);
}
);
loggerConsole.Info("Completed {0} Nodes", nodesList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + nodesList.Count;
}
#endregion
#region Backend to Tier Mappings
List <AppDRESTTier> tiersRESTList = FileIOHelper.LoadListOfObjectsFromFile <AppDRESTTier>(FilePathMap.APMTiersDataFilePath(jobTarget));
if (tiersRESTList != null)
{
loggerConsole.Info("Backend to Tier Mappings ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
string backendMappingsJSON = controllerApi.GetAPMBackendToTierMapping(tier.id);
if (backendMappingsJSON != String.Empty && backendMappingsJSON != "[ ]")
{
FileIOHelper.SaveFileToPath(backendMappingsJSON, FilePathMap.APMBackendToTierMappingDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
stepTimingTarget.NumEntities = stepTimingTarget.NumEntities + tiersRESTList.Count;
}
#endregion
#region Overflow Business Transactions
if (tiersRESTList != null)
{
loggerConsole.Info("Contents of Overflow Business Transaction in Tiers ({0} entities)", tiersRESTList.Count);
int j = 0;
foreach (AppDRESTTier tier in tiersRESTList)
{
JArray droppedBTsArray = new JArray();
JArray droppedBTsDebugModeArray = new JArray();
bool noMoreBTs = false;
long currentFetchedEventCount = 0;
long endEventID = 0;
while (noMoreBTs == false)
{
string batchOfBTsJSON = controllerApi.GetAPMBusinessTransactionsInOverflow(tier.id, currentFetchedEventCount, endEventID, fromTimeUnix, toTimeUnix, differenceInMinutes);
if (batchOfBTsJSON != String.Empty)
{
JObject batchOfBTsContainer = JObject.Parse(batchOfBTsJSON);
if (batchOfBTsContainer != null)
{
// Copy out both of the containers, not sure why there are multiple
if (isTokenPropertyNull(batchOfBTsContainer, "droppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["droppedTransactionItemList"])
{
droppedBTsArray.Add(btObject);
}
}
if (isTokenPropertyNull(batchOfBTsContainer, "debugModeDroppedTransactionItemList") == false)
{
foreach (JObject btObject in batchOfBTsContainer["debugModeDroppedTransactionItemList"])
{
droppedBTsDebugModeArray.Add(btObject);
}
}
currentFetchedEventCount = getLongValueFromJToken(batchOfBTsContainer, "eventSummariesCount");
endEventID = getLongValueFromJToken(batchOfBTsContainer, "endEventId");
if (currentFetchedEventCount == 0 || endEventID == 0)
{
// Done getting batches
noMoreBTs = true;
}
}
}
else
{
noMoreBTs = true;
}
}
if (droppedBTsArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionRegularDataFilePath(jobTarget, tier));
}
if (droppedBTsDebugModeArray.Count > 0)
{
FileIOHelper.SaveFileToPath(droppedBTsDebugModeArray.ToString(), FilePathMap.APMTierOverflowBusinessTransactionDebugDataFilePath(jobTarget, tier));
}
if (j % 10 == 0)
{
Console.Write("[{0}].", j);
}
j++;
}
loggerConsole.Info("Completed {0} Tiers", tiersRESTList.Count);
}
#endregion
}
}
catch (Exception ex)
{
logger.Warn(ex);
loggerConsole.Warn(ex);
return(false);
}
finally
{
stopWatchTarget.Stop();
this.DisplayJobTargetEndedStatus(jobConfiguration, jobTarget, i + 1, stopWatchTarget);
stepTimingTarget.EndTime = DateTime.Now;
stepTimingTarget.Duration = stopWatchTarget.Elapsed;
stepTimingTarget.DurationMS = stopWatchTarget.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingTarget);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
return(true);
}
catch (Exception ex)
{
logger.Error(ex);
loggerConsole.Error(ex);
return(false);
}
finally
{
stopWatch.Stop();
this.DisplayJobStepEndedStatus(jobConfiguration, stopWatch);
stepTimingFunction.EndTime = DateTime.Now;
stepTimingFunction.Duration = stopWatch.Elapsed;
stepTimingFunction.DurationMS = stopWatch.ElapsedMilliseconds;
List <StepTiming> stepTimings = new List <StepTiming>(1);
stepTimings.Add(stepTimingFunction);
FileIOHelper.WriteListToCSVFile(stepTimings, new StepTimingReportMap(), FilePathMap.StepTimingReportFilePath(), true);
}
}
public virtual bool ShouldExecute(JobConfiguration jobConfiguration)
{
return(false);
}
public static void On_TileSelectionManager_SetMouseAction(Game.GUI.TileSelectionManager self, Game.JobType job, Game.JobData data, bool multiselect, bool rotatable, bool groundSelect, bool snapToGround)
{
initiatingJob = new JobConfiguration()
{
Type = job,
Data = data,
MultiSelect = multiselect,
Rotatable = rotatable,
GroundSelect = groundSelect,
SnapToGround = snapToGround
};
if (data != null)
{
data.ToString();
return;
}
}
private static void Do_UpdateRightClickMenu(JobConfiguration newJob)
{
rebuildMenu.Enabled = true;
for (var i = 0; i < lastJobs.Count; i++)
{
var job = lastJobs[i].Configuration;
if ((job.Type == newJob.Type) && (job.ConstructionID == newJob.ConstructionID) && (job.Materials.SequenceEqual(newJob.Materials)))
{
rebuildMenu.Items.Remove(lastJobs[i].MenuItem);
lastJobs.RemoveAt(i);
}
}
while (lastJobs.Count > 5)
{
var last = lastJobs.Last();
rebuildMenu.Items.Remove(last.MenuItem);
lastJobs.Remove(last);
}
var newJobHistory = new JobHistoryItem()
{
Configuration = newJob,
MenuItem = new Game.GUI.Controls.MenuItem(newJob.ToString())
};
newJobHistory.MenuItem.Click += new Game.GUI.Controls.EventHandler(RedoMenuItem_Click);
lastJobs.Insert(0, newJobHistory);
rebuildMenu.Items.Insert(0, newJobHistory.MenuItem);
}
void UploadData(Table table)
{
Job job = new Job();
var config = new JobConfiguration();
var configLoad = new JobConfigurationLoad {
Schema = table.Schema,
DestinationTable = table.TableReference,
Encoding = "ISO-8859-1",
CreateDisposition = "CREATE_IF_NEEDED",
WriteDisposition = "",
FieldDelimiter = ",",
AllowJaggedRows = true,
SourceFormat = "CSV"
};
config.Load = configLoad;
job.Configuration = config;
var jobId = "---" + Environment.TickCount;
var jobRef = new JobReference {
JobId = jobId,
ProjectId = connection.ProjectId
};
job.JobReference = jobRef;
using (
Stream stream =
Assembly.GetExecutingAssembly()
.GetManifestResourceStream(string.Format("DevExpress.DataAccess.BigQuery.Tests.{0}.csv",
table.TableReference.TableId))) {
var insertMediaUpload = new JobsResource.InsertMediaUpload(connection.Service,
job, job.JobReference.ProjectId, stream, "application/octet-stream");
insertMediaUpload.Upload();
}
while (true) {
Job job1 = connection.Service.Jobs.Get(connection.ProjectId, jobId).Execute();
if (job1.Status.State.Equals("DONE")) {
break;
}
Thread.Sleep(5000);
}
}
