private static void JobSetup(JobBase job, IDictionary <string, string> jobArgsDictionary, ref int?sleepDuration)
{
if (JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, "dbg"))
{
throw new ArgumentException("-dbg is a special argument and should be the first argument...");
}
if (JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, "ConsoleLogOnly"))
{
throw new ArgumentException("-ConsoleLogOnly is a special argument and should be the first argument (can be the second if '-dbg' is used)...");
}
if (sleepDuration == null)
{
Trace.TraceWarning("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
sleepDuration = 5000;
}
// Initialize the job once with everything needed.
// JobTraceListener(s) are already initialized
if (!job.Init(jobArgsDictionary))
{
// If the job could not be initialized successfully, STOP!
Trace.TraceError("Exiting. The job could not be initialized successfully with the arguments passed");
}
}
/// <summary>
/// Initializes an <see cref="ISqlConnectionFactory"/>, for use by non-validation jobs.
/// </summary>
/// <returns>ConnectionStringBuilder, used for diagnostics.</returns>
public SqlConnectionStringBuilder RegisterDatabase(
IServiceContainer serviceContainer,
IDictionary <string, string> jobArgsDictionary,
string connectionStringArgName,
bool testConnection = true)
{
if (serviceContainer == null)
{
throw new ArgumentNullException(nameof(serviceContainer));
}
if (jobArgsDictionary == null)
{
throw new ArgumentNullException(nameof(jobArgsDictionary));
}
if (string.IsNullOrEmpty(connectionStringArgName))
{
throw new ArgumentException("Argument cannot be null or empty.", nameof(connectionStringArgName));
}
var secretInjector = (ISecretInjector)serviceContainer.GetService(typeof(ISecretInjector));
var connectionString = JobConfigurationManager.GetArgument(jobArgsDictionary, connectionStringArgName);
return(RegisterDatabase(connectionStringArgName, connectionString, testConnection, secretInjector));
}
public ISecretReader CreateSecretReader(IDictionary <string, string> settings)
{
if (JobConfigurationManager.TryGetArgument(settings, JobArgumentNames.VaultName) == null)
{
return(new EmptySecretReader());
}
var storeName = JobConfigurationManager.TryGetArgument(settings, JobArgumentNames.StoreName);
var storeLocation = JobConfigurationManager.TryGetArgument(settings, JobArgumentNames.StoreLocation);
var certificate = CertificateUtility.FindCertificateByThumbprint(
storeName != null ? (StoreName)Enum.Parse(typeof(StoreName), storeName) : StoreName.My,
storeLocation != null ? (StoreLocation)Enum.Parse(typeof(StoreLocation), storeLocation) : StoreLocation.LocalMachine,
JobConfigurationManager.GetArgument(settings, JobArgumentNames.CertificateThumbprint),
JobConfigurationManager.TryGetBoolArgument(settings, JobArgumentNames.ValidateCertificate, defaultValue: true));
var keyVaultConfiguration =
new KeyVaultConfiguration(
JobConfigurationManager.GetArgument(settings, JobArgumentNames.VaultName),
JobConfigurationManager.GetArgument(settings, JobArgumentNames.ClientId),
certificate);
var refreshIntervalSec = JobConfigurationManager.TryGetIntArgument(settings,
JobArgumentNames.RefreshIntervalSec) ?? CachingSecretReader.DefaultRefreshIntervalSec;
return(new CachingSecretReader(new KeyVaultReader(keyVaultConfiguration), refreshIntervalSec));
}
public override void Init(IServiceContainer serviceContainer, IDictionary <string, string> jobArgsDictionary)
{
var configurationFilename = JobConfigurationManager.GetArgument(jobArgsDictionary, ConfigurationArgument);
var configurationRoot = GetConfigurationRoot(configurationFilename, out var secretInjector);
_serviceProvider = GetServiceProvider(configurationRoot, secretInjector);
RegisterDatabases(_serviceProvider);
}
/// <summary>
/// This is a static method to run a job whose args are passed in
/// By default,
/// a) The job will be run continuously in a while loop. Could be overridden using 'once' argument
/// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument
/// </summary>
/// <param name="job">Job to run</param>
/// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param>
/// <returns></returns>
public static async Task Run(JobBase job, string[] commandLineArgs)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-dbg", StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
bool consoleLogOnly = false;
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-ConsoleLogOnly", StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
consoleLogOnly = true;
}
// Set the default trace listener, so if we get args parsing issues they will be printed. This will be overriden by the configured trace listener
// after config is parsed.
job.SetJobTraceListener(new JobTraceListener());
try
{
Trace.TraceInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(commandLineArgs, job.JobName, (ISecretReaderFactory)ServiceContainer.GetService(typeof(ISecretReaderFactory)));
// Set JobTraceListener. This will be done on every job run as well
SetJobTraceListener(job, consoleLogOnly, jobArgsDictionary);
var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
if (!sleepDuration.HasValue)
{
sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
if (sleepDuration.HasValue)
{
sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
}
}
// Setup the job for running
JobSetup(job, consoleLogOnly, jobArgsDictionary, ref sleepDuration);
// Run the job loop
await JobLoop(job, runContinuously, sleepDuration.Value, consoleLogOnly, jobArgsDictionary);
}
catch (Exception ex)
{
ex.TraceException();
}
// Flush here. This is VERY IMPORTANT!
// Exception messages from when the job faults are still in the queue and need to be flushed.
// Also, if the job is only run once, this is what flushes the queue.
job.JobTraceListener.Close();
}
/// <summary>
/// This is a static method to run a job whose args are passed in
/// By default,
/// a) The job will be run continuously in a while loop. Could be overridden using 'once' argument
/// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument
/// </summary>
/// <param name="job">Job to run</param>
/// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param>
/// <returns></returns>
public static async Task Run(JobBase job, string[] commandLineArgs)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-dbg", StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
bool consoleLogOnly = false;
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-ConsoleLogOnly", StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
consoleLogOnly = true;
}
// Set JobTraceListener. This will be done on every job run as well
SetJobTraceListener(job, consoleLogOnly);
try
{
Trace.TraceInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(job.JobTraceListener, commandLineArgs, job.JobName);
bool runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
int? sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep);
// Setup the job for running
JobSetup(job, jobArgsDictionary, ref sleepDuration);
// Run the job loop
await JobLoop(job, runContinuously, sleepDuration.Value, consoleLogOnly);
}
catch (AggregateException ex)
{
var innerEx = ex.InnerExceptions.Count > 0 ? ex.InnerExceptions[0] : null;
if (innerEx != null)
{
Trace.TraceError("[FAILED]: " + innerEx);
}
else
{
Trace.TraceError("[FAILED]: " + ex);
}
}
catch (Exception ex)
{
Trace.TraceError("[FAILED]: " + ex);
}
// Call FlushAll here. This is VERY IMPORTANT
// Exception message(s) when the job faults are still in the queue. Need to be flushed
// Also, when the job is only run once, FlushAll is important again
job.JobTraceListener.Close();
}
private static void SetJobTraceListener(JobBase job, bool consoleLogOnly, IDictionary <string, string> argsDictionary)
{
if (consoleLogOnly)
{
job.SetJobTraceListener(new JobTraceListener());
}
else
{
var connectionString = JobConfigurationManager.GetArgument(argsDictionary, JobArgumentNames.LogsAzureStorageConnectionString);
job.SetJobTraceListener(new AzureBlobJobTraceListener(job.JobName, connectionString));
}
}
public SqlExportArguments(IDictionary <string, string> jobArgsDictionary, string defaultContainerName, string defaultName)
{
var connStrBldr = new SqlConnectionStringBuilder(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.SourceDatabase, EnvironmentVariableKeys.SqlGallery));
ConnectionString = connStrBldr.ToString();
OutputDirectory = JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.OutputDirectory);
if (string.IsNullOrEmpty(OutputDirectory))
{
Destination = CloudStorageAccount.Parse(
JobConfigurationManager.GetArgument(jobArgsDictionary, JobArgumentNames.PrimaryDestination, EnvironmentVariableKeys.StoragePrimary));
DestinationContainerName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.DestinationContainerName) ?? defaultContainerName;
DestinationContainer = Destination.CreateCloudBlobClient().GetContainerReference(DestinationContainerName);
}
Name = defaultName;
}
private static void JobSetup(JobBase job, bool consoleLogOnly, IDictionary <string, string> jobArgsDictionary, ref int?sleepDuration)
{
if (JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, "dbg"))
{
throw new ArgumentException("-dbg is a special argument and should be the first argument...");
}
if (JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, "ConsoleLogOnly"))
{
throw new ArgumentException("-ConsoleLogOnly is a special argument and should be the first argument (can be the second if '-dbg' is used)...");
}
if (sleepDuration == null)
{
Trace.TraceWarning("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
sleepDuration = 5000;
}
job.ConsoleLogOnly = consoleLogOnly;
}
private static ApplicationInsightsConfiguration ConfigureApplicationInsights(JobBase job, IDictionary <string, string> jobArgsDictionary)
{
ApplicationInsightsConfiguration applicationInsightsConfiguration;
var instrumentationKey = JobConfigurationManager.TryGetArgument(
jobArgsDictionary,
JobArgumentNames.InstrumentationKey);
var heartbeatIntervalSeconds = JobConfigurationManager.TryGetIntArgument(
jobArgsDictionary,
JobArgumentNames.HeartbeatIntervalSeconds);
if (heartbeatIntervalSeconds.HasValue)
{
applicationInsightsConfiguration = ApplicationInsights.Initialize(
instrumentationKey,
TimeSpan.FromSeconds(heartbeatIntervalSeconds.Value));
}
else
{
applicationInsightsConfiguration = ApplicationInsights.Initialize(instrumentationKey);
}
// Determine job and instance name, for logging.
var jobName = job.GetType().Assembly.GetName().Name;
var instanceName = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstanceName) ?? jobName;
applicationInsightsConfiguration.TelemetryConfiguration.TelemetryInitializers.Add(
new JobPropertiesTelemetryInitializer(jobName, instanceName, job.GlobalTelemetryDimensions));
applicationInsightsConfiguration.TelemetryConfiguration
.ApplicationIdProvider = new ApplicationInsightsApplicationIdProvider();
job.SetApplicationInsightsConfiguration(applicationInsightsConfiguration);
return(applicationInsightsConfiguration);
}
private static async Task <int> Run(JobBase job, string[] commandLineArgs, bool?runContinuously)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
// Configure logging before Application Insights is enabled.
// This is done so, in case Application Insights fails to initialize, we still see output.
var loggerFactory = ConfigureLogging(job, telemetryConfiguration: null);
int exitCode;
try
{
_logger.LogInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(
ServiceContainer,
loggerFactory.CreateLogger(typeof(JobConfigurationManager)),
commandLineArgs);
// Setup logging
_applicationInsightsConfiguration = ConfigureApplicationInsights(job, jobArgsDictionary);
// Configure our logging again with Application Insights initialized.
loggerFactory = ConfigureLogging(job, _applicationInsightsConfiguration.TelemetryConfiguration);
var hasOnceArgument = JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
if (runContinuously.HasValue && hasOnceArgument)
{
_logger.LogWarning(
$"This job is designed to {(runContinuously.Value ? "run continuously" : "run once")} so " +
$"the -{JobArgumentNames.Once} argument is {(runContinuously.Value ? "ignored" : "redundant")}.");
}
runContinuously = runContinuously ?? !hasOnceArgument;
var reinitializeAfterSeconds = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.ReinitializeAfterSeconds);
var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
if (!sleepDuration.HasValue)
{
sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
if (sleepDuration.HasValue)
{
sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
}
}
if (!sleepDuration.HasValue)
{
if (runContinuously.Value)
{
_logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
}
sleepDuration = 5000;
}
else if (!runContinuously.Value)
{
_logger.LogWarning(
$"The job is designed to run once so the -{JobArgumentNames.Sleep} and " +
$"-{JobArgumentNames.Interval} arguments are ignored.");
}
if (!reinitializeAfterSeconds.HasValue)
{
_logger.LogInformation(
$"{JobArgumentNames.ReinitializeAfterSeconds} command line argument is not provided or is not a valid integer. " +
"The job will reinitialize on every iteration");
}
else if (!runContinuously.Value)
{
_logger.LogWarning(
$"The job is designed to run once so the -{JobArgumentNames.ReinitializeAfterSeconds} " +
$"argument is ignored.");
}
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Run the job loop
exitCode = await JobLoop(job, runContinuously.Value, sleepDuration.Value, reinitializeAfterSeconds, jobArgsDictionary);
}
catch (Exception ex)
{
exitCode = 1;
_logger.LogError("Job runner threw an exception: {Exception}", ex);
}
Trace.Close();
_applicationInsightsConfiguration.TelemetryConfiguration.TelemetryChannel.Flush();
_applicationInsightsConfiguration.Dispose();
return(exitCode);
}
/// <summary>
/// This is a static method to run a job whose args are passed in
/// By default,
/// a) The job will be run continuously in a while loop. Could be overridden using 'once' argument
/// b) The sleep duration between each run when running continuously is 5000 milliSeconds. Could be overridden using '-Sleep' argument
/// </summary>
/// <param name="job">Job to run</param>
/// <param name="commandLineArgs">Args contains args to the job runner like (dbg, once and so on) and for the job itself</param>
/// <returns></returns>
public static async Task Run(JobBase job, string[] commandLineArgs)
{
if (commandLineArgs.Length > 0 && string.Equals(commandLineArgs[0], "-" + JobArgumentNames.Dbg, StringComparison.OrdinalIgnoreCase))
{
commandLineArgs = commandLineArgs.Skip(1).ToArray();
Debugger.Launch();
}
// Configure logging before Application Insights is enabled.
// This is done so, in case Application Insights fails to initialize, we still see output.
var loggerFactory = ConfigureLogging(job);
try
{
_logger.LogInformation("Started...");
// Get the args passed in or provided as an env variable based on jobName as a dictionary of <string argName, string argValue>
var jobArgsDictionary = JobConfigurationManager.GetJobArgsDictionary(loggerFactory.CreateLogger(typeof(JobConfigurationManager)), commandLineArgs, job.JobName, (ISecretReaderFactory)ServiceContainer.GetService(typeof(ISecretReaderFactory)));
// Setup logging
if (!ApplicationInsights.Initialized)
{
string instrumentationKey = JobConfigurationManager.TryGetArgument(jobArgsDictionary, JobArgumentNames.InstrumentationKey);
if (!string.IsNullOrWhiteSpace(instrumentationKey))
{
ApplicationInsights.Initialize(instrumentationKey);
}
}
// Configure our logging again with Application Insights initialized.
loggerFactory = ConfigureLogging(job);
var runContinuously = !JobConfigurationManager.TryGetBoolArgument(jobArgsDictionary, JobArgumentNames.Once);
var sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Sleep); // sleep is in milliseconds
if (!sleepDuration.HasValue)
{
sleepDuration = JobConfigurationManager.TryGetIntArgument(jobArgsDictionary, JobArgumentNames.Interval);
if (sleepDuration.HasValue)
{
sleepDuration = sleepDuration.Value * 1000; // interval is in seconds
}
}
if (sleepDuration == null)
{
_logger.LogInformation("SleepDuration is not provided or is not a valid integer. Unit is milliSeconds. Assuming default of 5000 ms...");
sleepDuration = 5000;
}
// Ensure that SSLv3 is disabled and that Tls v1.2 is enabled.
ServicePointManager.SecurityProtocol &= ~SecurityProtocolType.Ssl3;
ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
// Run the job loop
await JobLoop(job, runContinuously, sleepDuration.Value, jobArgsDictionary);
}
catch (Exception ex)
{
_logger.LogError("Job runner threw an exception: {Exception}", ex);
}
}
