public TableFactory(AppConfig config)
{
var storageConnection = KeyVault.GetSecretFromKeyvault(config.StorageConnection);
var storageAccount = CloudStorageAccount.Parse(storageConnection);
_tableClient = storageAccount.CreateCloudTableClient();
}
private static string GetInstrumentationKey(AppConfig settings)
{
var secretName = settings?.AppInsightsIntrumentationKey;
var vaultName = settings.ServiceKeyVaultName;
return(string.IsNullOrWhiteSpace(secretName) || string.IsNullOrWhiteSpace(vaultName)
? Guid.Empty.ToString()
: KeyVault.GetSecretFromKeyvault(settings.ServiceKeyVaultName, settings.AppInsightsIntrumentationKey));
}
/// <summary>
/// Checks if the value is a keyvault and if it is, gets the value from a keyvault
/// Otherwise, returns as is
/// </summary>
/// <param name="value"></param>
/// <returns>value or value from secret</returns>
public static string GetSecretFromKeyvaultIfNeeded(string value)
{
if (IsKeyVault(value))
{
return(KeyVault.GetSecretFromKeyvault(value));
}
return(value);
}
public JobRunner(IConfiguration configuration, ScheduledJobTable jobTable)
{
IServiceCollection services = new ServiceCollection();
var appConfig = new AppConfig(configuration);
services.AddLogging(loggingBuilder =>
{
// Optional: Apply filters to configure LogLevel Trace or above is sent to ApplicationInsights for all
// categories.
loggingBuilder.AddFilter <ApplicationInsightsLoggerProvider>("", LogLevel.Information);
var abc = GetInstrumentationKey(appConfig);
loggingBuilder.AddApplicationInsights(GetInstrumentationKey(appConfig));
});
IServiceProvider serviceProvider = services.BuildServiceProvider();
_logger = serviceProvider.GetRequiredService <ILogger <JobRunner> >();
var sbConnection = KeyVault.GetSecretFromKeyvault(appConfig.ServiceBusConnectionString);
_primaryQueueClient = new QueueClient(sbConnection, appConfig.PrimaryQueueName);
_testQueueClient = new QueueClient(sbConnection, appConfig.TestQueueName);
_activeQueueName = appConfig.ActiveQueueName;
// NOTE: If you need to DI other things (e.g. for your jobs) here's where you'd do it, and then just add the injected class into your constructor.
_services.AddSingleton <AppConfig>(appConfig)
.AddApplicationInsightsTelemetry(new ApplicationInsightsServiceOptions()
{
EnableAdaptiveSampling = false,
EnableDebugLogger = false,
InstrumentationKey = GetInstrumentationKey(appConfig)
})
.AddLogging(logging =>
{
try
{
// In order to log ILogger logs
logging.AddApplicationInsights();
// Optional: Apply filters to configure LogLevel Information or above is sent to
// ApplicationInsights for all categories.
logging.AddFilter <ApplicationInsightsLoggerProvider>("", LogLevel.Information);
// Additional filtering For category starting in "Microsoft",
// only Warning or above will be sent to Application Insights.
logging.AddFilter <ApplicationInsightsLoggerProvider>("Microsoft", LogLevel.Warning);
}
catch (Exception e)
{
}
});
_services.AddSingleton <ILogger>(_logger);
LoadJobTypes();
_provider = _services.BuildServiceProvider();
this.ScheduledJobTable = jobTable;
}
/// <summary>
/// UDFPathResolver resolves the keyvault uri and gets the real path
/// </summary>
/// <param name="path">path</param>
/// <returns>Returns a string </returns>
private string UDFPathResolver(string path)
{
if (path != null && Config.Utility.KeyVaultUri.IsSecretUri(path))
{
SecretUriParser.ParseSecretUri(path, out string keyvalut, out string secret);
var secretUri = KeyVault.GetSecretFromKeyvault(keyvalut, secret);
return(secretUri);
}
return(path);
}
/// <summary>
/// Moving the method that sets the various environment variables
/// </summary>
/// <returns>This returns the success or failure as understood by the frontend</returns>
public async Task <ApiResult> GetEnvironmentVariables()
{
CosmosDBDatabaseName = "production";
var response = ServiceFabricUtil.GetServiceKeyVaultName();
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
string serviceKeyvaultName = response.Result.ToString();
var cosmosCon = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigConnectionString").Result.ToString());
CosmosDBDatabaseName = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigDatabaseName").Result.ToString());
var namePassword = Helper.ParseCosmosDBUserNamePassword(cosmosCon);
if (string.IsNullOrEmpty(cosmosCon) || string.IsNullOrEmpty(namePassword) || namePassword.Split(new char[] { ':' }).Count() != 2)
{
return(ApiResult.CreateError("Can't get UserName or Password from CosmosDB connection string"));
}
CosmosDBEndPoint = Helper.ParseCosmosDBEndPoint(cosmosCon);
CosmosDBUserName = namePassword.Split(new char[] { ':' })[0];
CosmosDBPassword = namePassword.Split(new char[] { ':' })[1];
response = await CosmosDB.DownloadConfigFromDocumentDB(CosmosDBDatabaseName, CosmosDBEndPoint, CosmosDBUserName, CosmosDBPassword, ServiceFabricUtil.GetServiceFabricConfigSetting("cosmosDBConfigCollectionName").Result.ToString());
if (response.Error.HasValue && response.Error.Value)
{
return(ApiResult.CreateError(response.Message));
}
var flowConfigObj = response.Result.ToObject <FlowConfigObject>();
if (flowConfigObj != null)
{
EngineFlowConfig = flowConfigObj;
ResourceCreation = flowConfigObj.ResourceCreation.ToLower().Equals("true") ? true : false;
FlowBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.StorageAccountName + "-blobconnectionstring");
OpsBlobConnectionString = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, flowConfigObj.ConfiggenSecretPrefix + flowConfigObj.OpsStorageAccountName + "-blobconnectionstring");
SparkConnInfo = Helper.ParseConnectionString(Helper.PathResolver(flowConfigObj.SparkConnectionString));
return(ApiResult.CreateSuccess(""));
}
return(ApiResult.CreateError("Failed to get environment variables"));
}
static GatewayController()
{
WinHttpHandler handler = new WinHttpHandler();
// We need to set timeout for handler first
// Setting timeout for httpClient alone is not good enough
var timeout = TimeSpan.FromSeconds(_DefaultHttpTimeoutSecs);
handler.SendTimeout = timeout;
handler.ReceiveDataTimeout = timeout;
handler.ReceiveHeadersTimeout = timeout;
_HttpClient = new HttpClient(handler)
{
// Default http timeout is 100s, increase it to 4 min since few key mainline scenarios
// can take longer than default 100s
Timeout = timeout
};
// Attach remote cert validation to ignore self-signed ssl cert error if reverseProxySslThumbprint is specified in the config
handler.ServerCertificateValidationCallback = ValidateRemoteCert;
_ReverseProxySslThumbprint = GetReverseProxySslThumbprintFromConfig();
_AllowedUserRoles = GetAllowedUserRolesFromConfig();
var serviceKeyvaultName = ServiceFabricUtil.GetServiceKeyVaultName().Result.ToString();
var configPackage = FabricRuntime.GetActivationContext().GetConfigurationPackageObject("Config");
var serviceEnvironmenConfig = configPackage.Settings.Sections["ServiceEnvironment"];
var appInsightsIntrumentationKey = serviceEnvironmenConfig.Parameters["AppInsightsIntrumentationKey"].Value;
var testClientId = serviceEnvironmenConfig.Parameters["TestClientId"].Value;
_IsUserInfoLoggingEnabled = IsUserInfoLoggingEnabled();
_StaticLogger = new ApplicationInsightsLogger("GatewayILogger", new Microsoft.ApplicationInsights.TelemetryClient(new TelemetryConfiguration(KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, appInsightsIntrumentationKey))), new ApplicationInsightsLoggerOptions());
try
{
// Each secret needs to be a list of unique Ids in the format {ObjectId}.{TenantId}
List <string> userIdList = KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, testClientId).Split(new char[] { ',' }).ToList();
foreach (string userId in userIdList)
{
_ClientWhitelist.Add(userId);
}
}
catch (Exception e)
{
// Do nothing in case the TestClientId is not set in the keyvault. This is set for testing purposes.
var message = e.Message;
_StaticLogger.LogError(e.Message);
}
}
public KafkaMessageBus(string brokerList, string connectionString, List <string> topics, string consumerGroup, string inputType, ILogger logger)
{
if (!File.Exists(_cacertLocation))
{
var certSource = KeyVault.GetSecretFromKeyvault(ServiceFabricUtil.GetServiceFabricConfigSetting("CACertificateLocation").Result.ToString());
WebClient webClient = new WebClient();
webClient.DownloadFile(certSource, _cacertLocation);
}
_brokerList = brokerList;
_connectionString = connectionString;
_topics = topics;
_consumerGroup = consumerGroup;
_inputType = inputType;
_logger = logger;
}
/// <summary>
/// UDFPathResolver resolves the keyvault uri and gets the real path
/// </summary>
/// <param name="path">path</param>
/// <returns>Returns a string </returns>
private string UDFPathResolver(string path)
{
if (path != null && Config.Utility.KeyVaultUri.IsSecretUri(path))
{
Regex r = new Regex(@"^((keyvault:?):\/\/)?([^:\/\s]+)(\/)(.*)?", RegexOptions.IgnoreCase);
var keyvault = string.Empty;
var secret = string.Empty;
MatchCollection match = r.Matches(path);
if (match != null && match.Count > 0)
{
foreach (Match m in match)
{
keyvault = m.Groups[3].Value.Trim();
secret = m.Groups[5].Value.Trim();
}
}
var secretUri = KeyVault.GetSecretFromKeyvault(keyvault, secret);
return(secretUri);
}
return(path);
}
/// <summary>
/// This is the method that gets called when the job starts running
/// </summary>
/// <returns></returns>
public async Task RunAsync()
{
if (string.IsNullOrWhiteSpace(_config.ServiceUrl))
{
string errorMessage = "Server URL is not available.";
_logger.LogError(_scenario.Description, "JobRunner ScenarioTester", new Dictionary <string, string>()
{
{ "scenario.errorMessage", errorMessage }
});
throw new InvalidOperationException(errorMessage);
}
using (var context = new ScenarioContext())
{
context[Context.ServiceUrl] = _config.ServiceUrl;
context[Context.ApplicationId] = KeyVault.GetSecretFromKeyvault(_config.ApplicationId);
// The flow config needs to be saved at this location
string blobUri = $"{_config.BlobUri}";
context[Context.FlowConfigContent] = await Task.Run(() => BlobUtility.GetBlobContent(KeyVault.GetSecretFromKeyvault(_config.BlobConnectionString), blobUri));
context[Context.ApplicationIdentifierUri] = _config.ApplicationIdentifierUri;
context[Context.SecretKey] = KeyVault.GetSecretFromKeyvault(_config.SecretKey);
context[Context.MicrosoftAuthority] = _config.MicrosoftAuthority;
using (_logger.BeginScope <IReadOnlyCollection <KeyValuePair <string, object> > >(
new Dictionary <string, object> {
{ "scenario.Description", _scenario.Description },
{ "scenarioCount", _scenarioCount.ToString() },
{ "scenario.Steps", $"[{string.Join(", ", _scenario.Steps.Select(s => s.Method.Name))}]" }
}))
{
// do actual logging inside the scope. All logs inside this will have the properties from the Dictionary used in begin scope.
_logger.LogInformation("JobRunner ScenarioTester: " + _scenario.Description);
}
var results = await ScenarioResult.RunAsync(_scenario, context, _scenarioCount);
int iterationCount = 0;
foreach (var result in results)
{
string scenarioResult = result.Failed ? "failed" : "succeeded";
// log failed steps.
foreach (var stepResult in result.StepResults.Where(r => !r.Success))
{
using (_logger.BeginScope <IReadOnlyCollection <KeyValuePair <string, object> > >(
new Dictionary <string, object> {
{ "Scenario iteration", $"Scenario iteration {_scenario.Description}.{iterationCount} " },
{ "ScenarioResult length", scenarioResult.Length }
}))
{
// do actual logging inside the scope. All logs inside this will have the properties from the Dictionary used in begin scope.
_logger.LogInformation(_scenario.Description);
}
if (stepResult.Exception != null)
{
_logger.LogError(stepResult.Exception, _scenario.Description);
}
_logger.LogError(stepResult.Value);
}
iterationCount++;
}
//emit metric on how many parallel executions passed.
using (_logger.BeginScope <IReadOnlyCollection <KeyValuePair <string, object> > >(
new Dictionary <string, object> {
{ $"SuccessRate:{_scenario.Description}", $"{(long)((double)results.Count(r => !r.Failed) / _scenarioCount * 100.0)}" }
}))
{
// do actual logging inside the scope. All logs inside this will have the properties from the Dictionary used in begin scope.
_logger.LogInformation(_scenario.Description);
}
}
}
static GatewayController()
{
WinHttpHandler handler = new WinHttpHandler();
// We need to set timeout for handler first
// Setting timeout for httpClient alone is not good enough
var timeout = TimeSpan.FromSeconds(_DefaultHttpTimeoutSecs);
handler.SendTimeout = timeout;
handler.ReceiveDataTimeout = timeout;
handler.ReceiveHeadersTimeout = timeout;
_HttpClient = new HttpClient(handler)
{
// Default http timeout is 100s, increase it to 4 min since few key mainline scenarios
// can take longer than default 100s
Timeout = timeout
};
// Attach remote cert validation to ignore self-signed ssl cert error if reverseProxySslThumbprint is specified in the config
handler.ServerCertificateValidationCallback = ValidateRemoteCert;
_ReverseProxySslThumbprint = GetReverseProxySslThumbprintFromConfig();
_AllowedUserRoles = GetAllowedUserRolesFromConfig();
var serviceKeyvaultName = ServiceFabricUtil.GetServiceKeyVaultName().Result.ToString();
var configPackage = FabricRuntime.GetActivationContext().GetConfigurationPackageObject("Config");
var serviceEnvironmenConfig = configPackage.Settings.Sections["ServiceEnvironment"];
var appInsightsIntrumentationKey = serviceEnvironmenConfig.Parameters["AppInsightsIntrumentationKey"].Value;
_IsUserInfoLoggingEnabled = IsUserInfoLoggingEnabled();
_StaticLogger = new ApplicationInsightsLogger("GatewayILogger", new Microsoft.ApplicationInsights.TelemetryClient(new TelemetryConfiguration(KeyVault.GetSecretFromKeyvault(serviceKeyvaultName, appInsightsIntrumentationKey))), new ApplicationInsightsLoggerOptions());
}
