public RavenConfiguration(string resourceName, ResourceType resourceType, string customConfigPath = null)
{
ResourceName = resourceName;
ResourceType = resourceType;
_configBuilder = new ConfigurationBuilder();
AddEnvironmentVariables();
AddJsonConfigurationVariables(customConfigPath);
Settings = _configBuilder.Build();
Core = new CoreConfiguration();
Http = new HttpConfiguration();
Replication = new ReplicationConfiguration();
Cluster = new ClusterConfiguration();
Etl = new EtlConfiguration();
Storage = new StorageConfiguration();
Security = new SecurityConfiguration();
PerformanceHints = new PerformanceHintsConfiguration();
Indexing = new IndexingConfiguration(this);
Monitoring = new MonitoringConfiguration();
Queries = new QueryConfiguration();
Patching = new PatchingConfiguration();
Logs = new LogsConfiguration();
Server = new ServerConfiguration();
Testing = new TestingConfiguration();
Databases = new DatabaseConfiguration();
Memory = new MemoryConfiguration();
Studio = new StudioConfiguration();
Licensing = new LicenseConfiguration();
Tombstones = new TombstoneConfiguration();
Subscriptions = new SubscriptionConfiguration();
}
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString
{
var result1 = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString));
Assert.NotNull(result1.RaftCommandIndex);
return(src.Maintenance.Send(new AddEtlOperation <T>(configuration)));
}
protected AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString
{
var putResult = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString));
Assert.NotNull(putResult.RaftCommandIndex);
var addResult = src.Maintenance.Send(new AddEtlOperation <T>(configuration));
return(addResult);
}
protected AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString
{
Console.WriteLine($"Adding ETL. Src: {src.Database}. Test: {Context.UniqueTestName}");
var putResult = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString));
Assert.NotNull(putResult.RaftCommandIndex);
var addResult = src.Maintenance.Send(new AddEtlOperation <T>(configuration));
Console.WriteLine($"Added ETL. Src: {src.Database}. Test: {Context.UniqueTestName}");
return(addResult);
}
private void LogConfigurationError <T>(EtlConfiguration <T> config, List <string> errors) where T : ConnectionString
{
var errorMessage = $"Invalid ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
$"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}.";
if (Logger.IsInfoEnabled)
{
Logger.Info(errorMessage);
}
var alert = AlertRaised.Create(_database.Name, AlertTitle, errorMessage, AlertType.Etl_Error, NotificationSeverity.Error);
_database.NotificationCenter.Add(alert);
}
private void LogConfigurationWarning <T>(EtlConfiguration <T> config, List <string> warnings) where T : ConnectionString
{
var warnMessage = $"Warning about ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " +
$"Reason{(warnings.Count > 1 ? "s" : string.Empty)}: {string.Join(";", warnings)}.";
if (Logger.IsInfoEnabled)
{
Logger.Info(warnMessage);
}
var alert = AlertRaised.Create(_database.Name, AlertTitle, warnMessage, AlertType.Etl_Warning, NotificationSeverity.Warning);
_database.NotificationCenter.Add(alert);
}
private static OngoingTaskState GetEtlTaskState <T>(EtlConfiguration <T> config) where T : ConnectionString
{
var taskState = OngoingTaskState.Enabled;
if (config.Disabled || config.Transforms.All(x => x.Disabled))
{
taskState = OngoingTaskState.Disabled;
}
else if (config.Transforms.Any(x => x.Disabled))
{
taskState = OngoingTaskState.PartiallyEnabled;
}
return(taskState);
}
private RavenConfiguration(string resourceName, ResourceType resourceType, string customConfigPath = null, bool skipEnvironmentVariables = false)
{
_logger = LoggingSource.Instance.GetLogger <RavenConfiguration>(resourceName);
ResourceName = resourceName;
ResourceType = resourceType;
_customConfigPath = customConfigPath;
PathSettingBase <string> .ValidatePath(_customConfigPath);
_configBuilder = new ConfigurationBuilder();
if (skipEnvironmentVariables == false)
{
AddEnvironmentVariables();
}
AddJsonConfigurationVariables(customConfigPath);
Settings = _configBuilder.Build();
Core = new CoreConfiguration();
Http = new HttpConfiguration();
Replication = new ReplicationConfiguration();
Cluster = new ClusterConfiguration();
Etl = new EtlConfiguration();
Storage = new StorageConfiguration();
Security = new SecurityConfiguration();
Backup = new BackupConfiguration();
PerformanceHints = new PerformanceHintsConfiguration();
Indexing = new IndexingConfiguration(this);
Monitoring = new MonitoringConfiguration();
Queries = new QueryConfiguration();
Patching = new PatchingConfiguration();
Logs = new LogsConfiguration();
Server = new ServerConfiguration();
Embedded = new EmbeddedConfiguration();
Databases = new DatabaseConfiguration(Storage.ForceUsing32BitsPager);
Memory = new MemoryConfiguration();
Studio = new StudioConfiguration();
Licensing = new LicenseConfiguration();
Tombstones = new TombstoneConfiguration();
Subscriptions = new SubscriptionsConfiguration();
TransactionMergerConfiguration = new TransactionMergerConfiguration(Storage.ForceUsing32BitsPager);
Notifications = new NotificationsConfiguration();
Updates = new UpdatesConfiguration();
Migration = new MigrationConfiguration();
Integrations = new IntegrationsConfiguration();
}
private bool ValidateConfiguration <T>(EtlConfiguration <T> config, HashSet <string> uniqueNames) where T : ConnectionString
{
if (config.Validate(out List <string> errors) == false)
{
LogConfigurationError(config, errors);
return(false);
}
if (_databaseRecord.Encrypted && config.UsingEncryptedCommunicationChannel() == false && config.AllowEtlOnNonEncryptedChannel == false)
{
LogConfigurationError(config,
new List <string>
{
$"{_database.Name} is encrypted, but connection to ETL destination {config.GetDestination()} does not use encryption, so ETL is not allowed. " +
$"You can change this behavior by setting {nameof(config.AllowEtlOnNonEncryptedChannel)} when creating the ETL configuration"
});
return(false);
}
if (_databaseRecord.Encrypted && config.UsingEncryptedCommunicationChannel() == false && config.AllowEtlOnNonEncryptedChannel)
{
LogConfigurationWarning(config,
new List <string>
{
$"{_database.Name} is encrypted and connection to ETL destination {config.GetDestination()} does not use encryption, " +
$"but {nameof(config.AllowEtlOnNonEncryptedChannel)} is set to true, so ETL is allowed"
});
return(true);
}
if (uniqueNames.Add(config.Name) == false)
{
LogConfigurationError(config,
new List <string>
{
$"ETL with name '{config.Name}' is already defined"
});
return(false);
}
return(true);
}
static void Main(string[] args)
{
try
{
//Load configuration file
EtlConfiguration config = new EtlConfiguration();
config.ConnectionStrings = ConfigurationManager.ConnectionStrings;
//Create job to execute
using (var ETL_Job = new ETL_AdWordAcctPerformance(config))
{
ETL_Job.Execute();
}
}
catch (Exception ex)
{
//Log error
log.Error(string.Format("**ERROR** - {0}", ex.Message), ex);
}
}
static void Main(string[] args)
{
//exception handling here
try
{
//Load configuration info
EtlConfiguration config = new EtlConfiguration();
config.ConnectionStrings = ConfigurationManager.ConnectionStrings;
//Instantiate ETL Process
using (var ETL_Job = new ETL_AdWordCampaignPerformance(config)) //can auto generate class "ctrl + '.'"
{
//Execute ETL_AdWordCampaignPerformance
ETL_Job.Execute();
}
}
catch(Exception ex)
{
//This will log any error that happens
log.Error(string.Format("**ERROR** - {0}", ex.Message), ex);
}
}
static void Main(string[] args)
{
//Load Configuration file
try
{
//Instantiate EtlConfiguration object
EtlConfiguration config = new EtlConfiguration();
config.ConnectionStrings = ConfigurationManager.ConnectionStrings;
//Instantiate object of new class you will auto gen and pass config in parameters
//then execute
using (var ETL_Job = new ETL_AdPerformance(config))
{
ETL_Job.Execute();
}
}
catch(Exception ex)
{
//log any error
log.Error(string.Format("**ERROR** - {0}", ex.Message), ex);
}
}
protected bool TryGetTransformationError <T>(string databaseName, EtlConfiguration <T> config, out EtlErrorInfo error) where T : ConnectionString
{
var database = GetDatabase(databaseName).Result;
string tag;
if (typeof(T) == typeof(ElasticSearchConnectionString))
{
tag = ElasticSearchEtl.ElasticSearchEtlTag;
}
else if (typeof(T) == typeof(SqlConnectionString))
{
tag = SqlEtl.SqlEtlTag;
}
else if (typeof(T) == typeof(RavenConnectionString))
{
tag = RavenEtl.RavenEtlTag;
}
else if (typeof(T) == typeof(OlapConnectionString))
{
tag = OlapEtl.OlaptEtlTag;
}
else
{
throw new NotSupportedException($"Unknown ETL type: {typeof(T)}");
}
var loadAlert = database.NotificationCenter.EtlNotifications.GetAlert <EtlErrorsDetails>(tag, $"{config.Name}/{config.Transforms.First().Name}", AlertType.Etl_TransformationError);
if (loadAlert.Errors.Count != 0)
{
error = loadAlert.Errors.First();
return(true);
}
error = null;
return(false);
}
//Constructor inherits base with config parameter
public ETL_AdWordCampaignPerformance(EtlConfiguration config)
: base(config)
{
}
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString
{
src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString));
return(src.Maintenance.Send(new AddEtlOperation <T>(configuration)));
}
/*<summary>
* Single Table Processes
* </summary>
* */
public static void RunProcess_RetrieveData(EtlConfiguration config)
{
}
/*
* Single Table Processes Below
*
**/
public static void RunProcess_AdwordsGroup(EtlConfiguration config)
{
}
public AddEtlCommand(EtlConfiguration <T> configuration, string databaseName)
{
_configuration = configuration;
_databaseName = databaseName;
}
public MeridianEtlProcess(EtlConfiguration config, string logConfigFilePath)
: base(logConfigFilePath)
{
EtlConfiguration = config;
}
public MeridianEtlProcess(EtlConfiguration config)
{
EtlConfiguration = config;
}
public MeridianEtlProcess()
{
EtlConfiguration = new EtlConfiguration();
}
//class constructor takes EtlConfiguration type config & inherits
//base class with config parameter
public ETL_AdPerformance(EtlConfiguration config)
: base(config)
{
}
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString
{
src.Admin.Server.Send(new PutConnectionStringOperation <T>(connectionString, src.Database));
return(src.Admin.Server.Send(new AddEtlOperation <T>(configuration, src.Database)));
}
public UpdateEtlCommand(long taskId, EtlConfiguration <T> configuration, string databaseName)
{
_taskId = taskId;
_configuration = configuration;
_databaseName = databaseName;
}
