public RavenConfiguration(string resourceName, ResourceType resourceType, string customConfigPath = null) { ResourceName = resourceName; ResourceType = resourceType; _configBuilder = new ConfigurationBuilder(); AddEnvironmentVariables(); AddJsonConfigurationVariables(customConfigPath); Settings = _configBuilder.Build(); Core = new CoreConfiguration(); Http = new HttpConfiguration(); Replication = new ReplicationConfiguration(); Cluster = new ClusterConfiguration(); Etl = new EtlConfiguration(); Storage = new StorageConfiguration(); Security = new SecurityConfiguration(); PerformanceHints = new PerformanceHintsConfiguration(); Indexing = new IndexingConfiguration(this); Monitoring = new MonitoringConfiguration(); Queries = new QueryConfiguration(); Patching = new PatchingConfiguration(); Logs = new LogsConfiguration(); Server = new ServerConfiguration(); Testing = new TestingConfiguration(); Databases = new DatabaseConfiguration(); Memory = new MemoryConfiguration(); Studio = new StudioConfiguration(); Licensing = new LicenseConfiguration(); Tombstones = new TombstoneConfiguration(); Subscriptions = new SubscriptionConfiguration(); }
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString { var result1 = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString)); Assert.NotNull(result1.RaftCommandIndex); return(src.Maintenance.Send(new AddEtlOperation <T>(configuration))); }
protected AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString { var putResult = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString)); Assert.NotNull(putResult.RaftCommandIndex); var addResult = src.Maintenance.Send(new AddEtlOperation <T>(configuration)); return(addResult); }
protected AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString { Console.WriteLine($"Adding ETL. Src: {src.Database}. Test: {Context.UniqueTestName}"); var putResult = src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString)); Assert.NotNull(putResult.RaftCommandIndex); var addResult = src.Maintenance.Send(new AddEtlOperation <T>(configuration)); Console.WriteLine($"Added ETL. Src: {src.Database}. Test: {Context.UniqueTestName}"); return(addResult); }
private void LogConfigurationError <T>(EtlConfiguration <T> config, List <string> errors) where T : ConnectionString { var errorMessage = $"Invalid ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " + $"Reason{(errors.Count > 1 ? "s" : string.Empty)}: {string.Join(";", errors)}."; if (Logger.IsInfoEnabled) { Logger.Info(errorMessage); } var alert = AlertRaised.Create(_database.Name, AlertTitle, errorMessage, AlertType.Etl_Error, NotificationSeverity.Error); _database.NotificationCenter.Add(alert); }
private void LogConfigurationWarning <T>(EtlConfiguration <T> config, List <string> warnings) where T : ConnectionString { var warnMessage = $"Warning about ETL configuration for '{config.Name}'{(config.Connection != null ? $" ({config.GetDestination()})" : string.Empty)}. " + $"Reason{(warnings.Count > 1 ? "s" : string.Empty)}: {string.Join(";", warnings)}."; if (Logger.IsInfoEnabled) { Logger.Info(warnMessage); } var alert = AlertRaised.Create(_database.Name, AlertTitle, warnMessage, AlertType.Etl_Warning, NotificationSeverity.Warning); _database.NotificationCenter.Add(alert); }
private static OngoingTaskState GetEtlTaskState <T>(EtlConfiguration <T> config) where T : ConnectionString { var taskState = OngoingTaskState.Enabled; if (config.Disabled || config.Transforms.All(x => x.Disabled)) { taskState = OngoingTaskState.Disabled; } else if (config.Transforms.Any(x => x.Disabled)) { taskState = OngoingTaskState.PartiallyEnabled; } return(taskState); }
private RavenConfiguration(string resourceName, ResourceType resourceType, string customConfigPath = null, bool skipEnvironmentVariables = false) { _logger = LoggingSource.Instance.GetLogger <RavenConfiguration>(resourceName); ResourceName = resourceName; ResourceType = resourceType; _customConfigPath = customConfigPath; PathSettingBase <string> .ValidatePath(_customConfigPath); _configBuilder = new ConfigurationBuilder(); if (skipEnvironmentVariables == false) { AddEnvironmentVariables(); } AddJsonConfigurationVariables(customConfigPath); Settings = _configBuilder.Build(); Core = new CoreConfiguration(); Http = new HttpConfiguration(); Replication = new ReplicationConfiguration(); Cluster = new ClusterConfiguration(); Etl = new EtlConfiguration(); Storage = new StorageConfiguration(); Security = new SecurityConfiguration(); Backup = new BackupConfiguration(); PerformanceHints = new PerformanceHintsConfiguration(); Indexing = new IndexingConfiguration(this); Monitoring = new MonitoringConfiguration(); Queries = new QueryConfiguration(); Patching = new PatchingConfiguration(); Logs = new LogsConfiguration(); Server = new ServerConfiguration(); Embedded = new EmbeddedConfiguration(); Databases = new DatabaseConfiguration(Storage.ForceUsing32BitsPager); Memory = new MemoryConfiguration(); Studio = new StudioConfiguration(); Licensing = new LicenseConfiguration(); Tombstones = new TombstoneConfiguration(); Subscriptions = new SubscriptionsConfiguration(); TransactionMergerConfiguration = new TransactionMergerConfiguration(Storage.ForceUsing32BitsPager); Notifications = new NotificationsConfiguration(); Updates = new UpdatesConfiguration(); Migration = new MigrationConfiguration(); Integrations = new IntegrationsConfiguration(); }
private bool ValidateConfiguration <T>(EtlConfiguration <T> config, HashSet <string> uniqueNames) where T : ConnectionString { if (config.Validate(out List <string> errors) == false) { LogConfigurationError(config, errors); return(false); } if (_databaseRecord.Encrypted && config.UsingEncryptedCommunicationChannel() == false && config.AllowEtlOnNonEncryptedChannel == false) { LogConfigurationError(config, new List <string> { $"{_database.Name} is encrypted, but connection to ETL destination {config.GetDestination()} does not use encryption, so ETL is not allowed. " + $"You can change this behavior by setting {nameof(config.AllowEtlOnNonEncryptedChannel)} when creating the ETL configuration" }); return(false); } if (_databaseRecord.Encrypted && config.UsingEncryptedCommunicationChannel() == false && config.AllowEtlOnNonEncryptedChannel) { LogConfigurationWarning(config, new List <string> { $"{_database.Name} is encrypted and connection to ETL destination {config.GetDestination()} does not use encryption, " + $"but {nameof(config.AllowEtlOnNonEncryptedChannel)} is set to true, so ETL is allowed" }); return(true); } if (uniqueNames.Add(config.Name) == false) { LogConfigurationError(config, new List <string> { $"ETL with name '{config.Name}' is already defined" }); return(false); } return(true); }
static void Main(string[] args) { try { //Load configuration file EtlConfiguration config = new EtlConfiguration(); config.ConnectionStrings = ConfigurationManager.ConnectionStrings; //Create job to execute using (var ETL_Job = new ETL_AdWordAcctPerformance(config)) { ETL_Job.Execute(); } } catch (Exception ex) { //Log error log.Error(string.Format("**ERROR** - {0}", ex.Message), ex); } }
static void Main(string[] args) { //exception handling here try { //Load configuration info EtlConfiguration config = new EtlConfiguration(); config.ConnectionStrings = ConfigurationManager.ConnectionStrings; //Instantiate ETL Process using (var ETL_Job = new ETL_AdWordCampaignPerformance(config)) //can auto generate class "ctrl + '.'" { //Execute ETL_AdWordCampaignPerformance ETL_Job.Execute(); } } catch(Exception ex) { //This will log any error that happens log.Error(string.Format("**ERROR** - {0}", ex.Message), ex); } }
static void Main(string[] args) { //Load Configuration file try { //Instantiate EtlConfiguration object EtlConfiguration config = new EtlConfiguration(); config.ConnectionStrings = ConfigurationManager.ConnectionStrings; //Instantiate object of new class you will auto gen and pass config in parameters //then execute using (var ETL_Job = new ETL_AdPerformance(config)) { ETL_Job.Execute(); } } catch(Exception ex) { //log any error log.Error(string.Format("**ERROR** - {0}", ex.Message), ex); } }
protected bool TryGetTransformationError <T>(string databaseName, EtlConfiguration <T> config, out EtlErrorInfo error) where T : ConnectionString { var database = GetDatabase(databaseName).Result; string tag; if (typeof(T) == typeof(ElasticSearchConnectionString)) { tag = ElasticSearchEtl.ElasticSearchEtlTag; } else if (typeof(T) == typeof(SqlConnectionString)) { tag = SqlEtl.SqlEtlTag; } else if (typeof(T) == typeof(RavenConnectionString)) { tag = RavenEtl.RavenEtlTag; } else if (typeof(T) == typeof(OlapConnectionString)) { tag = OlapEtl.OlaptEtlTag; } else { throw new NotSupportedException($"Unknown ETL type: {typeof(T)}"); } var loadAlert = database.NotificationCenter.EtlNotifications.GetAlert <EtlErrorsDetails>(tag, $"{config.Name}/{config.Transforms.First().Name}", AlertType.Etl_TransformationError); if (loadAlert.Errors.Count != 0) { error = loadAlert.Errors.First(); return(true); } error = null; return(false); }
//Constructor inherits base with config parameter public ETL_AdWordCampaignPerformance(EtlConfiguration config) : base(config) { }
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString { src.Maintenance.Send(new PutConnectionStringOperation <T>(connectionString)); return(src.Maintenance.Send(new AddEtlOperation <T>(configuration))); }
/*<summary> * Single Table Processes * </summary> * */ public static void RunProcess_RetrieveData(EtlConfiguration config) { }
/* * Single Table Processes Below * **/ public static void RunProcess_AdwordsGroup(EtlConfiguration config) { }
public AddEtlCommand(EtlConfiguration <T> configuration, string databaseName) { _configuration = configuration; _databaseName = databaseName; }
public MeridianEtlProcess(EtlConfiguration config, string logConfigFilePath) : base(logConfigFilePath) { EtlConfiguration = config; }
public MeridianEtlProcess(EtlConfiguration config) { EtlConfiguration = config; }
public MeridianEtlProcess() { EtlConfiguration = new EtlConfiguration(); }
//class constructor takes EtlConfiguration type config & inherits //base class with config parameter public ETL_AdPerformance(EtlConfiguration config) : base(config) { }
protected static AddEtlOperationResult AddEtl <T>(DocumentStore src, EtlConfiguration <T> configuration, T connectionString) where T : ConnectionString { src.Admin.Server.Send(new PutConnectionStringOperation <T>(connectionString, src.Database)); return(src.Admin.Server.Send(new AddEtlOperation <T>(configuration, src.Database))); }
public UpdateEtlCommand(long taskId, EtlConfiguration <T> configuration, string databaseName) { _taskId = taskId; _configuration = configuration; _databaseName = databaseName; }