public void AzureFileDownload_Ok() { AzureFileDownloadActivity activity = new AzureFileDownloadActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("Prefix", @""); list.Add("Modified", @""); list.Add("OutputFolder", "C:\\Builds\\ZipFiles"); list.Add("FileShare", "xxx"); list.Add("AccountKey", "key"); list.Add("AccountName", "name"); list.Add("isSasToken", "true"); list.Add("Timeout", "0"); list.Add("SortOrder", "Desc"); list.Add("Count", ""); list.Add("CounterName", ""); list.Add("etl:BatchId", "1006"); list.Add("etl:StepId", "11"); list.Add("@RunId", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; _logger.Debug("In Delta Extractor Configure method..."); //Default Validations if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } //obfuscate the password SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]); _logger.Debug("Controller: {Server}.{Database}", builder.DataSource, builder.InitialCatalog); }
public void AwsS3Download_Ok() { AwsS3DownloadActivity activity = new AwsS3DownloadActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("Prefix", @""); list.Add("OutputFolder", "C:\\Builds\\ZipFiles\\AWS"); list.Add("BucketName", "thl-bias-messages-staging"); //list.Add("ProfileName", "testRunner"); list.Add("AccountName", ""); list.Add("AccountKey", ""); list.Add("RegionName", "us-west-2"); list.Add("Timeout", "0"); list.Add("SortOrder", "Desc"); list.Add("Count", ""); list.Add("CounterName", ""); list.Add("etl:BatchId", "1006"); list.Add("etl:StepId", "11"); list.Add("@RunId", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } if (attribute.Key.Equals(DECOMPRESS_MODE) && !supported_mode.Contains(attribute.Value.ToLower())) { throw new ArgumentException(String.Format("Unsupported {0}: {1}", DECOMPRESS_MODE, attribute.Value)); } } _logger.Information("TGZ: {InputFile} => {OutputFolder}, mode:{Mode}", _attributes[INPUT_FILE], _attributes[OUTPUT_FOLDER], _attributes[DECOMPRESS_MODE]); }
public virtual void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } SqlConnectionStringBuilder builder_controller = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]); SqlConnectionStringBuilder builder_register = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING_REGISTER]); _logger.Debug("Controller: {Server}.{Database}", builder_controller.DataSource, builder_controller.InitialCatalog); _logger.Debug("Register: {Server}.{Database}", builder_register.DataSource, builder_register.InitialCatalog); _logger.Information("Processing from File Source: {File}", _attributes[FILE_SOURCE]); }
public virtual void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } SqlConnectionStringBuilder builder_controller = new SqlConnectionStringBuilder(_attributes[CONTROLLER_CONNECTION_STRING]); _logger.Debug("Controller: {Server}.{Database}", builder_controller.DataSource, builder_controller.InitialCatalog); SqlConnectionStringBuilder builder_source = new SqlConnectionStringBuilder(_attributes[SOURCE_CONNECTION_STRING]); _logger.Debug("Source: {Server}.{Database}", builder_source.DataSource, builder_source.InitialCatalog); _logger.Information("Table: {TableName}, Control: {ControlName}", _attributes[TABLE_NAME], _attributes[TABLE_CONTROL_COLUMN]); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { if (attribute.Key.Equals(SORT_ORDER)) { if (!sortList.Contains <string>(attribute.Value, StringComparer.InvariantCultureIgnoreCase)) { throw new ArgumentException(String.Format("Invalid SortOrder:{0}. Supported: Asc,Desc.", attribute.Value)); } } _attributes.Add(attribute.Key, attribute.Value); } } _logger.Information("Download: {From} -> {To}", _attributes[INPUT_PREFIX], _attributes[OUTPUT_FOLDER]); _logger.Debug("Sort: {Sort}, Count: {Count}", _attributes[SORT_ORDER], _attributes[COUNT]); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]); _logger.Debug("SqlServer: {Server}.{Database}", builder.DataSource, builder.InitialCatalog); _logger.Information("Copy: {From} -> {To}", _attributes[AZURE_TABLE_NAME], _attributes[SQL_TABLE_NAME]); _logger.Information("Delta: {ControlName} > {ControlValue}", _attributes[CONTROL_COLUMN], _attributes[CONTROL_VALUE]); }
public virtual void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]); _logger.Debug("Controller: {Server}.{Database}", builder.DataSource, builder.InitialCatalog); _logger.Debug("Workflow: {WF}", _attributes[WORKFLOW_NAME]); _logger.Debug("Mode: ProcessorName:{ProcessorName}, Debug:{Debug}, Verbose:{Verbose}, Forcestart:{Forstart}" , _attributes[PROCESSOR_NAME], _attributes[PROCESSOR_MODE_DEBUG], _attributes[PROCESSOR_MODE_VERBOSE], _attributes[PROCESSOR_MODE_FORCESTART]); }
public virtual void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]); _logger.Debug("EventServer: {Server}.{Database}", builder.DataSource, builder.InitialCatalog); _logger.Debug("EventType: {Type}", _attributes[EVENT_TYPE]); _logger.Debug("WatermarkEventType: {Type}", _attributes[WATERMARK_EVENT_TYPE]); }
public void EventCheck_Ok() { CheckWorkflowEventActivity activity = new CheckWorkflowEventActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("EventType", @"Process2_FINISHED"); list.Add("WatermarkEventType", @"Process1_FINISHED"); list.Add("Timeout", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void BsonConvert_Ok() { BsonConverterActivity activity = new BsonConverterActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\*.bson"); list.Add("OutputFolder", "c:\\Builds\\JsonFiles\\10-19-2016\\edxapp"); list.Add("Timeout", "30"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void BsonSqlLoader_Ok() { BsonSqlLoaderActivity activity = new BsonSqlLoaderActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; "); list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\fs.files.bson"); list.Add("TableName", "dbo.staging_bson_test"); list.Add("Timeout", "600"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void TGZDecompress_Ok() { TGZDecompressActivity activity = new TGZDecompressActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("InputFile", "c:\\Builds\\ZipFiles\\test.tar.gz"); list.Add("OutputFolder", "c:\\Builds\\UnzipFiles"); list.Add("Mode", "tgz"); list.Add("Timeout", "30"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void DERun_Ok() { DeltaExtractorActivity activity = new DeltaExtractorActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("@BatchId", "1006"); list.Add("@StepId", "11"); list.Add("@RunId", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void FileSetStatus_Ok() { FileSetProgressStatusActivity activity = new FileSetProgressStatusActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("FileId", "5"); list.Add("FileStatus", "Completed"); list.Add("Timeout", "30"); list.Add("etl:RunId", "1"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void EventPost_Ok() { PostWorkflowEventActivity activity = new PostWorkflowEventActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("EventType", @"Process2_FINISHED"); list.Add("EventPostDate", @"2016-12-16 18:04"); //list.Add("EventArgs", "<dwc:EventArgs xmlns:dwc=\"EventArgs.XSD\" Source=\"Process2 Finished\" PeriodGrain=\"Week\" Period=\"201652\" />"); list.Add("EventArgs", ""); list.Add("Timeout", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } _logger.Information("Bson : {From} => {To}", _attributes[INPUT_FILE], _attributes[OUTPUT_FOLDER]); }
public void FileRegister_Ok() { FileRegisterActivity activity = new FileRegisterActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; "); list.Add("RegisterPath", "c:\\Builds\\FlatFiles\\*.txt"); list.Add("SourceName", "testFiles"); list.Add("ProcessPriority", "1"); list.Add("Timeout", "30"); list.Add("@runId", "1"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void ExecuteSqlQuery_Ok() { SqlServerActivity activity = new SqlServerActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); //list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = False;User Id=test;Password=test; Connection Timeout = 120; "); list.Add("Query", @" print 'this is print test'; raiserror ('this is err test',11,11); "); list.Add("Timeout", "0"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } _timeout = TimeSpan.FromSeconds(Int32.Parse(_attributes[TIMEOUT])); _logger.Information("Wait Timeout: {Timeout}", _attributes[TIMEOUT]); }
public void FileDequeue_Ok() { FileGetProcessListActivity activity = new FileGetProcessListActivity(); WorkflowActivityArgs wfa = new WorkflowActivityArgs(); WorkflowAttributeCollection list = new WorkflowAttributeCollection(); list.Add("ConnectionString", @"Server=.\sql14; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; "); list.Add("SourceName", "testFiles"); list.Add("Timeout", "30"); list.Add("etl:RunId", "1"); list.Add("etl:BatchId", "101"); list.Add("etl:StepId", "1"); wfa.RequiredAttributes = list; wfa.Logger = _logger; activity.Configure(wfa); WfResult result = activity.Run(CancellationToken.None); Assert.IsTrue(result.StatusCode == WfStatus.Succeeded); }
public void Configure(WorkflowActivityArgs args) { _logger = args.Logger; if (_required_attributes.Count != args.RequiredAttributes.Count) { //_logger.WriteError(String.Format("Not all required attributes are provided"), -11); throw new ArgumentException("Not all required attributes are provided"); } foreach (var attribute in args.RequiredAttributes) { if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase)) { _attributes.Add(attribute.Key, attribute.Value); } } _logger.Information("console: {Exe} {Args}", _attributes[APP_NAME], _attributes[APP_ARGS]); }