public void AzureFileDownload_Ok()
{
AzureFileDownloadActivity activity = new AzureFileDownloadActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("Prefix", @"");
list.Add("Modified", @"");
list.Add("OutputFolder", "C:\\Builds\\ZipFiles");
list.Add("FileShare", "xxx");
list.Add("AccountKey", "key");
list.Add("AccountName", "name");
list.Add("isSasToken", "true");
list.Add("Timeout", "0");
list.Add("SortOrder", "Desc");
list.Add("Count", "");
list.Add("CounterName", "");
list.Add("etl:BatchId", "1006");
list.Add("etl:StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
_logger.Debug("In Delta Extractor Configure method...");
//Default Validations
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
//obfuscate the password
SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]);
_logger.Debug("Controller: {Server}.{Database}", builder.DataSource, builder.InitialCatalog);
}
public void AwsS3Download_Ok()
{
AwsS3DownloadActivity activity = new AwsS3DownloadActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("Prefix", @"");
list.Add("OutputFolder", "C:\\Builds\\ZipFiles\\AWS");
list.Add("BucketName", "thl-bias-messages-staging");
//list.Add("ProfileName", "testRunner");
list.Add("AccountName", "");
list.Add("AccountKey", "");
list.Add("RegionName", "us-west-2");
list.Add("Timeout", "0");
list.Add("SortOrder", "Desc");
list.Add("Count", "");
list.Add("CounterName", "");
list.Add("etl:BatchId", "1006");
list.Add("etl:StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
if (attribute.Key.Equals(DECOMPRESS_MODE) &&
!supported_mode.Contains(attribute.Value.ToLower()))
{
throw new ArgumentException(String.Format("Unsupported {0}: {1}", DECOMPRESS_MODE, attribute.Value));
}
}
_logger.Information("TGZ: {InputFile} => {OutputFolder}, mode:{Mode}", _attributes[INPUT_FILE], _attributes[OUTPUT_FOLDER], _attributes[DECOMPRESS_MODE]);
}
public virtual void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
SqlConnectionStringBuilder builder_controller = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]);
SqlConnectionStringBuilder builder_register = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING_REGISTER]);
_logger.Debug("Controller: {Server}.{Database}", builder_controller.DataSource, builder_controller.InitialCatalog);
_logger.Debug("Register: {Server}.{Database}", builder_register.DataSource, builder_register.InitialCatalog);
_logger.Information("Processing from File Source: {File}", _attributes[FILE_SOURCE]);
}
public virtual void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
SqlConnectionStringBuilder builder_controller = new SqlConnectionStringBuilder(_attributes[CONTROLLER_CONNECTION_STRING]);
_logger.Debug("Controller: {Server}.{Database}", builder_controller.DataSource, builder_controller.InitialCatalog);
SqlConnectionStringBuilder builder_source = new SqlConnectionStringBuilder(_attributes[SOURCE_CONNECTION_STRING]);
_logger.Debug("Source: {Server}.{Database}", builder_source.DataSource, builder_source.InitialCatalog);
_logger.Information("Table: {TableName}, Control: {ControlName}", _attributes[TABLE_NAME], _attributes[TABLE_CONTROL_COLUMN]);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
if (attribute.Key.Equals(SORT_ORDER))
{
if (!sortList.Contains <string>(attribute.Value, StringComparer.InvariantCultureIgnoreCase))
{
throw new ArgumentException(String.Format("Invalid SortOrder:{0}. Supported: Asc,Desc.", attribute.Value));
}
}
_attributes.Add(attribute.Key, attribute.Value);
}
}
_logger.Information("Download: {From} -> {To}", _attributes[INPUT_PREFIX], _attributes[OUTPUT_FOLDER]);
_logger.Debug("Sort: {Sort}, Count: {Count}", _attributes[SORT_ORDER], _attributes[COUNT]);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]);
_logger.Debug("SqlServer: {Server}.{Database}", builder.DataSource, builder.InitialCatalog);
_logger.Information("Copy: {From} -> {To}", _attributes[AZURE_TABLE_NAME], _attributes[SQL_TABLE_NAME]);
_logger.Information("Delta: {ControlName} > {ControlValue}", _attributes[CONTROL_COLUMN], _attributes[CONTROL_VALUE]);
}
public virtual void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]);
_logger.Debug("Controller: {Server}.{Database}", builder.DataSource, builder.InitialCatalog);
_logger.Debug("Workflow: {WF}", _attributes[WORKFLOW_NAME]);
_logger.Debug("Mode: ProcessorName:{ProcessorName}, Debug:{Debug}, Verbose:{Verbose}, Forcestart:{Forstart}"
, _attributes[PROCESSOR_NAME], _attributes[PROCESSOR_MODE_DEBUG], _attributes[PROCESSOR_MODE_VERBOSE], _attributes[PROCESSOR_MODE_FORCESTART]);
}
public virtual void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
SqlConnectionStringBuilder builder = new SqlConnectionStringBuilder(_attributes[CONNECTION_STRING]);
_logger.Debug("EventServer: {Server}.{Database}", builder.DataSource, builder.InitialCatalog);
_logger.Debug("EventType: {Type}", _attributes[EVENT_TYPE]);
_logger.Debug("WatermarkEventType: {Type}", _attributes[WATERMARK_EVENT_TYPE]);
}
public void EventCheck_Ok()
{
CheckWorkflowEventActivity activity = new CheckWorkflowEventActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("EventType", @"Process2_FINISHED");
list.Add("WatermarkEventType", @"Process1_FINISHED");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void BsonConvert_Ok()
{
BsonConverterActivity activity = new BsonConverterActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\*.bson");
list.Add("OutputFolder", "c:\\Builds\\JsonFiles\\10-19-2016\\edxapp");
list.Add("Timeout", "30");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void BsonSqlLoader_Ok()
{
BsonSqlLoaderActivity activity = new BsonSqlLoaderActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\fs.files.bson");
list.Add("TableName", "dbo.staging_bson_test");
list.Add("Timeout", "600");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void TGZDecompress_Ok()
{
TGZDecompressActivity activity = new TGZDecompressActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("InputFile", "c:\\Builds\\ZipFiles\\test.tar.gz");
list.Add("OutputFolder", "c:\\Builds\\UnzipFiles");
list.Add("Mode", "tgz");
list.Add("Timeout", "30");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void DERun_Ok()
{
DeltaExtractorActivity activity = new DeltaExtractorActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("@BatchId", "1006");
list.Add("@StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void FileSetStatus_Ok()
{
FileSetProgressStatusActivity activity = new FileSetProgressStatusActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("FileId", "5");
list.Add("FileStatus", "Completed");
list.Add("Timeout", "30");
list.Add("etl:RunId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void EventPost_Ok()
{
PostWorkflowEventActivity activity = new PostWorkflowEventActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("EventType", @"Process2_FINISHED");
list.Add("EventPostDate", @"2016-12-16 18:04");
//list.Add("EventArgs", "<dwc:EventArgs xmlns:dwc=\"EventArgs.XSD\" Source=\"Process2 Finished\" PeriodGrain=\"Week\" Period=\"201652\" />");
list.Add("EventArgs", "");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
_logger.Information("Bson : {From} => {To}", _attributes[INPUT_FILE], _attributes[OUTPUT_FOLDER]);
}
public void FileRegister_Ok()
{
FileRegisterActivity activity = new FileRegisterActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
list.Add("RegisterPath", "c:\\Builds\\FlatFiles\\*.txt");
list.Add("SourceName", "testFiles");
list.Add("ProcessPriority", "1");
list.Add("Timeout", "30");
list.Add("@runId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void ExecuteSqlQuery_Ok()
{
SqlServerActivity activity = new SqlServerActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
//list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = False;User Id=test;Password=test; Connection Timeout = 120; ");
list.Add("Query", @"
print 'this is print test';
raiserror ('this is err test',11,11);
");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
_timeout = TimeSpan.FromSeconds(Int32.Parse(_attributes[TIMEOUT]));
_logger.Information("Wait Timeout: {Timeout}", _attributes[TIMEOUT]);
}
public void FileDequeue_Ok()
{
FileGetProcessListActivity activity = new FileGetProcessListActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.\sql14; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("SourceName", "testFiles");
list.Add("Timeout", "30");
list.Add("etl:RunId", "1");
list.Add("etl:BatchId", "101");
list.Add("etl:StepId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void Configure(WorkflowActivityArgs args)
{
_logger = args.Logger;
if (_required_attributes.Count != args.RequiredAttributes.Count)
{
//_logger.WriteError(String.Format("Not all required attributes are provided"), -11);
throw new ArgumentException("Not all required attributes are provided");
}
foreach (var attribute in args.RequiredAttributes)
{
if (_required_attributes.Contains(attribute.Key, StringComparer.InvariantCultureIgnoreCase))
{
_attributes.Add(attribute.Key, attribute.Value);
}
}
_logger.Information("console: {Exe} {Args}", _attributes[APP_NAME], _attributes[APP_ARGS]);
}
