public void Test_Workflow_Attributes_Get_Ok()
{
DBController db = DBController.Create(connectionString);
WorkflowAttributeCollection attributes = db.WorkflowAttributeCollectionGet(100, 1, 0, 0);
Assert.IsTrue(attributes.Count > 0);
}
public void AwsS3Download_Ok()
{
AwsS3DownloadActivity activity = new AwsS3DownloadActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("Prefix", @"");
list.Add("OutputFolder", "C:\\Builds\\ZipFiles\\AWS");
list.Add("BucketName", "thl-bias-messages-staging");
//list.Add("ProfileName", "testRunner");
list.Add("AccountName", "");
list.Add("AccountKey", "");
list.Add("RegionName", "us-west-2");
list.Add("Timeout", "0");
list.Add("SortOrder", "Desc");
list.Add("Count", "");
list.Add("CounterName", "");
list.Add("etl:BatchId", "1006");
list.Add("etl:StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void AzureFileDownload_Ok()
{
AzureFileDownloadActivity activity = new AzureFileDownloadActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("Prefix", @"");
list.Add("Modified", @"");
list.Add("OutputFolder", "C:\\Builds\\ZipFiles");
list.Add("FileShare", "xxx");
list.Add("AccountKey", "key");
list.Add("AccountName", "name");
list.Add("isSasToken", "true");
list.Add("Timeout", "0");
list.Add("SortOrder", "Desc");
list.Add("Count", "");
list.Add("CounterName", "");
list.Add("etl:BatchId", "1006");
list.Add("etl:StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public virtual WfResult Run(CancellationToken token)
{
WfResult result = WfResult.Unknown;
//_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING]));
try
{
WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();
attributes.Add(PROCESSOR_NAME, _attributes[PROCESSOR_NAME]);
attributes.Add(PROCESSOR_MODE_DEBUG, _attributes[PROCESSOR_MODE_DEBUG]);
attributes.Add(PROCESSOR_MODE_VERBOSE, _attributes[PROCESSOR_MODE_VERBOSE]);
attributes.Add(PROCESSOR_MODE_FORCESTART, _attributes[PROCESSOR_MODE_FORCESTART]);
attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, _attributes[WORKFLOW_NAME]);
attributes.Add(CONNECTION_STRING, _attributes[CONNECTION_STRING]);
attributes.Add(REQUEST_ID, _attributes[REQUEST_ID]);
WorkflowProcessor wfp = new WorkflowProcessor(attributes);
result = wfp.Run(token);
_logger.Information("Activity finished with result {WfStatus}: {Message}", result.StatusCode, result.Message);
}
catch (SqlException ex)
{
throw ex;
//_logger.Write(String.Format("SqlServer exception: {0}", ex.Message));
//result = WfResult.Create(WfStatus.Failed, ex.Message, ex.ErrorCode);
}
return(result);
}
public void WFRun_Ok()
{
//var builder = new ConfigurationBuilder()
// .AddJsonFile("appsettings.json")
// .Build();
//string runnerName = builder.GetSection("Data:Runner").Value;
//string connectionString = builder.GetSection("Data:Controller").Value;
var settings = ConfigurationManager.AppSettings;
string runnerName = settings["Runner"];
string connectionString = settings["Controller"];
Log.Logger = new LoggerConfiguration()
.ReadFrom.AppSettings()
//.ReadFrom.Configuration(builder)
.MinimumLevel.Debug()
//.WriteTo.File(path: @"c:\logs\log.txt",
// rollOnFileSizeLimit: true,
// rollingInterval: RollingInterval.Hour,
// fileSizeLimitBytes: 1024000
// )
.WriteTo.WorkflowLogger(connectionString: connectionString)
.CreateLogger();
string WFName = "test107";
WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();
//attributes.Add(WorkflowConstants.ATTRIBUTE_PROCESSOR_NAME, runnerName);
attributes.Add(WorkflowConstants.ATTRIBUTE_DEBUG, "false");
attributes.Add(WorkflowConstants.ATTRIBUTE_VERBOSE, "true");
attributes.Add(WorkflowConstants.ATTRIBUTE_FORCESTART, "true");
attributes.Add(WorkflowConstants.ATTRIBUTE_CONTROLLER_CONNECTIONSTRING, connectionString);
attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, WFName);
//attributes.Add(WorkflowConstants.ATTRIBUTE_REQUEST_ID, Guid.NewGuid().ToString());
WfResult wr = WfResult.Unknown;
using (CancellationTokenSource cts = new CancellationTokenSource())
{
WorkflowProcessor wfp = new WorkflowProcessor(attributes);
wr = wfp.Run(cts.Token);
}
Assert.IsTrue(wr.StatusCode == WfStatus.Succeeded);
}
public WfResult Start(WorkflowAttributeCollection args, CancellationToken token)
{
WfResult result = WfResult.Succeeded;
_args = args;
Parameters parameters = new Parameters();
try
{
Version v = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;
string inputXml;
if (_args.ContainsKey(XML))
{
inputXml = _args[XML];
}
else
{
inputXml = DeParameters();
}
if (!inputXml.StartsWith("<?xml version="))
{
byte[] base64ByteArr = Convert.FromBase64String(XML);
inputXml = System.Text.UnicodeEncoding.Unicode.GetString(base64ByteArr);
}
parameters = Parameters.DeSerializefromXml(inputXml);
_logger.Information("Running DE v.{Version} ({Bit} bit)", v.ToString(), 8 * IntPtr.Size);
_logger.Information("Executing as: {User}", WindowsIdentity.GetCurrent().Name.ToString());
_logger.Debug("DE input parameter: {DeXml}", inputXml);
//logger.WriteDebug("DE XML: " + inputXml);
DEController controller = new DEController(_logger);
controller.Execute(parameters, token);
}
catch (Exception ex)
{
_logger.Error(ex, "Exception: {Message}", ex.Message);
result = WfResult.Failed;
}
return(result);
}
public void BsonConvert_Ok()
{
BsonConverterActivity activity = new BsonConverterActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\*.bson");
list.Add("OutputFolder", "c:\\Builds\\JsonFiles\\10-19-2016\\edxapp");
list.Add("Timeout", "30");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void EventCheck_Ok()
{
CheckWorkflowEventActivity activity = new CheckWorkflowEventActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("EventType", @"Process2_FINISHED");
list.Add("WatermarkEventType", @"Process1_FINISHED");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void BsonSqlLoader_Ok()
{
BsonSqlLoaderActivity activity = new BsonSqlLoaderActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\fs.files.bson");
list.Add("TableName", "dbo.staging_bson_test");
list.Add("Timeout", "600");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
/// <summary>
/// Entry point to Delta Extractor. Takes parameters defined by an XML string based
/// on the parameters.XSD schema.
/// </summary>
/// <param name="args"></param>
/// <returns></returns>
public static void Main(string[] args)
{
if (args.Length == 0 ||
args.Contains(@"/help", StringComparer.InvariantCultureIgnoreCase))
{
help();
Environment.Exit(0);
}
bool debug = args.Contains(@"/D", StringComparer.InvariantCultureIgnoreCase);
bool verbose = args.Contains(@"/V", StringComparer.InvariantCultureIgnoreCase);
try
{
var minLogLevel = (debug) ? LogEventLevel.Debug : LogEventLevel.Information;
ILogger logger = new LoggerConfiguration()
.MinimumLevel.Is(minLogLevel)
.WriteTo.Console()
.CreateLogger();
;
WorkflowAttributeCollection param = new WorkflowAttributeCollection();
param.Add("XML", args[0]);
using (CancellationTokenSource cts = new CancellationTokenSource())
{
DERun runner = new DERun(logger);
WfResult result = runner.Start(param, cts.Token);
if (result.StatusCode != WfStatus.Succeeded)
{
throw new Exception(String.Format("DE returned Status: {0}", result.StatusCode.ToString()));
}
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Environment.Exit(1);
}
Environment.Exit(0);
}
public void DERun_Ok()
{
DeltaExtractorActivity activity = new DeltaExtractorActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("@BatchId", "1006");
list.Add("@StepId", "11");
list.Add("@RunId", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void TGZDecompress_Ok()
{
TGZDecompressActivity activity = new TGZDecompressActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("InputFile", "c:\\Builds\\ZipFiles\\test.tar.gz");
list.Add("OutputFolder", "c:\\Builds\\UnzipFiles");
list.Add("Mode", "tgz");
list.Add("Timeout", "30");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void FileSetStatus_Ok()
{
FileSetProgressStatusActivity activity = new FileSetProgressStatusActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("FileId", "5");
list.Add("FileStatus", "Completed");
list.Add("Timeout", "30");
list.Add("etl:RunId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void EventPost_Ok()
{
PostWorkflowEventActivity activity = new PostWorkflowEventActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("EventType", @"Process2_FINISHED");
list.Add("EventPostDate", @"2016-12-16 18:04");
//list.Add("EventArgs", "<dwc:EventArgs xmlns:dwc=\"EventArgs.XSD\" Source=\"Process2 Finished\" PeriodGrain=\"Week\" Period=\"201652\" />");
list.Add("EventArgs", "");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void FileRegister_Ok()
{
FileRegisterActivity activity = new FileRegisterActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
list.Add("RegisterPath", "c:\\Builds\\FlatFiles\\*.txt");
list.Add("SourceName", "testFiles");
list.Add("ProcessPriority", "1");
list.Add("Timeout", "30");
list.Add("@runId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void ExecuteSqlQuery_Ok()
{
SqlServerActivity activity = new SqlServerActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
//list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = False;User Id=test;Password=test; Connection Timeout = 120; ");
list.Add("Query", @"
print 'this is print test';
raiserror ('this is err test',11,11);
");
list.Add("Timeout", "0");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
public void FileDequeue_Ok()
{
FileGetProcessListActivity activity = new FileGetProcessListActivity();
WorkflowActivityArgs wfa = new WorkflowActivityArgs();
WorkflowAttributeCollection list = new WorkflowAttributeCollection();
list.Add("ConnectionString", @"Server=.\sql14; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
list.Add("SourceName", "testFiles");
list.Add("Timeout", "30");
list.Add("etl:RunId", "1");
list.Add("etl:BatchId", "101");
list.Add("etl:StepId", "1");
wfa.RequiredAttributes = list;
wfa.Logger = _logger;
activity.Configure(wfa);
WfResult result = activity.Run(CancellationToken.None);
Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
}
static int Main(string[] args)
{
if (args.Length == 0 ||
args.Contains(@"/help", StringComparer.InvariantCultureIgnoreCase))
{
help();
return(0);
}
WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();
attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, args[0].Replace("\"", ""));
attributes.Add(WorkflowConstants.ATTRIBUTE_DEBUG, "false");
attributes.Add(WorkflowConstants.ATTRIBUTE_VERBOSE, "false");
attributes.Add(WorkflowConstants.ATTRIBUTE_FORCESTART, "false");
var minLogLevel = LogEventLevel.Information;
//bool debug = false;
if (args.Contains(@"/D", StringComparer.InvariantCultureIgnoreCase))
{
attributes[WorkflowConstants.ATTRIBUTE_DEBUG] = "true";
//debug = true;
minLogLevel = LogEventLevel.Debug;
}
//bool forcestart = false;
if (args.Contains(@"/R", StringComparer.InvariantCultureIgnoreCase))
{
attributes[WorkflowConstants.ATTRIBUTE_FORCESTART] = "true";
//forcestart = true;
}
bool verbose = false;
if (args.Contains(@"/V", StringComparer.InvariantCultureIgnoreCase))
{
attributes[WorkflowConstants.ATTRIBUTE_VERBOSE] = "true";
verbose = true;
}
//var builder = new ConfigurationBuilder()
// .AddJsonFile("appsettings.json")
// .Build();
//string runnerName = builder.GetSection("Data:Runner").Value;
//string connectionString = builder.GetSection("Data:Controller").Value;
var settings = ConfigurationManager.AppSettings;
string runnerName = settings["Runner"];
string connectionString = settings["Controller"];
attributes.Add(WorkflowConstants.ATTRIBUTE_CONTROLLER_CONNECTIONSTRING, connectionString);
//string runnerName = settings["Runner"];
if (String.IsNullOrEmpty(runnerName))
{
runnerName = "Default";
}
attributes.Add(WorkflowConstants.ATTRIBUTE_PROCESSOR_NAME, runnerName);
if (verbose)
{
Log.Logger = new LoggerConfiguration()
//.ReadFrom.Configuration(builder)
.ReadFrom.AppSettings()
.MinimumLevel.Is(minLogLevel)
//.WriteTo.Console()
.WriteTo.WorkflowLogger(connectionString: connectionString)
.CreateLogger();
}
else
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Is(minLogLevel)
.WriteTo.WorkflowLogger(connectionString: connectionString)
.CreateLogger();
}
try
{
WfResult wr = WfResult.Unknown;
using (CancellationTokenSource cts = new CancellationTokenSource())
{
WorkflowProcessor wfp = new WorkflowProcessor(attributes);
wr = wfp.Run(cts.Token);
}
if (wr.StatusCode != WfStatus.Succeeded)
{
return(1);
}
}
catch (Exception ex)
{
Console.WriteLine(String.Format("Error: {0}, {1}", ex.HResult, ex.Message));
return(1);
}
return(0);
}