コード例 #1
0
        public void Test_Workflow_Attributes_Get_Ok()
        {
            DBController db = DBController.Create(connectionString);
            WorkflowAttributeCollection attributes = db.WorkflowAttributeCollectionGet(100, 1, 0, 0);

            Assert.IsTrue(attributes.Count > 0);
        }
コード例 #2
0
        public void AwsS3Download_Ok()
        {
            AwsS3DownloadActivity       activity = new AwsS3DownloadActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("Prefix", @"");
            list.Add("OutputFolder", "C:\\Builds\\ZipFiles\\AWS");
            list.Add("BucketName", "thl-bias-messages-staging");
            //list.Add("ProfileName", "testRunner");
            list.Add("AccountName", "");
            list.Add("AccountKey", "");
            list.Add("RegionName", "us-west-2");
            list.Add("Timeout", "0");
            list.Add("SortOrder", "Desc");
            list.Add("Count", "");
            list.Add("CounterName", "");
            list.Add("etl:BatchId", "1006");
            list.Add("etl:StepId", "11");
            list.Add("@RunId", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;

            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #3
0
        public void AzureFileDownload_Ok()
        {
            AzureFileDownloadActivity   activity = new AzureFileDownloadActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.; Database=etl_Controller; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("Prefix", @"");
            list.Add("Modified", @"");
            list.Add("OutputFolder", "C:\\Builds\\ZipFiles");
            list.Add("FileShare", "xxx");
            list.Add("AccountKey", "key");
            list.Add("AccountName", "name");
            list.Add("isSasToken", "true");
            list.Add("Timeout", "0");
            list.Add("SortOrder", "Desc");
            list.Add("Count", "");
            list.Add("CounterName", "");
            list.Add("etl:BatchId", "1006");
            list.Add("etl:StepId", "11");
            list.Add("@RunId", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;

            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #4
0
        public virtual WfResult Run(CancellationToken token)
        {
            WfResult result = WfResult.Unknown;

            //_logger.Write(String.Format("SqlServer: {0} query: {1}", _attributes[CONNECTION_STRING], _attributes[QUERY_STRING]));

            try
            {
                WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();
                attributes.Add(PROCESSOR_NAME, _attributes[PROCESSOR_NAME]);
                attributes.Add(PROCESSOR_MODE_DEBUG, _attributes[PROCESSOR_MODE_DEBUG]);
                attributes.Add(PROCESSOR_MODE_VERBOSE, _attributes[PROCESSOR_MODE_VERBOSE]);
                attributes.Add(PROCESSOR_MODE_FORCESTART, _attributes[PROCESSOR_MODE_FORCESTART]);
                attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, _attributes[WORKFLOW_NAME]);
                attributes.Add(CONNECTION_STRING, _attributes[CONNECTION_STRING]);
                attributes.Add(REQUEST_ID, _attributes[REQUEST_ID]);

                WorkflowProcessor wfp = new WorkflowProcessor(attributes);
                result = wfp.Run(token);
                _logger.Information("Activity finished with result {WfStatus}: {Message}", result.StatusCode, result.Message);
            }
            catch (SqlException ex)
            {
                throw ex;
                //_logger.Write(String.Format("SqlServer exception: {0}", ex.Message));
                //result = WfResult.Create(WfStatus.Failed, ex.Message, ex.ErrorCode);
            }
            return(result);
        }
コード例 #5
0
        public void WFRun_Ok()
        {
            //var builder = new ConfigurationBuilder()
            //    .AddJsonFile("appsettings.json")
            //    .Build();

            //string runnerName = builder.GetSection("Data:Runner").Value;
            //string connectionString = builder.GetSection("Data:Controller").Value;

            var    settings         = ConfigurationManager.AppSettings;
            string runnerName       = settings["Runner"];
            string connectionString = settings["Controller"];


            Log.Logger = new LoggerConfiguration()
                         .ReadFrom.AppSettings()
                         //.ReadFrom.Configuration(builder)
                         .MinimumLevel.Debug()
                         //.WriteTo.File(path: @"c:\logs\log.txt",
                         //             rollOnFileSizeLimit: true,
                         //             rollingInterval: RollingInterval.Hour,
                         //             fileSizeLimitBytes: 1024000
                         //            )
                         .WriteTo.WorkflowLogger(connectionString: connectionString)
                         .CreateLogger();

            string WFName = "test107";

            WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();

            //attributes.Add(WorkflowConstants.ATTRIBUTE_PROCESSOR_NAME, runnerName);
            attributes.Add(WorkflowConstants.ATTRIBUTE_DEBUG, "false");
            attributes.Add(WorkflowConstants.ATTRIBUTE_VERBOSE, "true");
            attributes.Add(WorkflowConstants.ATTRIBUTE_FORCESTART, "true");
            attributes.Add(WorkflowConstants.ATTRIBUTE_CONTROLLER_CONNECTIONSTRING, connectionString);
            attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, WFName);
            //attributes.Add(WorkflowConstants.ATTRIBUTE_REQUEST_ID, Guid.NewGuid().ToString());


            WfResult wr = WfResult.Unknown;

            using (CancellationTokenSource cts = new CancellationTokenSource())
            {
                WorkflowProcessor wfp = new WorkflowProcessor(attributes);
                wr = wfp.Run(cts.Token);
            }

            Assert.IsTrue(wr.StatusCode == WfStatus.Succeeded);
        }
コード例 #6
0
        public WfResult Start(WorkflowAttributeCollection args, CancellationToken token)
        {
            WfResult result = WfResult.Succeeded;

            _args = args;

            Parameters parameters = new Parameters();

            try
            {
                Version v = System.Reflection.Assembly.GetExecutingAssembly().GetName().Version;

                string inputXml;
                if (_args.ContainsKey(XML))
                {
                    inputXml = _args[XML];
                }
                else
                {
                    inputXml = DeParameters();
                }

                if (!inputXml.StartsWith("<?xml version="))
                {
                    byte[] base64ByteArr = Convert.FromBase64String(XML);
                    inputXml = System.Text.UnicodeEncoding.Unicode.GetString(base64ByteArr);
                }

                parameters = Parameters.DeSerializefromXml(inputXml);

                _logger.Information("Running DE v.{Version} ({Bit} bit)", v.ToString(), 8 * IntPtr.Size);
                _logger.Information("Executing as: {User}", WindowsIdentity.GetCurrent().Name.ToString());
                _logger.Debug("DE input parameter: {DeXml}", inputXml);

                //logger.WriteDebug("DE XML: " + inputXml);

                DEController controller = new DEController(_logger);
                controller.Execute(parameters, token);
            }
            catch (Exception ex)
            {
                _logger.Error(ex, "Exception: {Message}", ex.Message);
                result = WfResult.Failed;
            }

            return(result);
        }
コード例 #7
0
        public void BsonConvert_Ok()
        {
            BsonConverterActivity       activity = new BsonConverterActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\*.bson");
            list.Add("OutputFolder", "c:\\Builds\\JsonFiles\\10-19-2016\\edxapp");
            list.Add("Timeout", "30");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #8
0
        public void EventCheck_Ok()
        {
            CheckWorkflowEventActivity  activity = new CheckWorkflowEventActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("EventType", @"Process2_FINISHED");
            list.Add("WatermarkEventType", @"Process1_FINISHED");
            list.Add("Timeout", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;

            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #9
0
        public void BsonSqlLoader_Ok()
        {
            BsonSqlLoaderActivity       activity = new BsonSqlLoaderActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
            list.Add("InputFile", "C:\\Builds\\UnzipFiles\\mongobackup_10-19-2016-230145\\edxapp\\fs.files.bson");
            list.Add("TableName", "dbo.staging_bson_test");
            list.Add("Timeout", "600");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #10
0
        /// <summary>
        /// Entry point to Delta Extractor. Takes parameters defined by an XML string based
        /// on the parameters.XSD schema.
        /// </summary>
        /// <param name="args"></param>
        /// <returns></returns>
        public static void Main(string[] args)
        {
            if (args.Length == 0 ||
                args.Contains(@"/help", StringComparer.InvariantCultureIgnoreCase))
            {
                help();
                Environment.Exit(0);
            }

            bool debug   = args.Contains(@"/D", StringComparer.InvariantCultureIgnoreCase);
            bool verbose = args.Contains(@"/V", StringComparer.InvariantCultureIgnoreCase);


            try
            {
                var     minLogLevel = (debug) ? LogEventLevel.Debug : LogEventLevel.Information;
                ILogger logger      = new LoggerConfiguration()
                                      .MinimumLevel.Is(minLogLevel)
                                      .WriteTo.Console()
                                      .CreateLogger();


                ;
                WorkflowAttributeCollection param = new WorkflowAttributeCollection();
                param.Add("XML", args[0]);

                using (CancellationTokenSource cts = new CancellationTokenSource())
                {
                    DERun    runner = new DERun(logger);
                    WfResult result = runner.Start(param, cts.Token);
                    if (result.StatusCode != WfStatus.Succeeded)
                    {
                        throw new Exception(String.Format("DE returned Status: {0}", result.StatusCode.ToString()));
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                Environment.Exit(1);
            }
            Environment.Exit(0);
        }
コード例 #11
0
        public void DERun_Ok()
        {
            DeltaExtractorActivity      activity = new DeltaExtractorActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("@BatchId", "1006");
            list.Add("@StepId", "11");
            list.Add("@RunId", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #12
0
        public void TGZDecompress_Ok()
        {
            TGZDecompressActivity       activity = new TGZDecompressActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("InputFile", "c:\\Builds\\ZipFiles\\test.tar.gz");
            list.Add("OutputFolder", "c:\\Builds\\UnzipFiles");
            list.Add("Mode", "tgz");
            list.Add("Timeout", "30");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #13
0
        public void FileSetStatus_Ok()
        {
            FileSetProgressStatusActivity activity = new FileSetProgressStatusActivity();
            WorkflowActivityArgs          wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection   list     = new WorkflowAttributeCollection();

            list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("FileId", "5");
            list.Add("FileStatus", "Completed");
            list.Add("Timeout", "30");
            list.Add("etl:RunId", "1");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #14
0
        public void EventPost_Ok()
        {
            PostWorkflowEventActivity   activity = new PostWorkflowEventActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.; Database=etl_event; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("EventType", @"Process2_FINISHED");
            list.Add("EventPostDate", @"2016-12-16 18:04");
            //list.Add("EventArgs", "<dwc:EventArgs xmlns:dwc=\"EventArgs.XSD\" Source=\"Process2 Finished\" PeriodGrain=\"Week\" Period=\"201652\" />");
            list.Add("EventArgs", "");
            list.Add("Timeout", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;

            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #15
0
        public void FileRegister_Ok()
        {
            FileRegisterActivity        activity = new FileRegisterActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection=True; Connection Timeout=120; ");
            list.Add("RegisterPath", "c:\\Builds\\FlatFiles\\*.txt");
            list.Add("SourceName", "testFiles");
            list.Add("ProcessPriority", "1");
            list.Add("Timeout", "30");
            list.Add("@runId", "1");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #16
0
        public void ExecuteSqlQuery_Ok()
        {
            SqlServerActivity           activity = new SqlServerActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            //list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("ConnectionString", @"Server=.; Database=etl_staging; Trusted_Connection = False;User Id=test;Password=test; Connection Timeout = 120; ");
            list.Add("Query", @"
print 'this is print test';
raiserror ('this is err test',11,11);
");
            list.Add("Timeout", "0");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;

            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #17
0
        public void FileDequeue_Ok()
        {
            FileGetProcessListActivity  activity = new FileGetProcessListActivity();
            WorkflowActivityArgs        wfa      = new WorkflowActivityArgs();
            WorkflowAttributeCollection list     = new WorkflowAttributeCollection();

            list.Add("ConnectionString", @"Server=.\sql14; Database=etl_controller; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("RegisterConnectionString", @"Server=.\sql14; Database=etl_staging; Trusted_Connection = True; Connection Timeout = 120; ");
            list.Add("SourceName", "testFiles");
            list.Add("Timeout", "30");
            list.Add("etl:RunId", "1");
            list.Add("etl:BatchId", "101");
            list.Add("etl:StepId", "1");
            wfa.RequiredAttributes = list;
            wfa.Logger             = _logger;


            activity.Configure(wfa);
            WfResult result = activity.Run(CancellationToken.None);

            Assert.IsTrue(result.StatusCode == WfStatus.Succeeded);
        }
コード例 #18
0
        static int Main(string[] args)
        {
            if (args.Length == 0 ||
                args.Contains(@"/help", StringComparer.InvariantCultureIgnoreCase))
            {
                help();
                return(0);
            }

            WorkflowAttributeCollection attributes = new WorkflowAttributeCollection();

            attributes.Add(WorkflowConstants.ATTRIBUTE_WORKFLOW_NAME, args[0].Replace("\"", ""));
            attributes.Add(WorkflowConstants.ATTRIBUTE_DEBUG, "false");
            attributes.Add(WorkflowConstants.ATTRIBUTE_VERBOSE, "false");
            attributes.Add(WorkflowConstants.ATTRIBUTE_FORCESTART, "false");

            var minLogLevel = LogEventLevel.Information;

            //bool debug = false;
            if (args.Contains(@"/D", StringComparer.InvariantCultureIgnoreCase))
            {
                attributes[WorkflowConstants.ATTRIBUTE_DEBUG] = "true";
                //debug = true;
                minLogLevel = LogEventLevel.Debug;
            }

            //bool forcestart = false;
            if (args.Contains(@"/R", StringComparer.InvariantCultureIgnoreCase))
            {
                attributes[WorkflowConstants.ATTRIBUTE_FORCESTART] = "true";
                //forcestart = true;
            }

            bool verbose = false;

            if (args.Contains(@"/V", StringComparer.InvariantCultureIgnoreCase))
            {
                attributes[WorkflowConstants.ATTRIBUTE_VERBOSE] = "true";
                verbose = true;
            }


            //var builder = new ConfigurationBuilder()
            //    .AddJsonFile("appsettings.json")
            //    .Build();

            //string runnerName = builder.GetSection("Data:Runner").Value;
            //string connectionString = builder.GetSection("Data:Controller").Value;

            var    settings         = ConfigurationManager.AppSettings;
            string runnerName       = settings["Runner"];
            string connectionString = settings["Controller"];

            attributes.Add(WorkflowConstants.ATTRIBUTE_CONTROLLER_CONNECTIONSTRING, connectionString);


            //string runnerName = settings["Runner"];
            if (String.IsNullOrEmpty(runnerName))
            {
                runnerName = "Default";
            }

            attributes.Add(WorkflowConstants.ATTRIBUTE_PROCESSOR_NAME, runnerName);


            if (verbose)
            {
                Log.Logger = new LoggerConfiguration()
                             //.ReadFrom.Configuration(builder)
                             .ReadFrom.AppSettings()
                             .MinimumLevel.Is(minLogLevel)
                             //.WriteTo.Console()
                             .WriteTo.WorkflowLogger(connectionString: connectionString)
                             .CreateLogger();
            }
            else
            {
                Log.Logger = new LoggerConfiguration()
                             .MinimumLevel.Is(minLogLevel)
                             .WriteTo.WorkflowLogger(connectionString: connectionString)
                             .CreateLogger();
            }

            try
            {
                WfResult wr = WfResult.Unknown;
                using (CancellationTokenSource cts = new CancellationTokenSource())
                {
                    WorkflowProcessor wfp = new WorkflowProcessor(attributes);
                    wr = wfp.Run(cts.Token);
                }

                if (wr.StatusCode != WfStatus.Succeeded)
                {
                    return(1);
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(String.Format("Error: {0}, {1}", ex.HResult, ex.Message));
                return(1);
            }
            return(0);
        }