Beispiel #1
0
        public int Execute(ConsoleOptions options)
        {
            this.workDir = options.work;
            if (workDir == null || workDir == string.Empty)
            {
                workDir = Environment.CurrentDirectory;
            }
            else
            {
                workDir = Path.GetFullPath(workDir);
                if (!Directory.Exists(workDir))
                {
                    Directory.CreateDirectory(workDir);
                }
            }

            TextWriter outWriter      = Console.Out;
            bool       redirectOutput = options.output != null && options.output != string.Empty;

            if (redirectOutput)
            {
                StreamWriter outStreamWriter = new StreamWriter(Path.Combine(workDir, options.output));
                outStreamWriter.AutoFlush = true;
                outWriter = outStreamWriter;
            }

            TextWriter errorWriter   = Console.Error;
            bool       redirectError = options.err != null && options.err != string.Empty;

            if (redirectError)
            {
                StreamWriter errorStreamWriter = new StreamWriter(Path.Combine(workDir, options.err));
                errorStreamWriter.AutoFlush = true;
                errorWriter = errorStreamWriter;
            }

            TestPackage package = MakeTestPackage(options);

            ProcessModel processModel = package.Settings.Contains("ProcessModel")
                ? (ProcessModel)package.Settings["ProcessModel"]
                : ProcessModel.Default;

            DomainUsage domainUsage = package.Settings.Contains("DomainUsage")
                ? (DomainUsage)package.Settings["DomainUsage"]
                : DomainUsage.Default;

            RuntimeFramework framework = package.Settings.Contains("RuntimeFramework")
                ? (RuntimeFramework)package.Settings["RuntimeFramework"]
                : RuntimeFramework.CurrentFramework;

#if CLR_2_0 || CLR_4_0
            Console.WriteLine("ProcessModel: {0}    DomainUsage: {1}", processModel, domainUsage);

            Console.WriteLine("Execution Runtime: {0}", framework);
#else
            Console.WriteLine("DomainUsage: {0}", domainUsage);

            if (processModel != ProcessModel.Default && processModel != ProcessModel.Single)
            {
                Console.WriteLine("Warning: Ignoring project setting 'processModel={0}'", processModel);
            }

            if (!RuntimeFramework.CurrentFramework.Supports(framework))
            {
                Console.WriteLine("Warning: Ignoring project setting 'runtimeFramework={0}'", framework);
            }
#endif

            using (TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
            {
                testRunner.Load(package);

                if (testRunner.Test == null)
                {
                    testRunner.Unload();
                    Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
                    return(FIXTURE_NOT_FOUND);
                }

                EventCollector collector = new EventCollector(options, outWriter, errorWriter);

                TestFilter       testFilter = TestFilter.Empty;
                SimpleNameFilter nameFilter = new SimpleNameFilter();

                if (options.run != null && options.run != string.Empty)
                {
                    Console.WriteLine("Selected test(s): " + options.run);
                    foreach (string name in TestNameParser.Parse(options.run))
                    {
                        nameFilter.Add(name);
                    }
                    testFilter = nameFilter;
                }

                if (options.runlist != null && options.runlist != string.Empty)
                {
                    Console.WriteLine("Run list: " + options.runlist);
                    using (StreamReader rdr = new StreamReader(options.runlist))
                    {
                        // NOTE: We can't use rdr.EndOfStream because it's
                        // not present in .NET 1.x.
                        string line = rdr.ReadLine();
                        while (line != null)
                        {
                            if (line[0] != '#')
                            {
                                nameFilter.Add(line);
                            }
                            line = rdr.ReadLine();
                        }
                    }
                    testFilter = nameFilter;
                }

                if (options.include != null && options.include != string.Empty)
                {
                    TestFilter includeFilter = new CategoryExpression(options.include).Filter;
                    Console.WriteLine("Included categories: " + includeFilter.ToString());

                    if (testFilter.IsEmpty)
                    {
                        testFilter = includeFilter;
                    }
                    else
                    {
                        testFilter = new AndFilter(testFilter, includeFilter);
                    }
                }

                if (options.exclude != null && options.exclude != string.Empty)
                {
                    TestFilter excludeFilter = new NotFilter(new CategoryExpression(options.exclude).Filter);
                    Console.WriteLine("Excluded categories: " + excludeFilter.ToString());

                    if (testFilter.IsEmpty)
                    {
                        testFilter = excludeFilter;
                    }
                    else if (testFilter is AndFilter)
                    {
                        ((AndFilter)testFilter).Add(excludeFilter);
                    }
                    else
                    {
                        testFilter = new AndFilter(testFilter, excludeFilter);
                    }
                }

                if (testFilter is NotFilter)
                {
                    ((NotFilter)testFilter).TopLevel = true;
                }

                TestResult result         = null;
                string     savedDirectory = Environment.CurrentDirectory;
                TextWriter savedOut       = Console.Out;
                TextWriter savedError     = Console.Error;

                try
                {
                    result = testRunner.Run(collector, testFilter, false, LoggingThreshold.Off);
                }
                finally
                {
                    outWriter.Flush();
                    errorWriter.Flush();

                    if (redirectOutput)
                    {
                        outWriter.Close();
                    }
                    if (redirectError)
                    {
                        errorWriter.Close();
                    }

                    Environment.CurrentDirectory = savedDirectory;
                    Console.SetOut(savedOut);
                    Console.SetError(savedError);
                }

                Console.WriteLine();

                int returnCode = UNEXPECTED_ERROR;

                if (result != null)
                {
                    string           xmlOutput = CreateXmlOutput(result);
                    ResultSummarizer summary   = new ResultSummarizer(result);

                    if (options.xmlConsole)
                    {
                        Console.WriteLine(xmlOutput);
                    }
                    else
                    {
                        WriteSummaryReport(summary);
                        if (summary.ErrorsAndFailures > 0 || result.IsError || result.IsFailure)
                        {
                            if (options.stoponerror)
                            {
                                Console.WriteLine("Test run was stopped after first error, as requested.");
                                Console.WriteLine();
                            }

                            WriteErrorsAndFailuresReport(result);
                        }
                        if (summary.TestsNotRun > 0)
                        {
                            WriteNotRunReport(result);
                        }

                        if (!options.noresult)
                        {
                            // Write xml output here
                            string xmlResultFile = options.result == null || options.result == string.Empty
                                ? "TestResult.xml" : options.result;

                            using (StreamWriter writer = new StreamWriter(Path.Combine(workDir, xmlResultFile)))
                            {
                                writer.Write(xmlOutput);
                            }
                        }
                    }

                    returnCode = summary.ErrorsAndFailures;
                }

                if (collector.HasExceptions)
                {
                    collector.WriteExceptions();
                    returnCode = UNEXPECTED_ERROR;
                }

                return(returnCode);
            }
        }
        public int Execute(ExtendedConsoleOptions options)
        {
            TextWriter outWriter      = Console.Out;
            bool       redirectOutput = options.output != null && options.output != string.Empty;

            if (redirectOutput)
            {
                StreamWriter outStreamWriter = new StreamWriter(options.output);
                outStreamWriter.AutoFlush = true;
                outWriter = outStreamWriter;
            }

            TextWriter errorWriter   = Console.Error;
            bool       redirectError = options.err != null && options.err != string.Empty;

            if (redirectError)
            {
                StreamWriter errorStreamWriter = new StreamWriter(options.err);
                errorStreamWriter.AutoFlush = true;
                errorWriter = errorStreamWriter;
            }

            TextWriter testResultWriter = null;

            if (options.IsResults)
            {
                testResultWriter = new StreamWriter(options.results, false, Encoding.UTF8);
                ((StreamWriter)testResultWriter).AutoFlush = true;
            }

            TestPackage package = MakeTestPackage(options);

            Console.WriteLine("ProcessModel: {0}    DomainUsage: {1}",
                              package.Settings.Contains("ProcessModel")
                    ? package.Settings["ProcessModel"]
                    : "Default",
                              package.Settings.Contains("DomainUsage")
                    ? package.Settings["DomainUsage"]
                    : "Default");

            Console.WriteLine("Execution Runtime: {0}",
                              package.Settings.Contains("RuntimeFramework")
                    ? package.Settings["RuntimeFramework"]
                    : "Default");

            using (TestRunner testRunner = new DefaultTestRunnerFactory().MakeTestRunner(package))
            {
                testRunner.Load(package);

                if (testRunner.Test == null)
                {
                    testRunner.Unload();
                    Console.Error.WriteLine("Unable to locate fixture {0}", options.fixture);
                    return(FIXTURE_NOT_FOUND);
                }

                ExtendedEventCollector collector = new ExtendedEventCollector(options, outWriter, errorWriter, testResultWriter);

                TestFilter testFilter = TestFilter.Empty;
                if (options.run != null && options.run != string.Empty)
                {
                    Console.WriteLine("Selected test(s): " + options.run);
                    testFilter = new SimpleNameFilter(TestNameParser.Parse(options.run));
                }

                if (options.include != null && options.include != string.Empty)
                {
                    Console.WriteLine("Included categories: " + options.include);
                    TestFilter includeFilter = new CategoryExpression(options.include).Filter;
                    if (testFilter.IsEmpty)
                    {
                        testFilter = includeFilter;
                    }
                    else
                    {
                        testFilter = new AndFilter(testFilter, includeFilter);
                    }
                }

                if (options.exclude != null && options.exclude != string.Empty)
                {
                    Console.WriteLine("Excluded categories: " + options.exclude);
                    TestFilter excludeFilter = new NotFilter(new CategoryExpression(options.exclude).Filter);
                    if (testFilter.IsEmpty)
                    {
                        testFilter = excludeFilter;
                    }
                    else if (testFilter is AndFilter)
                    {
                        ((AndFilter)testFilter).Add(excludeFilter);
                    }
                    else
                    {
                        testFilter = new AndFilter(testFilter, excludeFilter);
                    }
                }

                if (testFilter is NotFilter)
                {
                    ((NotFilter)testFilter).TopLevel = true;
                }

                TestResult result         = null;
                string     savedDirectory = Environment.CurrentDirectory;
                TextWriter savedOut       = Console.Out;
                TextWriter savedError     = Console.Error;

                try
                {
                    result = testRunner.Run(collector, testFilter);
                }
                finally
                {
                    outWriter.Flush();
                    errorWriter.Flush();

                    if (redirectOutput)
                    {
                        outWriter.Close();
                    }
                    if (redirectError)
                    {
                        errorWriter.Close();
                    }
                    if (options.IsResults)
                    {
                        testResultWriter.Close();
                    }

                    Environment.CurrentDirectory = savedDirectory;
                    Console.SetOut(savedOut);
                    Console.SetError(savedError);
                }

                Console.WriteLine();

                int returnCode = UNEXPECTED_ERROR;

                if (result != null)
                {
                    string           xmlOutput = CreateXmlOutput(result);
                    ResultSummarizer summary   = new ResultSummarizer(result);

                    if (options.xmlConsole)
                    {
                        Console.WriteLine(xmlOutput);
                    }
                    else
                    {
                        WriteSummaryReport(summary);
                        if (summary.ErrorsAndFailures > 0 || result.IsError || result.IsFailure)
                        {
                            WriteErrorsAndFailuresReport(result);
                        }
                        if (summary.TestsNotRun > 0)
                        {
                            WriteNotRunReport(result);
                        }
                    }

                    // Write xml output here
                    string xmlResultFile = options.xml == null || options.xml == string.Empty
                        ? "TestResult.xml" : options.xml;

                    if (!String.IsNullOrEmpty(options.xml))
                    {
                        using (StreamWriter writer = new StreamWriter(xmlResultFile))
                        {
                            writer.Write(xmlOutput);
                        }
                    }
                    returnCode = summary.ErrorsAndFailures;
                }

                if (collector.HasExceptions)
                {
                    collector.WriteExceptions();
                    returnCode = UNEXPECTED_ERROR;
                }

                return(returnCode);
            }
        }
        internal static bool CreateTestFilter(ConsoleOptions options, out TestFilter testFilter)
        {
            testFilter = TestFilter.Empty;

            SimpleNameFilter nameFilter = new SimpleNameFilter();

            if (options.run != null && options.run != string.Empty)
            {
                Console.WriteLine("Selected test(s): " + options.run);

                foreach (string name in TestNameParser.Parse(options.run))
                {
                    nameFilter.Add(name);
                }

                testFilter = nameFilter;
            }

            if (options.runlist != null && options.runlist != string.Empty)
            {
                Console.WriteLine("Run list: " + options.runlist);

                try
                {
                    using (StreamReader rdr = new StreamReader(options.runlist))
                    {
                        // NOTE: We can't use rdr.EndOfStream because it's
                        // not present in .NET 1.x.
                        string line = rdr.ReadLine();
                        while (line != null && line.Length > 0)
                        {
                            if (line[0] != '#')
                            {
                                nameFilter.Add(line);
                            }
                            line = rdr.ReadLine();
                        }
                    }
                }
                catch (Exception e)
                {
                    if (e is FileNotFoundException || e is DirectoryNotFoundException)
                    {
                        Console.WriteLine("Unable to locate file: " + options.runlist);
                        return(false);
                    }
                    throw;
                }

                testFilter = nameFilter;
            }

            if (options.include != null && options.include != string.Empty)
            {
                TestFilter includeFilter = new CategoryExpression(options.include).Filter;
                Console.WriteLine("Included categories: " + includeFilter.ToString());

                if (testFilter.IsEmpty)
                {
                    testFilter = includeFilter;
                }
                else
                {
                    testFilter = new AndFilter(testFilter, includeFilter);
                }
            }

            if (options.exclude != null && options.exclude != string.Empty)
            {
                TestFilter excludeFilter = new NotFilter(new CategoryExpression(options.exclude).Filter);
                Console.WriteLine("Excluded categories: " + excludeFilter.ToString());

                if (testFilter.IsEmpty)
                {
                    testFilter = excludeFilter;
                }
                else if (testFilter is AndFilter)
                {
                    ((AndFilter)testFilter).Add(excludeFilter);
                }
                else
                {
                    testFilter = new AndFilter(testFilter, excludeFilter);
                }
            }

            if (testFilter is NotFilter)
            {
                ((NotFilter)testFilter).TopLevel = true;
            }

            return(true);
        }
Beispiel #4
0
        public ConsoleOptions(params string[] args)
        {
            // NOTE: The order in which patterns are added
            // determines the display order for the help.

            // Old Options no longer supported:
            //   fixture
            //   xmlConsole
            //   noshadow
            //   nothread
            //   nodots

            // Options to be added:
            //   teamcity
            //   workers

            // Select Tests
            this.Add("test=", "Comma-separated list of {NAMES} of tests to run or explore. This option may be repeated.",
                     v => ((List <string>)TestList).AddRange(TestNameParser.Parse(RequiredValue(v, "--test"))));

            this.Add("include=", "Test {CATEGORIES} to be included. May be a single category, a comma-separated list of categories or a category expression.",
                     v => Include = RequiredValue(v, "--include"));

            this.Add("exclude=", "Test {CATEGORIES} to be excluded. May be a single category, a comma-separated list of categories or a category expression.",
                     v => Exclude = RequiredValue(v, "--exclude"));

            this.Add("config=", "{NAME} of a project configuration to load (e.g.: Debug).",
                     v => ActiveConfig = RequiredValue(v, "--config"));

            // Where to Run Tests
            this.Add("process=", "{PROCESS} isolation for test assemblies.\nValues: Single, Separate, Multiple",
                     v => ProcessModel = RequiredValue(v, "--process", "Single", "Separate", "Multiple"));

            this.Add("domain=", "{DOMAIN} isolation for test assemblies.\nValues: None, Single, Multiple",
                     v => DomainUsage = RequiredValue(v, "--domain", "None", "Single", "Multiple"));

            // How to Run Tests
            this.Add("framework=", "{FRAMEWORK} type/version to use for tests.\nExamples: mono, net-3.5, v4.0, 2.0, mono-4.0",
                     v => Framework = RequiredValue(v, "--framework"));

            this.Add("timeout=", "Set timeout for each test case in {MILLISECONDS}.",
                     v => defaultTimeout = RequiredInt(v, "--timeout"));

            this.Add("seed=", "Set the random {SEED} used to generate test cases.",
                     v => randomSeed = RequiredInt(v, "--seed"));

            this.Add("workers=", "Specify the {NUMBER} of worker threads to be used in running tests.",
                     v => numWorkers = RequiredInt(v, "--workers"));

            this.Add("stoponerror", "Stop run immediately upon any test failure or error.",
                     v => StopOnError = v != null);

            this.Add("wait", "Wait for input before closing console window.",
                     v => WaitBeforeExit = v != null);

            this.Add("pause", "Pause before run to allow debugging.",
                     v => PauseBeforeRun = v != null);

            // Output Control
            this.Add("work=", "{PATH} of the directory to use for output files.",
                     v => WorkDirectory = RequiredValue(v, "--work"));

            this.Add("output|out=", "File {PATH} to contain text output from the tests.",
                     v => OutFile = RequiredValue(v, "--output"));

            this.Add("err=", "File {PATH} to contain error output from the tests.",
                     v => ErrFile = RequiredValue(v, "--err"));

            this.Add("result=", "An output {SPEC} for saving the test results.\nThis option may be repeated.",
                     v => resultOutputSpecifications.Add(new OutputSpecification(RequiredValue(v, "--resultxml"))));

            this.Add("explore:", "Display or save test info rather than running tests. Optionally provide an output {SPEC} for saving the test info. This option may be repeated.", v =>
            {
                Explore = true;
                if (v != null)
                {
                    ExploreOutputSpecifications.Add(new OutputSpecification(v));
                }
            });

            this.Add("noresult", "Don't save any test results.",
                     v => noresult = v != null);

            this.Add("labels=", "Specify whether to write test case names to the output. Values: Off, On, All",
                     v => DisplayTestLabels = RequiredValue(v, "--labels", "Off", "On", "All"));

            this.Add("trace=", "Set internal trace {LEVEL}.\nValues: Off, Error, Warning, Info, Verbose (Debug)",
                     v => InternalTraceLevel = RequiredValue(v, "--trace", "Off", "Error", "Warning", "Info", "Verbose", "Debug"));

            this.Add("noheader|noh", "Suppress display of program information at start of run.",
                     v => NoHeader = v != null);

            this.Add("color|c", "Displays console output in color.",
                     v => Color = v != null);

            this.Add("verbose|v", "Display additional information as the test runs.",
                     v => Verbose = v != null);

            this.Add("help|h", "Display this message and exit.",
                     v => ShowHelp = v != null);

            // Default
            this.Add("<>", v =>
            {
                if (v.StartsWith("-") || v.StartsWith("/") && Path.DirectorySeparatorChar != '/')
                {
                    ErrorMessages.Add("Invalid argument: " + v);
                }
                else
                {
                    InputFiles.Add(v);
                }
            });

            if (args != null)
            {
                this.Parse(args);
            }
        }