Esempio n. 1
0
        /// <summary>
        /// Run the unit tests.
        /// </summary>
        public async void RunAsync()
        {
            // Ensure there's an interface to display the test results.
            if (this.Reporter == null)
            {
                throw new ArgumentNullException("Reporter");
            }

            // Setup the progress/failure counters
            this.Progress = 0;
            this.Failures = 0;

            // Filter out any test methods based on the current settings
            int filteredTestCount = FilterTests();

            // Write out the test status message which may be modified by the
            // filters
            Reporter.Status(this.Settings.TestRunStatusMessage);

            // Enumerators that track the current group and method to execute
            // (which allows reentrancy into the test loop below to resume at
            // the correct location).
            IEnumerator <TestGroup>  groups  = this.Groups.OrderBy(g => g.Name).GetEnumerator();
            IEnumerator <TestMethod> methods = null;

            // Keep a reference to the current group so we can pass it to the
            // Reporter's EndGroup (we don't need one for the test method,
            // however, because we close over in the continuation).
            TestGroup currentGroup = null;

            // Setup the UI
            this.Reporter.StartRun(this);

            // The primary test loop is a "recursive" closure that will pass
            // itself as the continuation to async tests.
            //
            // Note: It's really important for performance to note that any
            // calls to testLoop only occur in the tail position.
            DateTime    RunStartTime = DateTime.UtcNow;
            Func <Task> testLoop     = null;

            testLoop =
                async() =>
            {
                if (methods != null && methods.MoveNext())
                {
                    // If we were in the middle of a test group and there
                    // are more methods to execute, let's move to the next
                    // test method.

                    // Update the progress
                    this.Progress++;
                    Reporter.Progress(this);
                    // Start the test method
                    Reporter.StartTest(methods.Current);
                    if (methods.Current.Excluded)
                    {
                        // Ignore excluded tests and immediately recurse.
                        Reporter.EndTest(methods.Current);
                        await testLoop();
                    }
                    else
                    {
                        // Get the start time for individual tests
                        DateTime testStartTime = DateTime.UtcNow;

                        // Record the test result, upload the test log as a blob and clear the
                        // log for next test
                        Func <Task> recordTestResult = async() =>
                        {
                            if (!Settings.ManualMode)
                            {
                                // upload test log to sunlight blob container
                                string relativeFilePath = this.Platform + "/" + Guid.NewGuid().ToString() + ".txt";

                                string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
                                                                                this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
                                                                                relativeFilePath);

                                await UploadToBlobContainerAsync(blobStorageSasUrl, LogDump.ToString());

                                // record the test result
                                var testResult = new TestResult()
                                {
                                    FullName     = methods.Current.Name,
                                    StartTime    = testStartTime,
                                    EndTime      = DateTime.UtcNow,
                                    Outcome      = methods.Current.Passed ? "Passed" : "Failed",
                                    Source       = currentGroup.Name,
                                    ReferenceUrl = relativeFilePath
                                };

                                // Add the test result to the test result collection
                                testRun.AddTestResult(testResult);
                                LogDump.Clear();
                            }
                        };
                        // Execute the test method
                        methods.Current.Test.Start(
                            new ActionContinuation
                        {
                            OnSuccess = async() =>
                            {
                                // Mark the test as passing, update the
                                // UI, and continue with the next test.
                                methods.Current.Passed = true;
                                methods.Current.Test   = null;
                                Reporter.EndTest(methods.Current);
                                await recordTestResult();
                                await testLoop();
                            },
                            OnError = async(message) =>
                            {
                                // Mark the test as failing, update the
                                // UI, and continue with the next test.
                                methods.Current.Passed    = false;
                                methods.Current.Test      = null;
                                methods.Current.ErrorInfo = message;
                                this.Failures++;
                                System.Diagnostics.Debug.WriteLine(message);
                                Reporter.Error(message);
                                LogDump.AppendLine(message);
                                Reporter.EndTest(methods.Current);
                                await recordTestResult();
                                await testLoop();
                            }
                        });
                    }
                }
                else if (groups.MoveNext())
                {
                    // If we've finished a test group and there are more,
                    // then move to the next one.

                    // Finish the UI for the last group.
                    if (currentGroup != null)
                    {
                        Reporter.EndGroup(currentGroup);
                        currentGroup = null;
                    }

                    // Setup the UI for this next group
                    currentGroup = groups.Current;
                    Reporter.StartGroup(currentGroup);

                    // Get the methods and immediately recurse which will
                    // start executing them.
                    methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
                    await testLoop();
                }
                else
                {
                    if (!Settings.ManualMode)
                    {
                        // upload test suite result to sunlight blob container
                        string blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
                                                                        this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
                                                                        this.Platform + "-detail.json");
                        string fileContent = JsonConvert.SerializeObject(testRun.TestResults.ToList(), Formatting.Indented);
                        await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);

                        // upload test result summary to blob container
                        var masterResult = new MasterTestResult()
                        {
                            FullName     = this.Platform + "-" + Settings.Custom["RuntimeVersion"],
                            Outcome      = Failures > 0 ? "Failed" : "Passed",
                            TotalCount   = testRun.TestCount,
                            Passed       = filteredTestCount - Failures,
                            Failed       = Failures,
                            Skipped      = testRun.TestCount - filteredTestCount,
                            StartTime    = RunStartTime,
                            EndTime      = DateTime.UtcNow,
                            ReferenceUrl = this.Platform + "-detail.json"
                        };

                        // upload test suite result to sunlight blob container
                        blobStorageSasUrl = GetBlobStorageSasUrl(this.Settings.Custom["TestFrameworkStorageContainerUrl"],
                                                                 this.Settings.Custom["TestFrameworkStorageContainerSasToken"],
                                                                 this.Platform + "-master.json");
                        fileContent = JsonConvert.SerializeObject(masterResult, Formatting.Indented);
                        await UploadToBlobContainerAsync(blobStorageSasUrl, fileContent);
                    }
                    // Otherwise if we've finished the entire test run

                    // Finish the UI for the last group and update the
                    // progress after the very last test method.
                    Reporter.EndGroup(currentGroup);
                    Reporter.Progress(this);

                    // Finish the UI for the test run.
                    Reporter.EndRun(this);
                }
            };

            // Start running the tests
            await testLoop();
        }
        /// <summary>
        /// Run the unit tests.
        /// </summary>
        public async void RunAsync()
        {
            // Ensure there's an interface to display the test results.
            if (this.Reporter == null)
            {
                throw new ArgumentNullException("Reporter");
            }

            /// Record the test metadata to the test run object at the start of the test run
            FillTestRunMetaData();

            // Setup the progress/failure counters
            this.Progress = 0;
            this.Failures = 0;

            // Filter out any test methods based on the current settings
            int filteredTestCount = FilterTests();

            // get the actual count of tests going to run
            testRun.TestCount = filteredTestCount;

            // Write out the test status message which may be modified by the
            // filters
            Reporter.Status(this.Settings.TestRunStatusMessage);

            // Enumerators that track the current group and method to execute
            // (which allows reentrancy into the test loop below to resume at
            // the correct location).
            IEnumerator <TestGroup>  groups  = this.Groups.OrderBy(g => g.Name).GetEnumerator();
            IEnumerator <TestMethod> methods = null;

            // Keep a reference to the current group so we can pass it to the
            // Reporter's EndGroup (we don't need one for the test method,
            // however, because we close over in the continuation).
            TestGroup currentGroup = null;

            // Create daylight test run and get the run id
            string runId = String.Empty;

            if (!Settings.ManualMode)
            {
                runId = await TestLogger.CreateDaylightRun(testRun, Settings);
            }

            // Setup the UI
            this.Reporter.StartRun(this);

            // The primary test loop is a "recursive" closure that will pass
            // itself as the continuation to async tests.
            //
            // Note: It's really important for performance to note that any
            // calls to testLoop only occur in the tail position.
            JObject     sasResponseObject = null;
            DateTime    RunStartTime      = DateTime.UtcNow;
            Func <Task> testLoop          = null;

            // Get the SAS token to upload the test logs to upload test logs to blob store
            if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
            {
                sasResponseObject = await TestLogger.GetSaSToken(Settings);
            }
            testLoop =
                async() =>
            {
                if (methods != null && methods.MoveNext())
                {
                    // If we were in the middle of a test group and there
                    // are more methods to execute, let's move to the next
                    // test method.

                    // Update the progress
                    this.Progress++;
                    Reporter.Progress(this);
                    // Start the test method
                    Reporter.StartTest(methods.Current);
                    if (methods.Current.Excluded)
                    {
                        // Ignore excluded tests and immediately recurse.
                        Reporter.EndTest(methods.Current);
                        await testLoop();
                    }
                    else
                    {
                        // Get the start time for individual tests
                        DateTime testStartTime = DateTime.UtcNow;

                        // Record the test result, upload the test log as a blob and clear the
                        // log for next test
                        Func <Task> recordTestResult = async() =>
                        {
                            if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
                            {
                                var log = new TestLogs()
                                {
                                    LogLines = LogDump.ToString(),
                                    LogHash  = Guid.NewGuid().ToString()
                                };
                                // record the test result
                                var testResult = new TestResult()
                                {
                                    FullName  = methods.Current.Name,
                                    StartTime = testStartTime,
                                    EndTime   = DateTime.UtcNow,
                                    Outcome   = methods.Current.Passed ? "Passed" : "Failed",
                                    RunId     = runId,
                                    Tags      = new List <string> {
                                        Platform
                                    },
                                    Source = currentGroup.Name,
                                    Logs   = sasResponseObject != null ? log : null
                                };

                                // upload test log to the blob store
                                if (sasResponseObject != null)
                                {
                                    await TestLogger.UploadTestLog(log, sasResponseObject);
                                }

                                // Add the test result to the test result collection, which will eventually
                                // be uploaded to daylight
                                testRun.AddTestResult(testResult);
                                LogDump.Clear();
                            }
                        };
                        // Execute the test method
                        methods.Current.Test.Start(
                            new ActionContinuation
                        {
                            OnSuccess = async() =>
                            {
                                // Mark the test as passing, update the
                                // UI, and continue with the next test.
                                methods.Current.Passed = true;
                                methods.Current.Test   = null;
                                Reporter.EndTest(methods.Current);
                                await recordTestResult();
                                await testLoop();
                            },
                            OnError = async(message) =>
                            {
                                // Mark the test as failing, update the
                                // UI, and continue with the next test.
                                methods.Current.Passed    = false;
                                methods.Current.Test      = null;
                                methods.Current.ErrorInfo = message;
                                this.Failures++;
                                System.Diagnostics.Debug.WriteLine(message);
                                Reporter.Error(message);
                                LogDump.AppendLine(message);
                                Reporter.EndTest(methods.Current);
                                await recordTestResult();
                                await testLoop();
                            }
                        });
                    }
                }
                else if (groups.MoveNext())
                {
                    // If we've finished a test group and there are more,
                    // then move to the next one.

                    // Finish the UI for the last group.
                    if (currentGroup != null)
                    {
                        Reporter.EndGroup(currentGroup);
                        currentGroup = null;
                    }

                    // Setup the UI for this next group
                    currentGroup = groups.Current;
                    Reporter.StartGroup(currentGroup);

                    // Get the methods and immediately recurse which will
                    // start executing them.
                    methods = groups.Current.Methods.OrderBy(m => m.Name).GetEnumerator();
                    await testLoop();
                }
                else
                {
                    if (!Settings.ManualMode && !String.IsNullOrEmpty(runId))
                    {
                        // post all the test results to daylight
                        await TestLogger.PostTestResults(testRun.TestResults.ToList(), Settings);

                        // String to store the test result summary of the entire suite
                        StringBuilder resultLog = new StringBuilder("Total Tests:" + filteredTestCount);
                        resultLog.AppendLine();
                        resultLog.AppendLine("Passed Tests:" + (filteredTestCount - Failures).ToString());
                        resultLog.AppendLine("Failed Tests:" + Failures);
                        resultLog.AppendLine("Detailed Results:" + Settings.Custom["DayLightUrl"] + "/" + Settings.Custom["DaylightProject"] + "/runs/" + runId);

                        var logs = new TestLogs()
                        {
                            LogLines = resultLog.ToString(),
                            LogHash  = Guid.NewGuid().ToString()
                        };
                        // Record the the result of the entire test suite to master test run
                        var testResult = new TestResult()
                        {
                            FullName  = Platform + " " + Settings.Custom["RuntimeVersion"],
                            Name      = Platform + " " + Settings.Custom["RuntimeVersion"],
                            StartTime = RunStartTime,
                            EndTime   = DateTime.UtcNow,
                            Outcome   = Failures > 0 ? "Failed" : "Passed",
                            RunId     = Settings.Custom["MasterRunId"],
                            Tags      = new List <string> {
                                Platform
                            },
                            Source = "Managed",
                            Logs   = sasResponseObject != null ? logs : null
                        };

                        // Upload the log of the test result summary for the test suite
                        if (sasResponseObject != null)
                        {
                            await TestLogger.UploadTestLog(logs, sasResponseObject);
                        }

                        // Post the test suite result to master run
                        await TestLogger.PostTestResults(new List <TestResult> {
                            testResult
                        }, Settings);
                    }
                    // Otherwise if we've finished the entire test run

                    // Finish the UI for the last group and update the
                    // progress after the very last test method.
                    Reporter.EndGroup(currentGroup);
                    Reporter.Progress(this);

                    // Finish the UI for the test run.
                    Reporter.EndRun(this);
                }
            };

            // Start running the tests
            await testLoop();
        }