Пример #1
0
 public NeuralNetworkTrainer(INetworkLayers networkLayers, ITrainSet trainSet, IValidationSet validationSet, ITestSet testSet)
 {
     _networkLayers = networkLayers;
     _trainSet      = trainSet;
     _validationSet = validationSet;
     _testSet       = testSet;
 }
Пример #2
0
        public TestSummary(ITestSet tests, IIntegrator integrator, double targetError)
        {
            this.Results = new List <TestResult>();

            this.TestSet     = tests.Name;
            this.Integrator  = integrator.Name;
            this.TargetError = targetError;
        }
        /// <summary>
        /// gets a link string for the test run in Qc
        /// </summary>
        /// <param name="prevTest"></param>
        /// <param name="runid"></param>
        /// <returns></returns>
        private string GetTestRunLink(ITSTest prevTest, int runid)
        {
            bool     oldQc           = CheckIsOldQc();
            ITestSet set             = prevTest.TestSet;
            string   testRunLink     = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=IRun&EntityID=" + runid;
            string   testRunLinkQc10 = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindRun&TestSetID=" + set.ID + "&TestInstanceID=" + prevTest.ID + "&RunID=" + runid;
            string   linkStr         = (oldQc ? testRunLinkQc10 : testRunLink);

            return(linkStr);
        }
        /// <inheritdoc/>
        public void Log(ITestSet testSet)
        {
            ITestSetStatus testSetStatus = testSet.TestSetStatus;
            List <string>  str           = new List <string>();

            str.Add("Name:" + testSet.Name);
            str.Add("RunSuccessful:" + testSetStatus.RunSuccessful.ToString());

            foreach (string line in str)
            {
                Logger.Info(line);
            }
        }
Пример #5
0
        public void Run(ITestSet tests, IIntegrator integrator, double targetError)
        {
            listView1.Items.Clear();

            var summary = new TestSummary(tests, integrator, targetError);

            var functions = tests.GetTestFunctions();

            int total = 0, failed = 0;

            Util.Tic();

            foreach (var item in functions)
            {
                total++;

                try
                {
                    var value = integrator.Integrate(item, targetError);
                    Console.WriteLine(value);

                    var result = GetTestResult(item, value);

                    ShowTestResult(result, targetError);

                    summary.Add(result);

                    if (result.HasInvalidValue())
                    {
                        failed++;
                    }
                }
                catch (Exception e)
                {
                    ProcessError(item, e);

                    failed++;
                }
            }

            summary.Time = Util.Toc();

            summary.TotalCount  = total;
            summary.FailedCount = failed;

            ShowTestSummary(summary);
        }
        /// <summary>
        /// writes a summary of the test run after it's over
        /// </summary>
        /// <param name="prevTest"></param>
        private string GetTestInstancesString(ITestSet set)
        {
            string retVal = "";

            try
            {
                TSTestFactory factory = set.TSTestFactory;
                List          list    = factory.NewList("");

                if (list == null)
                {
                    return("");
                }

                foreach (ITSTest testInstance in list)
                {
                    retVal += testInstance.ID + ",";
                }
                retVal.TrimEnd(", \n".ToCharArray());
            }
            catch (Exception ex)
            { }
            return(retVal);
        }
        /// <summary>
        /// updates the test status in our list of tests
        /// </summary>
        /// <param name="targetTestSet"></param>
        /// <param name="testExecStatusObj"></param>
        private TestRunResults UpdateTestStatus(TestSuiteRunResults runResults, ITestSet targetTestSet, TestExecStatus testExecStatusObj, bool onlyUpdateState)
        {
            TestRunResults qTest       = null;
            ITSTest        currentTest = null;

            try
            {
                //find the test for the given status object
                currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId];

                //find the test in our list
                int testIndex = GetIdxByTestName(currentTest.Name, runResults);
                qTest = runResults.TestRuns[testIndex];

                if (qTest.TestType == null)
                {
                    qTest.TestType = GetTestType(currentTest);
                }

                //update the state
                qTest.PrevTestState = qTest.TestState;
                qTest.TestState     = GetTsStateFromQcState(testExecStatusObj.Status);

                if (!onlyUpdateState)
                {
                    try
                    {
                        //duration and status are updated according to the run
                        qTest.Runtime = TimeSpan.FromSeconds(currentTest.LastRun.Field("RN_DURATION"));
                    }
                    catch
                    {
                        //a problem getting duration, maybe the test isn't done yet - don't stop the flow..
                    }

                    switch (qTest.TestState)
                    {
                    case TestState.Failed:
                        qTest.FailureDesc = GenerateFailedLog(currentTest.LastRun);

                        if (string.IsNullOrWhiteSpace(qTest.FailureDesc))
                        {
                            qTest.FailureDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message;
                        }
                        break;

                    case TestState.Error:
                        qTest.ErrorDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message;
                        break;

                    default:
                        break;
                    }

                    //check qc version for link type
                    bool oldQc = CheckIsOldQc();

                    //string testLink = "<a href=\"testdirector:mydtqc01.isr.hp.com:8080/qcbin," + m_qcProject + "," + m_qcDomain + "," + targetTestSet.Name+ ";test-instance:" + testExecStatusObj.TestInstance + "\"> Alm link</a>";
                    string serverURl = m_qcServer.TrimEnd("/".ToCharArray());
                    if (serverURl.ToLower().StartsWith("http://"))
                    {
                        serverURl = serverURl.Substring(7);
                    }

                    //string testLinkInLabQc10 = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindTestInstance&TestSetID=" + targetTestSet.ID + "&TestInstanceID=" + testExecStatusObj.TSTestId;
                    //string testLinkInLab = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=ITestInstance&EntityID=" + testExecStatusObj.TSTestId;

                    int    runid   = GetTestRunId(currentTest);
                    string linkStr = GetTestRunLink(currentTest, runid);

                    string statusString = GetTsStateFromQcState(testExecStatusObj.Status as string).ToString();
                    ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerTestStat, currentTest.Name, statusString, testExecStatusObj.Message, linkStr));
                    runResults.TestRuns[testIndex] = qTest;
                }
            }
            catch (Exception ex)
            {
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerErrorGettingStat, currentTest.Name, ex.Message));
            }

            return(qTest);
        }
        /// <summary>
        /// runs a test set with given parameters (and a valid connection to the QC server)
        /// </summary>
        /// <param name="tsFolderName">testSet folder name</param>
        /// <param name="tsName">testSet name</param>
        /// <param name="timeout">-1 for unlimited, or number of miliseconds</param>
        /// <param name="runMode">run on LocalMachine or remote</param>
        /// <param name="runHost">if run on remote machine - remote machine name</param>
        /// <returns></returns>
        public TestSuiteRunResults RunTestSet(string tsFolderName, string tsName, double timeout, QcRunMode runMode, string runHost)
        {
            string currentTestSetInstances     = "";
            TestSuiteRunResults runDesc        = new TestSuiteRunResults();
            TestRunResults      activeTestDesc = null;

            var            tsFactory     = tdConnection.TestSetFactory;
            var            tsTreeManager = (ITestSetTreeManager)tdConnection.TestSetTreeManager;
            List           tsList        = null;
            string         tsPath        = "Root\\" + tsFolderName;
            ITestSetFolder tsFolder      = null;

            try
            {
                tsFolder = (ITestSetFolder)tsTreeManager.get_NodeByPath(tsPath);
            }
            catch (COMException ex)
            {
                //not found
                tsFolder = null;
            }

            if (tsFolder == null)
            {
                //node wasn't found, folder = null
                ConsoleWriter.WriteErrLine(string.Format(Resources.AlmRunnerNoSuchFolder, tsFolder));

                //this will make sure run will fail at the end. (since there was an error)
                Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
                return(null);
            }
            else
            {
                tsList = tsFolder.FindTestSets(tsName);
            }
            if (tsList == null)
            {
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerCantFindTest, tsName));

                //this will make sure run will fail at the end. (since there was an error)
                Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
                return(null);
            }
            ITestSet targetTestSet = null;

            foreach (ITestSet ts in tsList)
            {
                if (ts.Name.Equals(tsName, StringComparison.InvariantCultureIgnoreCase))
                {
                    targetTestSet = ts;
                    break;
                }
            }

            if (targetTestSet == null)
            {
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerCantFindTest, tsName));

                //this will make sure run will fail at the end. (since there was an error)
                Launcher.ExitCode = Launcher.ExitCodeEnum.Failed;
                return(null);
            }


            ConsoleWriter.WriteLine(Resources.GeneralDoubleSeperator);
            ConsoleWriter.WriteLine(Resources.AlmRunnerStartingExecution);
            ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerDisplayTest, tsName, targetTestSet.ID));

            ITSScheduler Scheduler = null;

            try
            {
                //need to run this to install everyhting needed http://AlmServer:8080/qcbin/start_a.jsp?common=true
                //start the scheduler
                Scheduler = targetTestSet.StartExecution("");
            }
            catch (Exception ex)
            {
                Scheduler = null;
            }
            try
            {
                currentTestSetInstances = GetTestInstancesString(targetTestSet);
            }
            catch (Exception ex)
            {
            }

            if (Scheduler == null)
            {
                Console.WriteLine(GetAlmNotInstalledError());

                //proceeding with program execution is tasteless, since nothing will run without a properly installed QC.
                Environment.Exit((int)Launcher.ExitCodeEnum.Failed);
            }

            TSTestFactory tsTestFactory = targetTestSet.TSTestFactory;
            ITDFilter2    tdFilter      = tsTestFactory.Filter;

            tdFilter["TC_CYCLE_ID"] = targetTestSet.ID.ToString();

            IList tList = tsTestFactory.NewList(tdFilter.Text);

            try
            {
                //set up for the run depending on where the test instances are to execute
                switch (runMode)
                {
                case QcRunMode.RUN_LOCAL:
                    // run all tests on the local machine
                    Scheduler.RunAllLocally = true;
                    break;

                case QcRunMode.RUN_REMOTE:
                    // run tests on a specified remote machine
                    Scheduler.TdHostName = runHost;
                    break;

                // RunAllLocally must not be set for remote invocation of tests. As such, do not do this: Scheduler.RunAllLocally = False
                case QcRunMode.RUN_PLANNED_HOST:
                    // run on the hosts as planned in the test set
                    Scheduler.RunAllLocally = false;
                    break;
                }
            }
            catch (Exception ex)
            {
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerProblemWithHost, ex.Message));
            }

            ConsoleWriter.WriteLine(Resources.AlmRunnerNumTests + tList.Count);

            int i = 1;

            foreach (ITSTest3 test in tList)
            {
                string runOnHost = runHost;
                if (runMode == QcRunMode.RUN_PLANNED_HOST)
                {
                    runOnHost = test.HostName;
                }

                //if host isn't taken from QC (PLANNED) and not from the test definition (REMOTE), take it from LOCAL (machineName)
                string hostName = runOnHost;
                if (runMode == QcRunMode.RUN_LOCAL)
                {
                    hostName = Environment.MachineName;
                }
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerDisplayTestRunOnHost, i, test.Name, hostName));

                Scheduler.RunOnHost[test.ID] = runOnHost;

                var testResults = new TestRunResults();
                testResults.TestName = test.Name;
                runDesc.TestRuns.Add(testResults);

                i = i + 1;
            }

            Stopwatch sw     = Stopwatch.StartNew();
            Stopwatch testSw = null;

            try
            {
                //tests are actually run
                Scheduler.Run();
            }
            catch (Exception ex)
            {
                ConsoleWriter.WriteLine(Resources.AlmRunnerRunError + ex.Message);
            }

            ConsoleWriter.WriteLine(Resources.AlmRunnerSchedStarted + DateTime.Now.ToString(Launcher.DateFormat));
            ConsoleWriter.WriteLine(Resources.SingleSeperator);
            IExecutionStatus executionStatus     = Scheduler.ExecutionStatus;
            bool             tsExecutionFinished = false;
            ITSTest          prevTest            = null;
            ITSTest          currentTest         = null;
            string           abortFilename       = System.IO.Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location) + "\\stop" + Launcher.UniqueTimeStamp + ".txt";

            //wait for the tests to end ("normally" or because of the timeout)
            while ((tsExecutionFinished == false) && (timeout == -1 || sw.Elapsed.TotalSeconds < timeout))
            {
                executionStatus.RefreshExecStatusInfo("all", true);
                tsExecutionFinished = executionStatus.Finished;

                if (System.IO.File.Exists(abortFilename))
                {
                    break;
                }

                for (int j = 1; j <= executionStatus.Count; ++j)
                {
                    TestExecStatus testExecStatusObj = executionStatus[j];

                    activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, true);

                    if (activeTestDesc.PrevTestState != activeTestDesc.TestState)
                    {
                        TestState tstate = activeTestDesc.TestState;
                        if (tstate == TestState.Running)
                        {
                            currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId];
                            int testIndex = GetIdxByTestName(currentTest.Name, runDesc);

                            int prevRunId = GetTestRunId(currentTest);
                            runDesc.TestRuns[testIndex].PrevRunId = prevRunId;

                            //closing previous test
                            if (prevTest != null)
                            {
                                WriteTestRunSummary(prevTest);
                            }

                            //starting new test
                            prevTest = currentTest;

                            //assign the new test the consol writer so it will gather the output

                            ConsoleWriter.ActiveTestRun = runDesc.TestRuns[testIndex];

                            ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " Running: " + currentTest.Name);

                            //tell user that the test is running
                            ConsoleWriter.WriteLine(DateTime.Now.ToString(Launcher.DateFormat) + " Running test: " + activeTestDesc.TestName + ", Test id: " + testExecStatusObj.TestId + ", Test instance id: " + testExecStatusObj.TSTestId);

                            //start timing the new test run
                            string         foldername = "";
                            ITestSetFolder folder     = targetTestSet.TestSetFolder as ITestSetFolder;

                            if (folder != null)
                            {
                                foldername = folder.Name.Replace(".", "_");
                            }

                            //the test group is it's test set. (dots are problematic since jenkins parses them as seperators between packadge and class)
                            activeTestDesc.TestGroup = foldername + "\\" + targetTestSet.Name;
                            activeTestDesc.TestGroup = activeTestDesc.TestGroup.Replace(".", "_");
                        }

                        TestState enmState     = GetTsStateFromQcState(testExecStatusObj.Status as string);
                        string    statusString = enmState.ToString();

                        if (enmState == TestState.Running)
                        {
                            ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerStat, activeTestDesc.TestName, testExecStatusObj.TSTestId, statusString));
                        }
                        else if (enmState != TestState.Waiting)
                        {
                            ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerStatWithMessage, activeTestDesc.TestName, testExecStatusObj.TSTestId, statusString, testExecStatusObj.Message));
                        }
                        if (System.IO.File.Exists(abortFilename))
                        {
                            break;
                        }
                    }
                }

                //wait 0.2 seconds
                Thread.Sleep(200);

                //check for abortion
                if (System.IO.File.Exists(abortFilename))
                {
                    _blnRunCancelled = true;

                    ConsoleWriter.WriteLine(Resources.GeneralStopAborted);

                    //stop all test instances in this testSet.
                    Scheduler.Stop(currentTestSetInstances);

                    ConsoleWriter.WriteLine(Resources.GeneralAbortedByUser);

                    //stop working
                    Environment.Exit((int)Launcher.ExitCodeEnum.Aborted);
                }
            }

            //check status for each test
            if (timeout == -1 || sw.Elapsed.TotalSeconds < timeout)
            {
                //close last test
                if (prevTest != null)
                {
                    WriteTestRunSummary(prevTest);
                }

                //done with all tests, stop collecting output in the testRun object.
                ConsoleWriter.ActiveTestRun = null;
                for (int k = 1; k <= executionStatus.Count; ++k)
                {
                    if (System.IO.File.Exists(abortFilename))
                    {
                        break;
                    }

                    TestExecStatus testExecStatusObj = executionStatus[k];
                    activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, false);

                    UpdateCounters(activeTestDesc, runDesc);

                    currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId];

                    string testPath = "Root\\" + tsFolderName + "\\" + tsName + "\\" + activeTestDesc.TestName;

                    activeTestDesc.TestPath = testPath;
                }

                //update the total runtime
                runDesc.TotalRunTime = sw.Elapsed;

                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerTestsetDone, tsName, DateTime.Now.ToString(Launcher.DateFormat)));
            }
            else
            {
                _blnRunCancelled = true;
                ConsoleWriter.WriteLine(Resources.GeneralTimedOut);
                Launcher.ExitCode = Launcher.ExitCodeEnum.Aborted;
            }

            return(runDesc);
        }
Пример #9
0
 public TestRun(ITestSet owner, IDictionary <string, string> param_set)
 {
     m_testSet = owner;
     m_conf    = new Config();
     m_conf.updateFrom(param_set);
 }
        /// <summary>
        /// writes a summary of the test run after it's over
        /// </summary>
        /// <param name="prevTest"></param>
        private string GetTestInstancesString(ITestSet set)
        {
            string retVal = "";
            try
            {
                TSTestFactory factory = set.TSTestFactory;
                List list = factory.NewList("");

                if (list == null)
                    return "";

                foreach (ITSTest testInstance in list)
                {
                    retVal += testInstance.ID + ",";
                }
                retVal.TrimEnd(", \n".ToCharArray());
            }
            catch (Exception ex)
            { }
            return retVal;
        }
        /// <summary>
        /// updates the test status in our list of tests
        /// </summary>
        /// <param name="targetTestSet"></param>
        /// <param name="testExecStatusObj"></param>
        private TestRunResults UpdateTestStatus(TestSuiteRunResults runResults, ITestSet targetTestSet, TestExecStatus testExecStatusObj, bool onlyUpdateState)
        {
            TestRunResults qTest = null;
            ITSTest currentTest = null;
            try
            {
                //find the test for the given status object
                currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId];

                if (currentTest == null)
                {
                    return qTest;
                }

                //find the test in our list
                int testIndex = GetIdxByTestName(currentTest.Name, runResults);
                qTest = runResults.TestRuns[testIndex];

                if (qTest.TestType == null)
                {
                    qTest.TestType = GetTestType(currentTest);
                }

                //update the state
                qTest.PrevTestState = qTest.TestState;
                qTest.TestState = GetTsStateFromQcState(testExecStatusObj.Status);

                if (!onlyUpdateState)
                {
                    try
                    {
                        //duration and status are updated according to the run
                        qTest.Runtime = TimeSpan.FromSeconds(currentTest.LastRun.Field("RN_DURATION"));
                    }
                    catch
                    {
                        //a problem getting duration, maybe the test isn't done yet - don't stop the flow..
                    }

                    switch (qTest.TestState)
                    {
                        case TestState.Failed:
                            qTest.FailureDesc = GenerateFailedLog(currentTest.LastRun);

                            if (string.IsNullOrWhiteSpace(qTest.FailureDesc))
                                qTest.FailureDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message;
                            break;
                        case TestState.Error:
                            qTest.ErrorDesc = testExecStatusObj.Status + " : " + testExecStatusObj.Message;
                            break;
                        default:
                            break;
                    }

                    //check qc version for link type
                    bool oldQc = CheckIsOldQc();

                    //string testLink = "<a href=\"testdirector:mydtqc01.isr.hp.com:8080/qcbin," + m_qcProject + "," + m_qcDomain + "," + targetTestSet.Name+ ";test-instance:" + testExecStatusObj.TestInstance + "\"> Alm link</a>";
                    string serverURl = m_qcServer.TrimEnd("/".ToCharArray());
                    if (serverURl.ToLower().StartsWith("http://"))
                        serverURl = serverURl.Substring(7);

                    //string testLinkInLabQc10 = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/Test%20Lab?Action=FindTestInstance&TestSetID=" + targetTestSet.ID + "&TestInstanceID=" + testExecStatusObj.TSTestId;
                    //string testLinkInLab = "td://" + m_qcProject + "." + m_qcDomain + "." + m_qcServer.Replace("http://", "") + "/TestLabModule-000000003649890581?EntityType=ITestInstance&EntityID=" + testExecStatusObj.TSTestId;

                    int runid = GetTestRunId(currentTest);
                    string linkStr = GetTestRunLink(currentTest, runid);

                    string statusString = GetTsStateFromQcState(testExecStatusObj.Status as string).ToString();
                    ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerTestStat, currentTest.Name, statusString, testExecStatusObj.Message, linkStr));
                    runResults.TestRuns[testIndex] = qTest;
                }
            }
            catch (Exception ex)
            {
                ConsoleWriter.WriteLine(string.Format(Resources.AlmRunnerErrorGettingStat, currentTest.Name, ex.Message));
            }

            return qTest;
        }
Пример #12
0
 public TestController(ITestSet testModels)
 {
     _testModels = testModels;
 }
Пример #13
0
        private void SetTestResults(ITSTest currentTest, IExecutionStatus executionStatus, ITestSet targetTestSet, TestRunResults activeTestDesc, TestSuiteRunResults runDesc, string testPath, string abortFilename)
        {
            // write the status for each test
            for (int k = 1; k <= executionStatus.Count; ++k)
            {
                if (System.IO.File.Exists(abortFilename))
                {
                    break;
                }

                TestExecStatus testExecStatusObj = executionStatus[k];
                currentTest = targetTestSet.TSTestFactory[testExecStatusObj.TSTestId];

                if (currentTest == null)
                {
                    ConsoleWriter.WriteLine(string.Format("currentTest is null for test.{0} after whole execution", k));
                    continue;
                }

                activeTestDesc = UpdateTestStatus(runDesc, targetTestSet, testExecStatusObj, false);
                UpdateCounters(activeTestDesc, runDesc);

                activeTestDesc.TestPath = testPath;
            }
        }