自动化调优——TestNG失败用力批量重试

public class TestRetryConfig {

    public static final String TEST_PACKAGE_PATH = "com.xxx.xxx.testcases.";
    public static final String RETRY_SUIT_NAME = "TestNG Retry Test";

    public static int retryCount = 0;
    public static int retryMax = 0;

    public static List<ITestContext> finishTestContextList = new ArrayList();
    public static ITestContext testContext = null;

    public static void setRetryMax(int max) {
        retryMax = max;
    }

}

public class TestNgController {

    /***
     * 需要重试的testNGMethod
     */
    List<ITestNGMethod> testNGMethodList;
    /***
     * 按类名将ITestNGMethod分类
     */
    Map<String, List<String>> failClassMethodMap;

    public TestNgController(Collection<ITestNGMethod> testNGMethodSet) {
        this.testNGMethodList = new ArrayList<>(testNGMethodSet);
    }

    private void getFialMap() {
        if (this.testNGMethodList != null && this.testNGMethodList.size() > 0) {
            failClassMethodMap = new HashMap<>();
            for (ITestNGMethod iTestNGMethod : testNGMethodList) {
                String className = iTestNGMethod.getTestClass().getName();
                if (failClassMethodMap.get(className) != null) {
                    failClassMethodMap.get(className).add(iTestNGMethod.getMethodName());
                } else {
                    List<String> methodList = new ArrayList<>();
                    methodList.add(iTestNGMethod.getMethodName());
                    failClassMethodMap.put(className, methodList);
                }
            }
        }
    }

    /**
     * TestNG测试程序化调用
     */
    public boolean executeTests() {

        //获取失败用例map
        getFialMap();

        //没有失败的用例
        if (failClassMethodMap == null || failClassMethodMap.keySet().size() == 0) {
            return false;
        }
        //构建testng.xml内存对象
        try {
            List<XmlSuite> suites = new ArrayList<XmlSuite>();

            XmlSuite suite = new XmlSuite();
            suite.setName(TestRetryConfig.RETRY_SUIT_NAME + TestRetryConfig.retryCount);
            List<String> suiteListeners = new ArrayList<String>();
            suiteListeners.add(TestListener.class.getName());
            suite.setListeners(suiteListeners);

            suites.add(suite);

            XmlTest test = new XmlTest(suite);
            test.setName(TestRetryConfig.RETRY_SUIT_NAME + TestRetryConfig.retryCount);

            List<XmlClass> classes = new ArrayList<XmlClass>();

            Set<String> failClassSet = failClassMethodMap.keySet();
            for (String className : failClassSet) {
                List<String> methodNameList = failClassMethodMap.get(className);
                List<XmlInclude> includedMethodList = new ArrayList<>();
                for (String methodName : methodNameList) {
                    includedMethodList.add(new XmlInclude(methodName));
                }

                XmlClass testClass = new XmlClass(className);
                testClass.setIncludedMethods(includedMethodList);

                classes.add(testClass);
            }
            test.setXmlClasses(classes);

            LogUtils.log(String.format("第%s次执行批量重试测试用例开始！！！！！！！！！ ",TestRetryConfig.retryCount));
            //设置TestNG，并开始执行用例
            TestNG testNG = new TestNG();
            testNG.setXmlSuites(suites);
            testNG.run();

            return true;

        } catch (Exception e) {
            LogUtils.log("批量重试测试用例执行失败： " + e.getMessage());
            e.printStackTrace();
        }
        return false;
    }

    public static void main(String[] args) {
        TestNgController testNgController = new TestNgController(Collections.EMPTY_LIST);
        testNgController.executeTests();
    }

}

public class TestListener implements ITestListener {

    @Override
    public void onTestStart(ITestResult iTestResult) {

    }

    @Override
    public void onTestSuccess(ITestResult iTestResult) {

    }

    @Override
    public void onTestFailure(ITestResult iTestResult) {

    }

    @Override
    public void onTestSkipped(ITestResult iTestResult) {

    }

    @Override
    public void onTestFailedButWithinSuccessPercentage(ITestResult iTestResult) {

    }

    @Override
    public void onStart(ITestContext iTestContext) {

    }

    @Override
    public void onFinish(ITestContext iTestContext) {
        TestRetryConfig.finishTestContextList.add(iTestContext);

        //记录首次执行的测试集
        if (TestRetryConfig.testContext == null) {
            TestRetryConfig.testContext = iTestContext;
        }

        //已重试次数<最大重试次数，继续执行失败case
        if (TestRetryConfig.retryCount < TestRetryConfig.retryMax) {
            TestRetryConfig.retryCount++;
            TestNgController testNgController = new TestNgController(iTestContext.getFailedTests().getAllMethods());

            //有失败case，继续执行，无需处理结果
            if (testNgController.executeTests()) {
                return;
            }
        }

        //若无失败case,没有继续执行，则需要继续处理结果

        //指向首次执行的测试集
        iTestContext = TestRetryConfig.testContext;

        List<ITestResult> testsToBeRemoved = new ArrayList<>();
        Set<Integer> passedTestIds = new HashSet<>();

        //筛选出多次重试之后所有成功的用例
        for (ITestContext testContext : TestRetryConfig.finishTestContextList) {
            for (ITestResult passedTest : testContext.getPassedTests().getAllResults()) {
//                LogUtils.log("PassedTests = " + passedTest.getMethod().getTestClass().getName() + "." + passedTest.getMethod().getMethodName());
                passedTestIds.add(getId(passedTest));
            }
        }

        //筛选出多次重试之后所有失败的用例
        Set<Integer> failedTestIds = new HashSet<Integer>();
        for (ITestResult failedTest : iTestContext.getFailedTests().getAllResults()) {
//            LogUtils.log("failedTest = " + failedTest.getMethod().getTestClass().getName() + "." + failedTest.getMethod().getMethodName());
            int failedTestId = getId(failedTest);
            if (failedTestIds.contains(failedTestId) || passedTestIds.contains(failedTestId)) {
                testsToBeRemoved.add(failedTest);
            } else {
                failedTestIds.add(failedTestId);
            }
        }

        //将重试之后成功的用例，从失败结果集中删除，并添加到成功的结果集
        for (Iterator<ITestResult> iterator = iTestContext.getFailedTests().getAllResults().iterator(); iterator
                .hasNext(); ) {
            ITestResult testResult = iterator.next();
            if (testsToBeRemoved.contains(testResult)) {
                LogUtils.log("Change failed test to passed: " + testResult.getMethod().getTestClass().getName() + "." + testResult.getMethod().getMethodName());
                iterator.remove();
                iTestContext.getPassedTests().addResult(testResult, testResult.getMethod());
            }
        }

    }

    private int getId(ITestResult result) {
        int id = result.getTestClass().getName().hashCode();
        id = id + result.getMethod().getMethodName().hashCode();
        id = id + (result.getParameters() != null ? Arrays.hashCode(result.getParameters()) : 0);
        return id;
    }

/**
 * The main entry for the XML generation operation
 */
public class XMLReporter implements IReporter {
    public static final String FILE_NAME = "testng-results.xml";
    private final XMLReporterConfig config = new XMLReporterConfig();
    private XMLStringBuffer rootBuffer;

    @Override
    public void generateReport(List<XmlSuite> xmlSuites, List<ISuite> suites, String outputDirectory) {

        ITestContext testContext = TestRetryConfig.testContext;

        if (Utils.isStringEmpty(config.getOutputDirectory())) {
            config.setOutputDirectory(outputDirectory);
        }

        // Calculate passed/failed/skipped
        int passed = testContext.getPassedTests().size();
        int failed = testContext.getFailedTests().size();
        int skipped = testContext.getSkippedTests().size();
        int ignored = 0;
        int retried = 0;

        rootBuffer = new XMLStringBuffer();
        Properties p = new Properties();
        p.put("passed", passed);
        p.put("failed", failed);
        p.put("skipped", skipped);
        if (retried > 0) {
            p.put("retried", retried);
        }
        p.put("ignored", ignored);
        p.put("total", passed + failed + skipped + ignored + retried);
        rootBuffer.push(XMLReporterConfig.TAG_TESTNG_RESULTS, p);
        writeReporterOutput(rootBuffer);
        for (ISuite suite : suites) {
            if (suite.getName().contains(TestRetryConfig.RETRY_SUIT_NAME)) {
                continue;
            }
            writeSuite(suite);
        }
        rootBuffer.pop();
        Utils.writeUtf8File(config.getOutputDirectory(), fileName(), rootBuffer, null /* no prefix */);
    }

    private static String fileName() {
        return FILE_NAME;
    }

    private void writeReporterOutput(XMLStringBuffer xmlBuffer) {
        // TODO: Cosmin - maybe a <line> element isn't indicated for each line
        xmlBuffer.push(XMLReporterConfig.TAG_REPORTER_OUTPUT);
        List<String> output = Reporter.getOutput();
        for (String line : output) {
            if (line != null) {
                xmlBuffer.push(XMLReporterConfig.TAG_LINE);
                xmlBuffer.addCDATA(line);
                xmlBuffer.pop();
            }
        }
        xmlBuffer.pop();
    }

    private void writeSuite(ISuite suite) {
        switch (config.getFileFragmentationLevel()) {
            case XMLReporterConfig.FF_LEVEL_NONE:
                writeSuiteToBuffer(rootBuffer, suite);
                break;
            case XMLReporterConfig.FF_LEVEL_SUITE:
            case XMLReporterConfig.FF_LEVEL_SUITE_RESULT:
                File suiteFile = referenceSuite(rootBuffer, suite);
                writeSuiteToFile(suiteFile, suite);
                break;
            default:
                throw new AssertionError("Unexpected value: " + config.getFileFragmentationLevel());
        }
    }

    private void writeSuiteToFile(File suiteFile, ISuite suite) {
        XMLStringBuffer xmlBuffer = new XMLStringBuffer();
        writeSuiteToBuffer(xmlBuffer, suite);
        File parentDir = suiteFile.getParentFile();
        suiteFile.getParentFile().mkdirs();
        if (parentDir.exists() || suiteFile.getParentFile().exists()) {
            Utils.writeUtf8File(parentDir.getAbsolutePath(), fileName(), xmlBuffer.toXML());
        }
    }

    private File referenceSuite(XMLStringBuffer xmlBuffer, ISuite suite) {
        String relativePath = suite.getName() + File.separatorChar + fileName();
        File suiteFile = new File(config.getOutputDirectory(), relativePath);
        Properties attrs = new Properties();
        attrs.setProperty(XMLReporterConfig.ATTR_URL, relativePath);
        xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_SUITE, attrs);
        return suiteFile;
    }

    private void writeSuiteToBuffer(XMLStringBuffer xmlBuffer, ISuite suite) {
        xmlBuffer.push(XMLReporterConfig.TAG_SUITE, getSuiteAttributes(suite));
        writeSuiteGroups(xmlBuffer, suite);

        Map<String, ISuiteResult> results = suite.getResults();
        XMLSuiteResultWriter suiteResultWriter = new XMLSuiteResultWriter(config);
        for (Map.Entry<String, ISuiteResult> result : results.entrySet()) {
            suiteResultWriter.writeSuiteResult(xmlBuffer, result.getValue());
        }

        xmlBuffer.pop();
    }

    private void writeSuiteGroups(XMLStringBuffer xmlBuffer, ISuite suite) {
        xmlBuffer.push(XMLReporterConfig.TAG_GROUPS);
        Map<String, Collection<ITestNGMethod>> methodsByGroups = suite.getMethodsByGroups();
        for (Map.Entry<String, Collection<ITestNGMethod>> entry : methodsByGroups.entrySet()) {
            Properties groupAttrs = new Properties();
            groupAttrs.setProperty(XMLReporterConfig.ATTR_NAME, entry.getKey());
            xmlBuffer.push(XMLReporterConfig.TAG_GROUP, groupAttrs);
            Set<ITestNGMethod> groupMethods = getUniqueMethodSet(entry.getValue());
            for (ITestNGMethod groupMethod : groupMethods) {
                Properties methodAttrs = new Properties();
                methodAttrs.setProperty(XMLReporterConfig.ATTR_NAME, groupMethod.getMethodName());
                methodAttrs.setProperty(XMLReporterConfig.ATTR_METHOD_SIG, groupMethod.toString());
                methodAttrs.setProperty(XMLReporterConfig.ATTR_CLASS, groupMethod.getRealClass().getName());
                xmlBuffer.addEmptyElement(XMLReporterConfig.TAG_METHOD, methodAttrs);
            }
            xmlBuffer.pop();
        }
        xmlBuffer.pop();
    }

    private Properties getSuiteAttributes(ISuite suite) {
        Properties props = new Properties();
        props.setProperty(XMLReporterConfig.ATTR_NAME, suite.getName());

        // Calculate the duration
        Map<String, ISuiteResult> results = suite.getResults();
        Date minStartDate = new Date();
        Date maxEndDate = null;
        // TODO: We could probably optimize this in order not to traverse this twice
        for (Map.Entry<String, ISuiteResult> result : results.entrySet()) {
            ITestContext testContext = result.getValue().getTestContext();
            Date startDate = testContext.getStartDate();
            Date endDate = testContext.getEndDate();
            if (minStartDate.after(startDate)) {
                minStartDate = startDate;
            }
            if (maxEndDate == null || maxEndDate.before(endDate)) {
                maxEndDate = endDate != null ? endDate : startDate;
            }
        }
        // The suite could be completely empty
        if (maxEndDate == null) {
            maxEndDate = minStartDate;
        }
        addDurationAttributes(config, props, minStartDate, maxEndDate);
        return props;
    }

    /**
     * Add started-at, finished-at and duration-ms attributes to the <suite> tag
     */
    public static void addDurationAttributes(
            XMLReporterConfig config, Properties attributes, Date minStartDate, Date maxEndDate) {

        String startTime = DateUtils.formatDate(minStartDate.getTime());

        String endTime = DateUtils.formatDate(maxEndDate.getTime());
        long duration = maxEndDate.getTime() - minStartDate.getTime();

        attributes.setProperty(XMLReporterConfig.ATTR_STARTED_AT, startTime);
        attributes.setProperty(XMLReporterConfig.ATTR_FINISHED_AT, endTime);
        attributes.setProperty(XMLReporterConfig.ATTR_DURATION_MS, Long.toString(duration));
    }

    private Set<ITestNGMethod> getUniqueMethodSet(Collection<ITestNGMethod> methods) {
        return new LinkedHashSet<>(methods);
    }


}

public class HtmlReporter implements IReporter {

    //生成的路径以及文件名
    private static final String OUTPUT_FOLDER = "test-output/";
    private static final String FILE_NAME = "report.html";

    private ExtentReports extent;

    @Override
    public void generateReport(List<XmlSuite> suites, List<ISuite> list1, String s) {

        init();

        //统计suite下的成功、失败、跳过的总用例数
        int suiteFailSize = 0;
        int suitePassSize = 0;
        int suiteSkipSize = 0;
        ExtentTest suiteTest = null;

        ExtentTest resultNode;
        ITestContext context = TestRetryConfig.testContext;

        boolean createSuiteResultNode = false;

        if (createSuiteResultNode) {
            //没有创建suite的情况下，将在SuiteResult的创建为一级节点，否则创建为suite的一个子节点。
            if (null == suiteTest) {
                resultNode = extent.createTest(context.getName());
            } else {
                resultNode = suiteTest.createNode(context.getName());
            }
        } else {
            resultNode = suiteTest;
        }

        if (resultNode != null) {
            resultNode.getModel().setName(context.getSuite().getName() + " : " + context.getName());
            if (resultNode.getModel().hasCategory()) {
                resultNode.assignCategory(context.getName());
            } else {
                resultNode.assignCategory(context.getSuite().getName(), context.getName());
            }
            resultNode.getModel().setStartTime(context.getStartDate());
            resultNode.getModel().setEndTime(context.getEndDate());
            //统计SuiteResult下的数据
            int passSize = context.getPassedTests().size();
            int failSize = context.getFailedTests().size();
            int skipSize = context.getSkippedTests().size();
            suitePassSize += passSize;
            suiteFailSize += failSize;
            suiteSkipSize += skipSize;
            if (failSize > 0) {
                resultNode.getModel().setStatus(Status.FAIL);
            }
            resultNode.getModel().setDescription(String.format("Pass: %s ; Fail: %s ; Skip: %s ;", passSize, failSize, skipSize));
        }
        buildTestNodes(resultNode, context.getFailedTests(), Status.FAIL);
        buildTestNodes(resultNode, context.getSkippedTests(), Status.SKIP);
        buildTestNodes(resultNode, context.getPassedTests(), Status.PASS);
        if (suiteTest != null) {
            suiteTest.getModel().setDescription(String.format("Pass: %s ; Fail: %s ; Skip: %s ;", suitePassSize, suiteFailSize, suiteSkipSize));
            if (suiteFailSize > 0) {
                suiteTest.getModel().setStatus(Status.FAIL);
            }
        }
        extent.flush();
    }

    private void init() {
        //文件夹不存在的话进行创建
        File reportDir = new File(OUTPUT_FOLDER);
        if (!reportDir.exists() && !reportDir.isDirectory()) {
            reportDir.mkdir();
        }
        ExtentHtmlReporter htmlReporter = new ExtentHtmlReporter(OUTPUT_FOLDER + FILE_NAME);
        // 设置静态文件的DNS
        htmlReporter.config().setResourceCDN(ResourceCDN.EXTENTREPORTS);

        htmlReporter.config().setDocumentTitle("TEST RESULT");
        htmlReporter.config().setReportName("TEST RESULT");
        htmlReporter.config().setChartVisibilityOnOpen(true);
        htmlReporter.config().setTestViewChartLocation(ChartLocation.TOP);
        htmlReporter.config().setTheme(Theme.STANDARD);
        htmlReporter.config().setCSS(".node.level-1  ul{ display:none;} .node.level-1.active ul{display:block;}");
        htmlReporter.config().setEncoding("gbk");
        extent = new ExtentReports();
        extent.attachReporter(htmlReporter);
        extent.setReportUsesManualConfiguration(true);
    }

    private void buildTestNodes(ExtentTest extenttest, IResultMap tests, Status status) {
        //存在父节点时，获取父节点的标签
        String[] categories = new String[0];
        if (extenttest != null) {
            List<TestAttribute> categoryList = extenttest.getModel().getCategoryContext().getAll();
            categories = new String[categoryList.size()];
            for (int index = 0; index < categoryList.size(); index++) {
                categories[index] = categoryList.get(index).getName();
            }
        }

        ExtentTest test;

        if (tests.size() > 0) {
            //调整用例排序，按时间排序
            Set<ITestResult> treeSet = new TreeSet<ITestResult>(new Comparator<ITestResult>() {
                @Override
                public int compare(ITestResult o1, ITestResult o2) {
                    return o1.getStartMillis() < o2.getStartMillis() ? -1 : 1;
                }
            });
            treeSet.addAll(tests.getAllResults());
            for (ITestResult result : treeSet) {
                Object[] parameters = result.getParameters();
                String name = "";
                //如果有参数，则使用参数的toString组合代替报告中的name
                for (Object param : parameters) {
                    name += param.toString();
                }
                if (name.length() > 50) {
                    name = name.substring(0, 49) + "...";
                } else {
                    name = result.getTestClass().getName().replace(TestRetryConfig.TEST_PACKAGE_PATH, "") + "." + result.getMethod().getMethodName();
                }
                if (extenttest == null) {
                    test = extent.createTest(name);
                } else {
                    //作为子节点进行创建时，设置同父节点的标签一致，便于报告检索。
                    test = extenttest.createNode(name).assignCategory(categories);
                }
                for (String group : result.getMethod().getGroups())
                    test.assignCategory(group);

                List<String> outputList = Reporter.getOutput(result);
                for (String output : outputList) {
                    //将用例的log输出报告中
                    test.debug(output);
                }
                if (result.getThrowable() != null) {
                    test.log(status, result.getThrowable());
                } else {
                    test.log(status, "Test " + status.toString().toLowerCase() + "ed");
                }

                test.getModel().setStartTime(getTime(result.getStartMillis()));
                test.getModel().setEndTime(getTime(result.getEndMillis()));
            }
        }
    }

    private Date getTime(long millis) {
        Calendar calendar = Calendar.getInstance();
        calendar.setTimeInMillis(millis);
        return calendar.getTime();
    }


}

public class Test1 {

    static int n = 0;
    static int i = 0;
    static int j = 0;
    static int k = 0;

    @Test(description = "重试3次会成功")
    public void test4() {
        k++;
        assertEquals(4, k);
    }

    @Test(description = "重试2次会成功")
    public void test3() {
        j++;
        assertEquals(3, j);
    }

    @Test(description = "重试1次会成功")
    public void test2() {
        i++;
        assertEquals(2, i);
    }

    @Test(description = "首次成功")
    public void test1() {
        n++;
        assertEquals(1, n);
    }

}

public class Test2 {

    static int n = 0;
    static int i = 0;

    @Test(description = "重试1次会成功")
    public void test2() {
        i++;
        assertEquals(2, i);
    }

    @Test(description = "首次成功")
    public void test1() {
        n++;
        assertEquals(1, n);
    }

}

  <?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE suite SYSTEM "http://testng.org/testng-1.0.dtd">
<suite name="Surefire_suite" verbose="1" configfailurepolicy="continue">
   <listeners>
      <listener class-name="com.xxx.xxx.testcases.test.TestListener"/>
      <listener class-name="com.xxx.xxx..testcases.test.HtmlReporter"/>
      <listener class-name="com.xxx.xxx..testcases.test.XMLReporter"/>
   </listeners>
   <test name="test">
      <packages>
         <package name="com.xxx.xxx..testcases.test.*"/>
      </packages>
   </test>
</suite>

<plugin>
   <groupId>org.apache.maven.plugins</groupId>
   <artifactId>maven-surefire-plugin</artifactId>
   <version>3.0.0-M3</version>
   <configuration>
      <properties>
         <property>
            <name>usedefaultlisteners</name>
            <value>false</value> <!-- disabling default listeners is optional -->
         </property>
      </properties>
      <testFailureIgnore>true</testFailureIgnore>
                 <suiteXmlFiles>${suiteXmlFile}</suiteXmlFiles>
   </configuration>
</plugin>