Test Reporting

pip install allure-pytest

# Generate results
pytest --alluredir=allure-results

# Generate HTML report from results
allure generate allure-results -o allure-report --clean

# Serve locally
allure serve allure-results

# Annotating tests for rich Allure output
import allure

@allure.epic("Checkout")
@allure.feature("Payment")
@allure.story("Card payment")
@allure.severity(allure.severity_level.CRITICAL)
def test_card_payment_success():
    with allure.step("Add item to cart"):
        cart = add_to_cart(product_id=1)

    with allure.step("Enter card details"):
        payment = enter_payment(card_number="4242424242424242", expiry="12/28", cvv="123")

    with allure.step("Complete purchase"):
        result = checkout(cart, payment)

    allure.attach(json.dumps(result), name="response", attachment_type=allure.attachment_type.JSON)
    assert result["status"] == "paid"

# GitHub Actions — publish Allure to GitHub Pages
- name: Run tests
  run: pytest --alluredir=allure-results

- name: Generate Allure report
  uses: simple-elf/allure-report-action@master
  with:
    allure_results: allure-results
    allure_history: allure-history
    keep_reports: 20

- name: Deploy to GitHub Pages
  uses: peaceiris/actions-gh-pages@v3
  with:
    github_token: ${{ secrets.GITHUB_TOKEN }}
    publish_dir: allure-history

# pytest
pytest --junitxml=test-results.xml

# Playwright
npx playwright test --reporter=junit

# Maven
# surefire plugin generates target/surefire-reports/*.xml by default

# GitHub Actions — parse JUnit results
- name: Publish test results
  uses: EnricoMi/publish-unit-test-result-action@v2
  if: always()
  with:
    files: |
      test-results.xml
      **/test-results/**/*.xml
    comment_mode: always
    check_name: Test Results

# pytest + coverage
pytest --cov=src --cov-report=xml --cov-report=html --cov-report=term-missing

# Upload to Codecov
- uses: codecov/codecov-action@v4
  with:
    token: ${{ secrets.CODECOV_TOKEN }}
    files: coverage.xml
    fail_ci_if_error: true
    threshold: 80

# TypeScript — c8
c8 --reporter=html --reporter=lcov npm test

# Post test results to Slack after CI run
import httpx

def post_test_results_to_slack(webhook_url: str, results: dict) -> None:
    colour = "good" if results["failed"] == 0 else "danger"

    httpx.post(webhook_url, json={
        "attachments": [{
            "color": colour,
            "title": f"Test Results — Build #{results['build_number']}",
            "fields": [
                {"title": "Passed", "value": str(results["passed"]), "short": True},
                {"title": "Failed", "value": str(results["failed"]), "short": True},
                {"title": "Duration", "value": results["duration"], "short": True},
                {"title": "Coverage", "value": f"{results['coverage']}%", "short": True},
            ],
            "footer": f"<{results['report_url']}|View full report>",
        }]
    })

# Parse JUnit XML to track flaky tests over time
import xml.etree.ElementTree as ET
from collections import defaultdict

def extract_failures(xml_file: str) -> list[str]:
    tree = ET.parse(xml_file)
    return [
        tc.attrib["classname"] + "::" + tc.attrib["name"]
        for tc in tree.iter("testcase")
        if tc.find("failure") is not None
    ]

# Store in SQLite, query for tests that pass and fail within the same week
# Any test that fails < 100% of the time it runs = flaky = quarantine candidate

Test execution: pass rate, duration, flaky count
Coverage: current %, delta from last sprint
Escaped defects: bugs found in production this sprint
Defect detection efficiency: bugs found in sprint / (sprint + post-sprint)
Automation growth: new automated tests added this sprint