diff options
| -rw-r--r-- | .github/workflows/main.yml | 16 | ||||
| -rw-r--r-- | python/servo/testing_commands.py | 3 | ||||
| -rw-r--r-- | tests/wpt/servowpt.py | 61 |
3 files changed, 64 insertions, 16 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c464b67154a..a62c62e3d25 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -216,14 +216,16 @@ jobs: --release --processes $(nproc) --timeout-multiplier 2 \ --total-chunks ${{ env.max_chunk_id }} --this-chunk ${{ matrix.chunk_id }} \ --log-raw test-wpt.${{ matrix.chunk_id }}.log \ - --filter-intermittents=filtered-wpt-results.${{ matrix.chunk_id }}.json + --log-raw-unexpected unexpected-test-wpt.${{ matrix.chunk_id }}.log \ + --filter-intermittents filtered-test-wpt.${{ matrix.chunk_id }}.json - name: Archive filtered results uses: actions/upload-artifact@v3 if: ${{ always() }} with: name: wpt-filtered-results-linux path: | - filtered-wpt-results.${{ matrix.chunk_id }}.json + filtered-test-wpt.${{ matrix.chunk_id }}.json + unexpected-test-wpt.${{ matrix.chunk_id }}.log - name: Archive logs uses: actions/upload-artifact@v3 if: ${{ failure() }} @@ -247,8 +249,16 @@ jobs: with: name: wpt-filtered-results-linux path: wpt-filtered-results-linux + - name: Create aggregated unexpected results + run: cat wpt-filtered-results-linux/*.log > unexpected-test-wpt.log + - name: Archive aggregate results + uses: actions/upload-artifact@v3 + with: + name: wpt-filtered-results-linux + path: | + unexpected-test-wpt.log - name: Comment on PR with results - run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/* + run: etc/ci/report_aggregated_expected_results.py wpt-filtered-results-linux/*.json env: GITHUB_CONTEXT: ${{ toJson(github) }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py index c017baacc91..0cb20c0127f 100644 --- a/python/servo/testing_commands.py +++ b/python/servo/testing_commands.py @@ -87,6 +87,9 @@ def create_parser_wpt(): parser.add_argument('--filter-intermittents', default=None, action="store", help="Filter intermittents against known intermittents " "and save the filtered output to the given file.") + parser.add_argument('--log-raw-unexpected', default=None, action="store", + help="Raw structured log messages for unexpected results." + " '--log-raw' Must also be passed in order to use this.") return parser diff --git a/tests/wpt/servowpt.py b/tests/wpt/servowpt.py index e66e359877f..241ee86a52a 100644 --- a/tests/wpt/servowpt.py +++ b/tests/wpt/servowpt.py @@ -122,6 +122,8 @@ def run_tests(**kwargs): kwargs["test_types"] = test_types[product] filter_intermittents_output = kwargs.pop("filter_intermittents", None) + unexpected_raw_log_output_file = kwargs.pop("log_raw_unexpected", None) + raw_log_outputs = kwargs.get("log_raw", []) wptcommandline.check_args(kwargs) update_args_for_layout_2020(kwargs) @@ -146,14 +148,28 @@ def run_tests(**kwargs): logger.add_handler(handler) wptrunner.run_tests(**kwargs) + return_value = 0 if not handler.unexpected_results else 1 + + # Filter intermittents if that was specified on the command-line. if handler.unexpected_results and filter_intermittents_output: all_filtered = filter_intermittents( handler.unexpected_results, - filter_intermittents_output, + filter_intermittents_output ) - return 0 if all_filtered else 1 - else: - return 0 if not handler.unexpected_results else 1 + return_value = 0 if all_filtered else 1 + + # Write the unexpected-only raw log if that was specified on the command-line. + if unexpected_raw_log_output_file: + if not raw_log_outputs: + print("'--log-raw-unexpected' not written without '--log-raw'.") + else: + write_unexpected_only_raw_log( + handler.unexpected_results, + raw_log_outputs[0].name, + unexpected_raw_log_output_file + ) + + return return_value def update_tests(**kwargs): @@ -228,15 +244,6 @@ def filter_intermittents( else: unexpected.append(result) - if output_file: - with open(output_file, "w", encoding="utf-8") as file: - file.write(json.dumps({ - "known_intermittents": - [dataclasses.asdict(result) for result in known_intermittents], - "unexpected": - [dataclasses.asdict(result) for result in unexpected], - })) - output = "\n".join([ f"{len(known_intermittents)} known-intermittent unexpected result", *[str(result) for result in known_intermittents], @@ -246,9 +253,37 @@ def filter_intermittents( ]) print(output) print(80 * "=") + + if output_file: + print(f"Writing filtered results to {output_file}") + with open(output_file, "w", encoding="utf-8") as file: + file.write(json.dumps({ + "known_intermittents": + [dataclasses.asdict(result) for result in known_intermittents], + "unexpected": + [dataclasses.asdict(result) for result in unexpected], + })) + return not unexpected +def write_unexpected_only_raw_log( + unexpected_results: List[UnexpectedResult], + raw_log_file: str, + filtered_raw_log_file: str +): + tests = [result.path for result in unexpected_results] + print(f"Writing unexpected-only raw log to {filtered_raw_log_file}") + + with open(filtered_raw_log_file, "w", encoding="utf-8") as output: + with open(raw_log_file) as input: + for line in input.readlines(): + data = json.loads(line) + if data["action"] in ["suite_start", "suite_end"] or \ + ("test" in data and data["test"] in tests): + output.write(line) + + def main(): from wptrunner import wptcommandline |