Merge pull request #8038 from uutils/renovate/clap_complete-4.x-lockfile #8685
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fuzzing | |
| # spell-checker:ignore fuzzer dtolnay Swatinem | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - '*' | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| # End the current execution if there is a new changeset in the PR. | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| jobs: | |
| fuzz-build: | |
| name: Build the fuzzers | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - name: Install `cargo-fuzz` | |
| run: cargo install cargo-fuzz | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "cargo-fuzz-cache-key" | |
| cache-directories: "fuzz/target" | |
| - name: Run `cargo-fuzz build` | |
| run: cargo +nightly fuzz build | |
| fuzz-run: | |
| needs: fuzz-build | |
| name: Fuzz | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| env: | |
| RUN_FOR: 60 | |
| strategy: | |
| matrix: | |
| test-target: | |
| - { name: fuzz_test, should_pass: true } | |
| # https://github.com/uutils/coreutils/issues/5311 | |
| - { name: fuzz_date, should_pass: false } | |
| - { name: fuzz_expr, should_pass: true } | |
| - { name: fuzz_printf, should_pass: true } | |
| - { name: fuzz_echo, should_pass: true } | |
| - { name: fuzz_seq, should_pass: false } | |
| - { name: fuzz_sort, should_pass: false } | |
| - { name: fuzz_wc, should_pass: false } | |
| - { name: fuzz_cut, should_pass: false } | |
| - { name: fuzz_split, should_pass: false } | |
| - { name: fuzz_tr, should_pass: false } | |
| - { name: fuzz_env, should_pass: false } | |
| - { name: fuzz_cksum, should_pass: false } | |
| - { name: fuzz_parse_glob, should_pass: true } | |
| - { name: fuzz_parse_size, should_pass: true } | |
| - { name: fuzz_parse_time, should_pass: true } | |
| - { name: fuzz_seq_parse_number, should_pass: true } | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - uses: dtolnay/rust-toolchain@nightly | |
| - name: Install `cargo-fuzz` | |
| run: cargo install cargo-fuzz | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| shared-key: "cargo-fuzz-cache-key" | |
| cache-directories: "fuzz/target" | |
| - name: Restore Cached Corpus | |
| uses: actions/cache/restore@v4 | |
| with: | |
| key: corpus-cache-${{ matrix.test-target.name }} | |
| path: | | |
| fuzz/corpus/${{ matrix.test-target.name }} | |
| - name: Run ${{ matrix.test-target.name }} for XX seconds | |
| id: run_fuzzer | |
| shell: bash | |
| continue-on-error: ${{ !matrix.test-target.name.should_pass }} | |
| run: | | |
| mkdir -p fuzz/stats | |
| STATS_FILE="fuzz/stats/${{ matrix.test-target.name }}.txt" | |
| cargo +nightly fuzz run ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -timeout=${{ env.RUN_FOR }} -detect_leaks=0 -print_final_stats=1 2>&1 | tee "$STATS_FILE" | |
| # Extract key stats from the output | |
| if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then | |
| RUNS=$(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') | |
| echo "runs=$RUNS" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "runs=unknown" >> "$GITHUB_OUTPUT" | |
| fi | |
| if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then | |
| EXEC_RATE=$(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') | |
| echo "exec_rate=$EXEC_RATE" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "exec_rate=unknown" >> "$GITHUB_OUTPUT" | |
| fi | |
| if grep -q "stat::new_units_added" "$STATS_FILE"; then | |
| NEW_UNITS=$(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') | |
| echo "new_units=$NEW_UNITS" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "new_units=unknown" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Save should_pass value to file for summary job to use | |
| echo "${{ matrix.test-target.should_pass }}" > "fuzz/stats/${{ matrix.test-target.name }}.should_pass" | |
| # Print stats to job output for immediate visibility | |
| echo "----------------------------------------" | |
| echo "FUZZING STATISTICS FOR ${{ matrix.test-target.name }}" | |
| echo "----------------------------------------" | |
| echo "Runs: $(grep -q "stat::number_of_executed_units" "$STATS_FILE" && grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" | |
| echo "Execution Rate: $(grep -q "stat::average_exec_per_sec" "$STATS_FILE" && grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}' || echo "unknown") execs/sec" | |
| echo "New Units: $(grep -q "stat::new_units_added" "$STATS_FILE" && grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}' || echo "unknown")" | |
| echo "Expected: ${{ matrix.test-target.name.should_pass }}" | |
| if grep -q "SUMMARY: " "$STATS_FILE"; then | |
| echo "Status: $(grep "SUMMARY: " "$STATS_FILE" | head -1)" | |
| else | |
| echo "Status: Completed" | |
| fi | |
| echo "----------------------------------------" | |
| # Add summary to GitHub step summary | |
| echo "### Fuzzing Results for ${{ matrix.test-target.name }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY | |
| echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY | |
| if grep -q "stat::number_of_executed_units" "$STATS_FILE"; then | |
| echo "| Runs | $(grep "stat::number_of_executed_units" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if grep -q "stat::average_exec_per_sec" "$STATS_FILE"; then | |
| echo "| Execution Rate | $(grep "stat::average_exec_per_sec" "$STATS_FILE" | awk '{print $2}') execs/sec |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| if grep -q "stat::new_units_added" "$STATS_FILE"; then | |
| echo "| New Units | $(grep "stat::new_units_added" "$STATS_FILE" | awk '{print $2}') |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "| Should pass | ${{ matrix.test-target.should_pass }} |" >> $GITHUB_STEP_SUMMARY | |
| if grep -q "SUMMARY: " "$STATS_FILE"; then | |
| echo "| Status | $(grep "SUMMARY: " "$STATS_FILE" | head -1) |" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "| Status | Completed |" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| - name: Save Corpus Cache | |
| uses: actions/cache/save@v4 | |
| with: | |
| key: corpus-cache-${{ matrix.test-target.name }} | |
| path: | | |
| fuzz/corpus/${{ matrix.test-target.name }} | |
| - name: Upload Stats | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fuzz-stats-${{ matrix.test-target.name }} | |
| path: | | |
| fuzz/stats/${{ matrix.test-target.name }}.txt | |
| fuzz/stats/${{ matrix.test-target.name }}.should_pass | |
| retention-days: 5 | |
| fuzz-summary: | |
| needs: fuzz-run | |
| name: Fuzzing Summary | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| persist-credentials: false | |
| - name: Download all stats | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: fuzz/stats-artifacts | |
| pattern: fuzz-stats-* | |
| merge-multiple: true | |
| - name: Prepare stats directory | |
| run: | | |
| mkdir -p fuzz/stats | |
| # Debug: List content of stats-artifacts directory | |
| echo "Contents of stats-artifacts directory:" | |
| find fuzz/stats-artifacts -type f | sort | |
| # Extract files from the artifact directories - handle nested directories | |
| find fuzz/stats-artifacts -type f -name "*.txt" -exec cp {} fuzz/stats/ \; | |
| find fuzz/stats-artifacts -type f -name "*.should_pass" -exec cp {} fuzz/stats/ \; | |
| # Debug information | |
| echo "Contents of stats directory after extraction:" | |
| ls -la fuzz/stats/ | |
| echo "Contents of should_pass files (if any):" | |
| cat fuzz/stats/*.should_pass 2>/dev/null || echo "No should_pass files found" | |
| - name: Generate Summary | |
| run: | | |
| echo "# Fuzzing Summary" > fuzzing_summary.md | |
| echo "" >> fuzzing_summary.md | |
| echo "| Target | Runs | Exec/sec | New Units | Should pass | Status |" >> fuzzing_summary.md | |
| echo "|--------|------|----------|-----------|-------------|--------|" >> fuzzing_summary.md | |
| TOTAL_RUNS=0 | |
| TOTAL_NEW_UNITS=0 | |
| for stat_file in fuzz/stats/*.txt; do | |
| TARGET=$(basename "$stat_file" .txt) | |
| SHOULD_PASS_FILE="${stat_file%.*}.should_pass" | |
| # Get expected status | |
| if [ -f "$SHOULD_PASS_FILE" ]; then | |
| EXPECTED=$(cat "$SHOULD_PASS_FILE") | |
| else | |
| EXPECTED="unknown" | |
| fi | |
| # Extract runs | |
| if grep -q "stat::number_of_executed_units" "$stat_file"; then | |
| RUNS=$(grep "stat::number_of_executed_units" "$stat_file" | awk '{print $2}') | |
| TOTAL_RUNS=$((TOTAL_RUNS + RUNS)) | |
| else | |
| RUNS="unknown" | |
| fi | |
| # Extract execution rate | |
| if grep -q "stat::average_exec_per_sec" "$stat_file"; then | |
| EXEC_RATE=$(grep "stat::average_exec_per_sec" "$stat_file" | awk '{print $2}') | |
| else | |
| EXEC_RATE="unknown" | |
| fi | |
| # Extract new units added | |
| if grep -q "stat::new_units_added" "$stat_file"; then | |
| NEW_UNITS=$(grep "stat::new_units_added" "$stat_file" | awk '{print $2}') | |
| if [[ "$NEW_UNITS" =~ ^[0-9]+$ ]]; then | |
| TOTAL_NEW_UNITS=$((TOTAL_NEW_UNITS + NEW_UNITS)) | |
| fi | |
| else | |
| NEW_UNITS="unknown" | |
| fi | |
| # Extract status | |
| if grep -q "SUMMARY: " "$stat_file"; then | |
| STATUS=$(grep "SUMMARY: " "$stat_file" | head -1) | |
| else | |
| STATUS="Completed" | |
| fi | |
| echo "| $TARGET | $RUNS | $EXEC_RATE | $NEW_UNITS | $EXPECTED | $STATUS |" >> fuzzing_summary.md | |
| done | |
| echo "" >> fuzzing_summary.md | |
| echo "## Overall Statistics" >> fuzzing_summary.md | |
| echo "" >> fuzzing_summary.md | |
| echo "- **Total runs:** $TOTAL_RUNS" >> fuzzing_summary.md | |
| echo "- **Total new units discovered:** $TOTAL_NEW_UNITS" >> fuzzing_summary.md | |
| echo "- **Average execution rate:** $(grep -h "stat::average_exec_per_sec" fuzz/stats/*.txt | awk '{sum += $2; count++} END {if (count > 0) print sum/count " execs/sec"; else print "unknown"}')" >> fuzzing_summary.md | |
| # Add count by expected status | |
| echo "- **Tests expected to pass:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "true")" >> fuzzing_summary.md | |
| echo "- **Tests expected to fail:** $(find fuzz/stats -name "*.should_pass" -exec cat {} \; | grep -c "false")" >> fuzzing_summary.md | |
| # Write to GitHub step summary | |
| cat fuzzing_summary.md >> $GITHUB_STEP_SUMMARY | |
| - name: Show Summary | |
| run: | | |
| cat fuzzing_summary.md | |
| - name: Upload Summary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fuzzing-summary | |
| path: fuzzing_summary.md | |
| retention-days: 5 |