Unverified Commit 24235f09 authored by Michael Müller's avatar Michael Müller Committed by GitHub
Browse files

Fuzz after master merge + automatic issue creation. (#537)

* Prefix fuzz tests

* Fuzz extensively after master merge + create issue on failure

* Do not fuzz on regular test runs

* Migrate deprecated 'only' to 'rules'

* Indent correctly

* Fuzz each crate separately

* Add feature ink-fuzz-tests to crates which are missing it

* Increase QUICKCHECK_TESTS to 50k

* Set timeout to 1h

* Improve issue title + body

* Revert "Set timeout to 1h"

This reverts commit 5888abaf.

* Revert "Add feature ink-fuzz-tests to crates which are missing it"

This reverts commit e0ece29e.

* Fuzz only crates which have the ink-fuzz-tests feature

* Indent consistently with two spaces

* Make GitLab CI linter happy

* Create issue when at least one test fails
parent 3f33c655
Pipeline #112052 failed with stages
in 4 minutes and 38 seconds
......@@ -10,6 +10,7 @@ stages:
- workspace
- examples
- publish
- fuzz
variables:
GIT_STRATEGY: fetch
......@@ -110,6 +111,12 @@ test:
needs:
- job: check-std
artifacts: false
variables:
# Since we run the tests with `--all-features` this implies the feature
# `ink-fuzz-tests` as well -- i.e. the fuzz tests are run.
# There's no way to disable a single feature while enabling all features
# at the same time, hence we use this workaround.
QUICKCHECK_TESTS: 0
script:
- cargo test --verbose --all-features --no-fail-fast --workspace
......@@ -120,6 +127,8 @@ codecov:
- job: check-std
artifacts: false
variables:
# For codecov it's sufficient to run the fuzz tests only once.
QUICKCHECK_TESTS: 1
# Variables partly came from https://github.com/mozilla/grcov/blob/master/README.md
CARGO_INCREMENTAL: 0
RUSTFLAGS: "-Zprofile -Zmir-opt-level=0 -Ccodegen-units=1
......@@ -294,3 +303,39 @@ publish-docs:
- git push origin gh-pages --force
after_script:
- rm -rf .git/ ./*
#### stage: fuzz
fuzz-tests:
stage: fuzz
<<: *docker-env
variables:
# The QUICKCHECK_TESTS default is 100
QUICKCHECK_TESTS: 50000
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
script:
# We fuzz-test only crates which possess the `ink-fuzz-tests` feature
- all_tests_passed=0
- for crate in ${ALL_CRATES}; do
if grep "ink-fuzz-tests =" crates/${crate}/Cargo.toml;
then
cargo test --verbose --features ink-fuzz-tests --manifest-path crates/${crate}/Cargo.toml --no-fail-fast -- fuzz_
let "all_tests_passed |= $?"
fi
done
- if [ $all_tests_passed -eq 0 ]; then exit 0; fi
- |
curl -X "POST" "https://api.github.com/repos/paritytech/ink/issues?state=all" \
-H "Cookie: logged_in=no" \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Content-Type: text/plain; charset=utf-8" \
-d $'{
"title": "[ci] Failing fuzz tests on master ('"$( date +"%d %b %Y" )"'),
"body": "This CI job '"${CI_JOB_ID}"' just failed: '"${CI_JOB_URL}"'.\n\nThe offending commit is `['"${CI_COMMIT_TITLE}"'](https://github.com/paritytech/ink/commit/'"${CI_COMMIT_SHA}"')`.",
"assignees": [],
"labels": [
"P-high"
]
}'
......@@ -408,7 +408,7 @@ fn pop_complexity_big_o_log_n() -> ink_env::Result<()> {
#[cfg(feature = "ink-fuzz-tests")]
#[quickcheck]
fn pop_always_returns_largest_element(xs: Vec<i32>) {
fn fuzz_pop_always_returns_largest_element(xs: Vec<i32>) {
ink_env::test::run_test::<ink_env::DefaultEnvironment, _>(|_| {
let mut sorted = xs.clone();
sorted.sort();
......
......@@ -65,7 +65,7 @@ fn put_and_take(xs: Vec<i32>, additional_puts_each: u8) {
}
#[quickcheck]
fn repeated_puts_and_takes(xs: Vec<i32>, additional_puts_each: u8) {
fn fuzz_repeated_puts_and_takes(xs: Vec<i32>, additional_puts_each: u8) {
ink_env::test::run_test::<ink_env::DefaultEnvironment, _>(|_| {
put_and_take(xs, additional_puts_each);
Ok(())
......
......@@ -75,7 +75,7 @@ fn insert_and_remove(xs: Vec<i32>, inserts_each: u8) -> StorageHashMap<i32, i32>
}
#[quickcheck]
fn inserts_and_removes(xs: Vec<i32>, inserts_each: u8) {
fn fuzz_inserts_and_removes(xs: Vec<i32>, inserts_each: u8) {
ink_env::test::run_test::<ink_env::DefaultEnvironment, _>(|_| {
let _ = insert_and_remove(xs, inserts_each);
Ok(())
......@@ -86,7 +86,7 @@ fn inserts_and_removes(xs: Vec<i32>, inserts_each: u8) {
/// Inserts all elements from `xs`. Then removes each `xth` element from the map
/// and asserts that all non-`xth` elements are still in the map.
#[quickcheck]
fn removes(xs: Vec<i32>, xth: usize) {
fn fuzz_removes(xs: Vec<i32>, xth: usize) {
ink_env::test::run_test::<ink_env::DefaultEnvironment, _>(|_| {
// given
let xs: Vec<i32> = xs.into_iter().unique().collect();
......@@ -134,7 +134,7 @@ fn removes(xs: Vec<i32>, xth: usize) {
}
#[quickcheck]
fn defrag(xs: Vec<i32>, inserts_each: u8) {
fn fuzz_defrag(xs: Vec<i32>, inserts_each: u8) {
ink_env::test::run_test::<ink_env::DefaultEnvironment, _>(|_| {
// Create a `HashMap<i32, i32>` and execute some pseudo-randomized
// insert/remove operations on it.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment