Unverified Commit 83e5955b authored by Denis_P's avatar Denis_P 🏑 Committed by GitHub
Browse files

CI: chore (#3957)

* CI: chore

* CI: lsof and jq are installed

* CI: generate docs with deps

* CI: remove check width, fmt checks it

* CI: simplify rules

* CI: make CI image variable

* CI: more chore

* rm check_line_width.sh

* CI: fix spellcheck; fix check-transaction

* CI: return installing jq

* lint: remove dupes
parent 0a86997f
Pipeline #165587 passed with stages
in 42 minutes and 46 seconds
......@@ -6,7 +6,7 @@
# setting DEPLOY_TAG will only deploy the tagged image
#
# please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
# &rules-pr-only preset. And "kubernetes-env" with "docker-env" to set a runner
# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
# which executes the job.
stages:
......@@ -15,8 +15,6 @@ stages:
- publish
- deploy
image: paritytech/ci-linux:production
workflow:
rules:
- if: $CI_COMMIT_TAG
......@@ -26,6 +24,7 @@ variables:
GIT_STRATEGY: fetch
GIT_DEPTH: 100
CI_SERVER_NAME: "GitLab CI"
CI_IMAGE: "paritytech/ci-linux:production"
DOCKER_OS: "debian:stretch"
ARCH: "x86_64"
VAULT_SERVER_URL: "https://vault.parity-mgmt-vault.parity.io"
......@@ -57,6 +56,7 @@ default:
- kubernetes-parity-build
.docker-env: &docker-env
image: "${CI_IMAGE}"
retry:
max: 2
when:
......@@ -78,16 +78,16 @@ default:
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- when: always
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
.pr-only: &rules-pr-only
# these jobs run only on PRs
.rules-test-and-rococo: &rules-test-and-rococo
rules:
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
when: never
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
#### Vault secrets
.vault-secrets: &vault-secrets
......@@ -144,7 +144,7 @@ check-runtime:
stage: test
image: paritytech/tools:latest
<<: *kubernetes-env
<<: *rules-pr-only
<<: *rules-test
<<: *vault-secrets
variables:
GITLAB_API: "https://gitlab.parity.io/api/v4"
......@@ -153,19 +153,10 @@ check-runtime:
- ./scripts/gitlab/check_runtime.sh
allow_failure: true
check-line-width:
stage: test
image: paritytech/tools:latest
<<: *kubernetes-env
<<: *rules-pr-only
script:
- ./scripts/gitlab/check_line_width.sh
allow_failure: true
cargo-fmt:
stage: test
<<: *kubernetes-env
<<: *rules-pr-only
<<: *docker-env
<<: *rules-test
script:
- cargo +nightly fmt --all -- --check
allow_failure: true
......@@ -239,11 +230,12 @@ check-no-default-features:
spellcheck:
stage: test
<<: *docker-env
<<: *rules-pr-only
<<: *rules-test
script:
- cargo spellcheck --version
# compare with the commit parent to the PR, given it's from a default branch
- git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
- echo "___Spellcheck is going to check your diff___"
- cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
- time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
$(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
......@@ -254,11 +246,7 @@ build-adder-collator:
<<: *collect-artifacts
<<: *docker-env
<<: *compiler-info
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
<<: *rules-test-and-rococo
script:
- time cargo build --release --verbose -p test-parachain-adder-collator
- sccache -s
......@@ -294,18 +282,19 @@ check-dependent-cumulus:
DEPENDENT_REPO: cumulus
check-transaction-versions:
# image must be ubuntu:20.04 based to match the linkers, this image has npm installed
image: paritytech/contracts-ci-linux:production
stage: build
<<: *rules-test
<<: *docker-env
<<: *vault-secrets
image: node:15
needs:
- job: test-build-linux-stable
artifacts: true
before_script:
- apt-get -y update; apt-get -y install jq lsof
- npm install --ignore-scripts -g @polkadot/metadata-cmp
# Set git config
- git config remote.origin.url "https://github.com/paritytech/polkadot.git"
- git fetch origin release
script:
- scripts/gitlab/check_extrinsics_ordering.sh
......@@ -334,7 +323,7 @@ build-rustdoc:
- ./crate-docs/
script:
# FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
- time cargo doc --no-deps --workspace --verbose
- time cargo doc --workspace --verbose
- rm -f ./target/doc/.lock
- mv ./target/doc ./crate-docs
# FIXME: remove me after CI image gets nonroot
......@@ -416,16 +405,12 @@ publish-adder-collator-image:
# service image for Simnet
stage: build
<<: *build-push-image
<<: *rules-test-and-rococo
variables:
<<: *image-variables
# scripts/dockerfiles/collator_injected.Dockerfile
DOCKERFILE: dockerfiles/collator_injected.Dockerfile
IMAGE_NAME: docker.io/paritypr/colander
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: build-adder-collator
artifacts: true
......@@ -439,22 +424,20 @@ publish-adder-collator-image:
# this artifact is used in simnet-tests job
dotenv: ./artifacts/collator.env
.update_weights: &update-weights
update_polkadot_weights: &update-weights
stage: build
when: manual
tags:
- weights
variables:
RUNTIME: polkadot
artifacts:
paths:
- ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
script: |
./scripts/run_benches_for_runtime.sh $RUNTIME
git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
update_polkadot_weights:
<<: *update-weights
script:
- ./scripts/run_benches_for_runtime.sh $RUNTIME
- git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
# uses the "shell" executors
tags:
- weights
update_kusama_weights:
<<: *update-weights
......@@ -547,6 +530,7 @@ publish-rustdoc:
- git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
echo "___Nothing to commit___"
- git push origin gh-pages --force
- echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
after_script:
- rm -rf .git/ ./*
......@@ -555,7 +539,6 @@ publish-rustdoc:
deploy-polkasync-kusama:
stage: deploy
rules:
# former .rules-build
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
when: never
- if: $CI_PIPELINE_SOURCE == "schedule"
......@@ -570,13 +553,9 @@ simnet-tests:
stage: deploy
image: "${SIMNET_IMAGE}"
<<: *kubernetes-env
<<: *rules-test-and-rococo
variables:
GH_DIR: "https://github.com/paritytech/polkadot/tree/master/simnet_tests"
rules:
- if: $CI_PIPELINE_SOURCE == "schedule"
- if: $CI_COMMIT_REF_NAME == "master"
- if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
- if: $CI_COMMIT_REF_NAME == "rococo-v1"
needs:
- job: publish-polkadot-image
- job: publish-adder-collator-image
......
# Contributing
`Substrate` projects is a **OPENISH Open Source Project**
## What?
Individuals making significant and valuable contributions are given commit-access to a project to contribute as they see fit. A project is more like an open wiki than a standard guarded open source project.
## Rules
There are a few basic ground-rules for contributors (including the maintainer(s) of the project):
......@@ -16,25 +10,25 @@ There are a few basic ground-rules for contributors (including the maintainer(s)
- A pull-request _must not be merged until CI_ has finished successfully.
- Contributors should adhere to the [house coding style](https://github.com/paritytech/polkadot/wiki/Style-Guide).
#### Merging pull requests once CI is successful:
### Merging pull requests once CI is successful
- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/substrate/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
- A pull request that does not alter any logic (e.g. comments, dependencies, docs) may be tagged [`insubstantial`](https://github.com/paritytech/polkadot/pulls?utf8=%E2%9C%93&q=is%3Apr+is%3Aopen+label%3AA2-insubstantial) and merged by its author.
- A pull request with no large change to logic that is an urgent fix may be merged after a non-author contributor has reviewed it well.
- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/substrate/pulls?q=is%3Apr+is%3Aopen+label%3AA0-pleasereview) tag in order to garner feedback.
- All other PRs should sit for 48 hours with the [`pleasereview`](https://github.com/paritytech/polkadot/pulls?q=is:pr+is:open+label:A0-pleasereview) tag in order to garner feedback.
- No PR should be merged until all reviews' comments are addressed.
#### Reviewing pull requests:
### Reviewing pull requests
When reviewing a pull request, the end-goal is to suggest useful changes to the author. Reviews should finish with approval unless there are issues that would result in:
- Buggy behaviour.
- Buggy behavior.
- Undue maintenance burden.
- Breaking with house coding style.
- Pessimisation (i.e. reduction of speed as measured in the projects benchmarks).
- Pessimization (i.e. reduction of speed as measured in the projects benchmarks).
- Feature reduction (i.e. it removes some aspect of functionality that a significant minority of users rely on).
- Uselessness (i.e. it does not strictly add a feature or fix a known issue).
#### Reviews may not be used as an effective veto for a PR because:
### Reviews may not be used as an effective veto for a PR because
- There exists a somewhat cleaner/better/faster way of accomplishing the same feature/fix.
- It does not fit well with some other contributors' longer-term vision for the project.
......@@ -49,4 +43,4 @@ This is an experiment and feedback is welcome! This document may also be subject
## Heritage
These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: https://github.com/Level/community/blob/master/CONTRIBUTING.md
These contributing guidelines are modified from the "OPEN Open Source Project" guidelines for the Level project: <https://github.com/Level/community/blob/master/CONTRIBUTING.md>
......@@ -33,7 +33,7 @@ the command-line.
Currently supports Debian 10 (Buster) and Ubuntu 20.04 (Focal), and
derivatives. Run the following commands as the `root` user.
```
```bash
# Import the security@parity.io GPG key
gpg --recv-keys --keyserver hkps://keys.mailvelope.com 9D4B2B6EB8F97156D19669A9FF0812D491B96798
gpg --export 9D4B2B6EB8F97156D19669A9FF0812D491B96798 > /usr/share/keyrings/parity.gpg
......@@ -52,7 +52,7 @@ apt install polkadot
Currently supports Fedora 32 and CentOS 8, and derivatives.
```
```bash
# Install dnf-plugins-core (This might already be installed)
dnf install dnf-plugins-core
# Add the repository and enable it
......@@ -67,7 +67,7 @@ dnf install polkadot
### Install via Cargo
Make sure you have the support software installed from the **Build from Source** section
Make sure you have the support software installed from the **Build from Source** section
below this section.
If you want to install Polkadot in your PATH, you can do so with with:
......@@ -244,9 +244,11 @@ polkadot --chain=polkadot-local --bob -d /tmp/bob --port 30334 --bootnodes '/ip4
Ensure you replace `ALICE_BOOTNODE_ID_HERE` with the node ID from the output of the first terminal.
### Using Docker
[Using Docker](doc/docker.md)
### Shell Completion
[Shell Completion](doc/shell-completion.md)
## Contributing
......
# Shell completion
## Shell completion
The Polkadot cli command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.
The Polkadot CLI command supports shell auto-completion. For this to work, you will need to run the completion script matching you build and system.
Assuming you built a release version using `cargo build --release` and use `bash` run the following:
......@@ -19,7 +18,7 @@ You can find completion scripts for:
To make this change persistent, you can proceed as follow:
### First install
## First install
```bash
COMPL_DIR=$HOME/.completion
......@@ -29,7 +28,7 @@ echo "source $COMPL_DIR/polkadot.bash" >> $HOME/.bash_profile
source $HOME/.bash_profile
```
### Update
## Update
When you build a new version of Polkadot, the following will ensure you auto-completion script matches the current binary:
......
......@@ -183,51 +183,50 @@ The implementation is yet to be completed, see the [implementation PR](https://g
struct BehaveMaleficient;
impl OverseerGen for BehaveMaleficient {
fn generate<'a, Spawner, RuntimeClient>(
&self,
connector: OverseerConnector,
args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
where
RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
Spawner: 'static + SpawnNamed + Clone + Unpin,
{
let spawner = args.spawner.clone();
let leaves = args.leaves.clone();
let runtime_client = args.runtime_client.clone();
let registry = args.registry.clone();
let candidate_validation_config = args.candidate_validation_config.clone();
// modify the subsystem(s) as needed:
let all_subsystems = create_default_subsystems(args)?.
fn generate<'a, Spawner, RuntimeClient>(
&self,
args: OverseerGenArgs<'a, Spawner, RuntimeClient>,
) -> Result<(Overseer<Spawner, Arc<RuntimeClient>>, OverseerHandler), Error>
where
RuntimeClient: 'static + ProvideRuntimeApi<Block> + HeaderBackend<Block> + AuxStore,
RuntimeClient::Api: ParachainHost<Block> + BabeApi<Block> + AuthorityDiscoveryApi<Block>,
Spawner: 'static + SpawnNamed + Clone + Unpin,
{
let spawner = args.spawner.clone();
let leaves = args.leaves.clone();
let runtime_client = args.runtime_client.clone();
let registry = args.registry.clone();
let candidate_validation_config = args.candidate_validation_config.clone();
// modify the subsystem(s) as needed:
let all_subsystems = create_default_subsystems(args)?.
// or spawn an entirely new set
replace_candidate_validation(
// create the filtered subsystem
FilteredSubsystem::new(
CandidateValidationSubsystem::with_config(
candidate_validation_config,
Metrics::register(registry)?,
),
// create the filtered subsystem
FilteredSubsystem::new(
CandidateValidationSubsystem::with_config(
candidate_validation_config,
Metrics::register(registry)?,
),
// an implementation of
Skippy::default(),
),
);
Skippy::default(),
),
);
Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner, connector)
.map_err(|e| e.into())
Overseer::new(leaves, all_subsystems, registry, runtime_client, spawner)
.map_err(|e| e.into())
// A builder pattern will simplify this further
// WIP https://github.com/paritytech/polkadot/pull/2962
}
}
}
fn main() -> eyre::Result<()> {
color_eyre::install()?;
let cli = Cli::from_args();
assert_matches::assert_matches!(cli.subcommand, None);
polkadot_cli::run_node(cli, BehaveMaleficient)?;
Ok(())
color_eyre::install()?;
let cli = Cli::from_args();
assert_matches::assert_matches!(cli.subcommand, None);
polkadot_cli::run_node(cli, BehaveMaleficient)?;
Ok(())
}
```
......
#!/bin/sh
#
# check if line width of rust source files is not beyond x characters
#
set -e
BASE_BRANCH="origin/master"
LINE_WIDTH="121"
GOOD_LINE_WIDTH="101"
git diff --name-only "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- \*.rs :^bridges | ( while read -r file
do
if [ ! -f "${file}" ];
then
echo "Skipping removed file."
elif git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${LINE_WIDTH}\}"
then
if [ -z "${FAIL}" ]
then
echo "| warning!"
echo "| Lines should not be longer than 120 characters."
echo "| "
echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
echo "|"
FAIL="true"
fi
echo "| file: ${file}"
git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
| grep -n "^+.\{${LINE_WIDTH}\}"
echo "|"
else
if git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" | grep -q "^+.\{${GOOD_LINE_WIDTH}\}"
then
if [ -z "${FAIL}" ]
then
echo "| warning!"
echo "| Lines should be longer than 100 characters only in exceptional circumstances!"
echo "| "
echo "| see more https://wiki.parity.io/Substrate-Style-Guide"
echo "|"
fi
echo "| file: ${file}"
git diff "${BASE_BRANCH}...${CI_COMMIT_SHA}" -- "${file}" \
| grep -n "^+.\{${LINE_WIDTH}\}"
echo "|"
fi
fi
done
test -z "${FAIL}"
)
150
A&V
accessor/MS
AccountId
activations
......@@ -15,7 +16,6 @@ assignee/SM
async
asynchrony
autogenerated
A&V
backable
backend/MS
benchmark/DSMG
......@@ -47,6 +47,7 @@ customizable/B
DDoS
Debian/M
decodable/MS
decrement
deduplicated
deinitializing
dequeue/SD
......@@ -56,6 +57,7 @@ DHT
disincentivize/D
dispatchable/SM
DLEQ
DM
DMP/SM
DMQ
DoS
......@@ -78,6 +80,7 @@ fedora/M
finalize/B
FRAME/MS
FSMs
functor
fungibility
gameable
getter/MS
......@@ -156,21 +159,24 @@ mutex
natively
NFA
NFT/SM
no_std
nonces
NPoS
no_std
NTB
offboard/DMSG
onboard/DMSG
oneshot/MS
onwards
OOM/S
OPENISH
others'
ourself
overseer/MS
ownerless
p2p
parablock/MS
parachain/MS
ParaId
parameterization
parameterize/D
parathread/MS
......@@ -178,6 +184,7 @@ passthrough
PDK
peerset/MS
permission/D
pessimization
phragmen
picosecond/SM
PoA/MS
......@@ -201,7 +208,6 @@ proxy/G
proxying
PRs
PVF/S
p2p
README/MS
redhat/M
register/CD
......@@ -217,10 +223,9 @@ rpc
RPC/MS
runtime/MS
rustc/MS
sybil
SAFT
scalable
scalability
scalable
Schnorr
schnorrkel
SDF
......@@ -230,8 +235,8 @@ shareable
Simnet/MS
spawn/SR
spawner
SS58
sr25519
SS58
SSL
startup/MS
stateful
......@@ -243,18 +248,19 @@ subsystem/MS
subsystems'
supermajority
SURI
sybil
systemwide
taskmanager/MS
TCP
teleport/D
teleport/RG
teleports
teleportation/SM
teleporter/SM
teleporters
teleports
testnet/MS
timestamp/MS
timeframe
timestamp/MS
tradeoff
transitionary
trie/MS
......@@ -309,7 +315,4 @@ XCM/S
XCMP/M
yeet
yml
decrement
DM
ParaId
functor
zsh
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment