.gitlab-ci.yml 25.4 KB
Newer Older
1
2
# .gitlab-ci.yml
#
3
4
# polkadot
#
5
6
# pipelines can be triggered manually in the web
# setting DEPLOY_TAG will only deploy the tagged image
7
8
#
# please do not add new jobs without "rules:" and "*-env". There are &rules-test for everything,
Denis_P's avatar
Denis_P committed
9
# &rules-test-and-rococo preset. And "kubernetes-env" with "docker-env" to set a runner
10
# which executes the job.
11

gabriel klawitter's avatar
gabriel klawitter committed
12
13
14
stages:
  - test
  - build
15
16
17
  - publish
  - deploy

18
19
20
21
22
workflow:
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH

gabriel klawitter's avatar
gabriel klawitter committed
23
variables:
Denis_P's avatar
Denis_P committed
24
  GIT_STRATEGY:                    fetch
25
  GIT_DEPTH:                       100
gabriel klawitter's avatar
gabriel klawitter committed
26
  CI_SERVER_NAME:                  "GitLab CI"
Denis_P's avatar
Denis_P committed
27
  CI_IMAGE:                        "paritytech/ci-linux:production"
Denis_P's avatar
Denis_P committed
28
  DOCKER_OS:                       "debian:stretch"
29
  ARCH:                            "x86_64"
30
  ZOMBIENET_IMAGE:                 "docker.io/paritypr/zombienet"
31
32
33
  VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
  VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
  VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
34
  PIPELINE_SCRIPTS_TAG:            "v0.1"
35

36
37
38
default:
  cache:                           {}

39
.collect-artifacts:                &collect-artifacts
gabriel klawitter's avatar
gabriel klawitter committed
40
41
42
  artifacts:
    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
    when:                          on_success
43
    expire_in:                     28 days
gabriel klawitter's avatar
gabriel klawitter committed
44
    paths:
Denis_P's avatar
Denis_P committed
45
      - ./artifacts/
gabriel klawitter's avatar
gabriel klawitter committed
46

47
.kubernetes-env:                   &kubernetes-env
Denis_P's avatar
Denis_P committed
48
49
50
51
52
53
54
  retry:
    max: 2
    when:
      - runner_system_failure
      - unknown_failure
      - api_failure
  interruptible:                   true
55
56
  tags:
    - kubernetes-parity-build
gabriel klawitter's avatar
gabriel klawitter committed
57

58
.docker-env:                       &docker-env
Denis_P's avatar
Denis_P committed
59
  image:                           "${CI_IMAGE}"
60
61
  retry:
    max: 2
62
    when:
63
64
65
      - runner_system_failure
      - unknown_failure
      - api_failure
66
  interruptible:                   true
67
68
69
  tags:
    - linux-docker

70
.compiler-info:                    &compiler-info
Denis_P's avatar
Denis_P committed
71
72
73
74
  before_script:
    - rustup show
    - cargo --version
    - sccache -s
gabriel klawitter's avatar
gabriel klawitter committed
75

76
77
.rules-test:                       &rules-test
  # these jobs run always*
78
  rules:
79
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
80
      when: never
Denis_P's avatar
Denis_P committed
81
82
83
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
84

Denis_P's avatar
Denis_P committed
85
.rules-test-and-rococo:            &rules-test-and-rococo
86
  rules:
87
    - if: $CI_PIPELINE_SOURCE == "schedule"
Denis_P's avatar
Denis_P committed
88
    - if: $CI_COMMIT_REF_NAME == "master"
89
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
Denis_P's avatar
Denis_P committed
90
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
91

92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
#### Vault secrets
.vault-secrets:                    &vault-secrets
  secrets:
    AWS_ACCESS_KEY_ID:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/AWS_ACCESS_KEY_ID@kv
      file:                        false
    AWS_SECRET_ACCESS_KEY:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/AWS_SECRET_ACCESS_KEY@kv
      file:                        false
    DOCKER_HUB_USER:
      vault:                       cicd/gitlab/parity/DOCKER_HUB_USER@kv
      file:                        false
    DOCKER_HUB_PASS:
      vault:                       cicd/gitlab/parity/DOCKER_HUB_PASS@kv
      file:                        false
    GITHUB_PR_TOKEN:
      vault:                       cicd/gitlab/parity/GITHUB_PR_TOKEN@kv
      file:                        false
    GITHUB_USER:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_USER@kv
      file:                        false
    GITHUB_RELEASE_TOKEN:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_RELEASE_TOKEN@kv
      file:                        false
    GITHUB_TOKEN:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/GITHUB_TOKEN@kv
118
      file:                        false
119
120
121
122
123
    MATRIX_ACCESS_TOKEN:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ACCESS_TOKEN@kv
      file:                        false
    MATRIX_ROOM_ID:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/MATRIX_ROOM_ID@kv
124
      file:                        false
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    PARITYPR_USER:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_USER@kv
      file:                        false
    PARITYPR_PASS:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PARITYPR_PASS@kv
      file:                        false
    PIPELINE_TOKEN:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/PIPELINE_TOKEN@kv
      file:                        false
    REL_MAN_ROOM_ID:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/REL_MAN_ROOM_ID@kv
      file:                        false
    SSH_PRIVATE_KEY:
      vault:                       cicd/gitlab/$CI_PROJECT_PATH/SSH_PRIVATE_KEY@kv
      file:                        false

141
142
143
144
#### stage:                        test

check-runtime:
  stage:                           test
Denis_P's avatar
Denis_P committed
145
  image:                           paritytech/tools:latest
146
  <<:                              *kubernetes-env
Denis_P's avatar
Denis_P committed
147
  <<:                              *rules-test
148
  <<:                              *vault-secrets
149
150
151
152
153
154
155
  variables:
    GITLAB_API:                    "https://gitlab.parity.io/api/v4"
    GITHUB_API_PROJECT:            "parity%2Finfrastructure%2Fgithub-api"
  script:
    - ./scripts/gitlab/check_runtime.sh
  allow_failure:                   true

Shawn Tabrizi's avatar
Shawn Tabrizi committed
156
157
cargo-fmt:
  stage:                           test
Denis_P's avatar
Denis_P committed
158
159
  <<:                              *docker-env
  <<:                              *rules-test
Shawn Tabrizi's avatar
Shawn Tabrizi committed
160
  script:
161
    - cargo +nightly --version
Shawn Tabrizi's avatar
Shawn Tabrizi committed
162
163
164
    - cargo +nightly fmt --all -- --check
  allow_failure:                   true

165
166
test-deterministic-wasm:
  stage:                           test
167
  <<:                              *rules-test
168
  <<:                              *docker-env
169
  <<:                              *compiler-info
170
  <<:                              *vault-secrets
171
  script:
172
    - ./scripts/gitlab/test_deterministic_wasm.sh
173

Denis_P's avatar
Denis_P committed
174
test-build-linux-stable:
gabriel klawitter's avatar
gabriel klawitter committed
175
  stage:                           test
176
  <<:                              *docker-env
177
  <<:                              *compiler-info
Denis_P's avatar
Denis_P committed
178
  <<:                              *collect-artifacts
179
  <<:                              *vault-secrets
180
181
182
183
  variables:
    RUST_TOOLCHAIN: stable
    # Enable debug assertions since we are running optimized builds for testing
    # but still want to have debug assertions.
184
    RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
Denis_P's avatar
Denis_P committed
185
186
187
188
189
190
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
191
  script:
192
    - ./scripts/gitlab/test_linux_stable.sh
Denis_P's avatar
Denis_P committed
193
    # we're using the bin built here, instead of having a parallel `build-linux-release`
194
    # disputes feature is needed for zombie-net parachains malus test
195
    - time cargo build --release --verbose --bin polkadot --features "disputes"
Denis_P's avatar
Denis_P committed
196
    - sccache -s
Denis_P's avatar
Denis_P committed
197
    # pack artifacts
Denis_P's avatar
Denis_P committed
198
199
200
201
202
203
204
205
206
207
    - mkdir -p ./artifacts
    - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name
    - mv ./target/release/polkadot ./artifacts/.
    - sha256sum ./artifacts/polkadot | tee ./artifacts/polkadot.sha256
    - EXTRATAG="$(./artifacts/polkadot --version |
        sed -n -r 's/^polkadot ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p')"
    - EXTRATAG="${CI_COMMIT_REF_NAME}-${EXTRATAG}-$(cut -c 1-8 ./artifacts/polkadot.sha256)"
    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
    - echo -n ${VERSION} > ./artifacts/VERSION
    - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG
Alexander Samusev's avatar
Alexander Samusev committed
208
    - cp -r scripts/* ./artifacts
209

210
check-runtime-benchmarks:
211
  stage:                           test
212
  <<:                              *rules-test
213
  <<:                              *docker-env
214
  <<:                              *compiler-info
215
  <<:                              *vault-secrets
216
217
  script:
    # Check that the node will compile with `runtime-benchmarks` feature flag.
218
    - ./scripts/gitlab/check_runtime_benchmarks.sh
219
220
    - sccache -s

221
222
223
224
225
226
227
228
229
230
231
check-no-default-features:
  stage:                           test
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *compiler-info
  <<:                              *vault-secrets
  script:
    # Check that polkadot-cli will compile no default features.
    - ./scripts/gitlab/check_no_default_features.sh
    - sccache -s

Denis_P's avatar
Denis_P committed
232
233
234
spellcheck:
  stage:                           test
  <<:                              *docker-env
Denis_P's avatar
Denis_P committed
235
  <<:                              *rules-test
Denis_P's avatar
Denis_P committed
236
237
238
239
  script:
    - cargo spellcheck --version
    # compare with the commit parent to the PR, given it's from a default branch
    - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH}
Denis_P's avatar
Denis_P committed
240
    - echo "___Spellcheck is going to check your diff___"
241
    - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
Shawn Tabrizi's avatar
Shawn Tabrizi committed
242
    - time cargo spellcheck check -vvv --cfg=scripts/gitlab/spellcheck.toml --checkers hunspell --code 1
243
        $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges))
Denis_P's avatar
Denis_P committed
244
245
  allow_failure:                   true

246
247
248
249
250
build-adder-collator:
  stage:                           test
  <<:                              *collect-artifacts
  <<:                              *docker-env
  <<:                              *compiler-info
Denis_P's avatar
Denis_P committed
251
  <<:                              *rules-test-and-rococo
252
253
254
255
256
257
258
259
260
  script:
    - time cargo build --release --verbose -p test-parachain-adder-collator
    - sccache -s
    # pack artifacts
    - mkdir -p ./artifacts
    - mv ./target/release/adder-collator ./artifacts/.
    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
    - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
Alexander Samusev's avatar
Alexander Samusev committed
261
    - cp -r scripts/* ./artifacts
Denis_P's avatar
Denis_P committed
262

263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
build-malus:
  stage:                           test
  <<:                              *collect-artifacts
  <<:                              *docker-env
  <<:                              *compiler-info
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
  script:
    - time cargo build --release --verbose -p polkadot-test-malus --features disputes
    - sccache -s
    # pack artifacts
    - mkdir -p ./artifacts
    - mv ./target/release/malus ./artifacts/.
    - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION
    - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG
    - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))"
    - cp -r scripts/* ./artifacts

283
284
#### stage:                        build

285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
.check-dependent-project:          &check-dependent-project
  stage:                           build
  <<:                              *docker-env
  <<:                              *vault-secrets
  script:
    - git clone
        --depth=1
        "--branch=$PIPELINE_SCRIPTS_TAG"
        https://github.com/paritytech/pipeline-scripts
    - ./pipeline-scripts/check_dependent_project.sh
        paritytech
        polkadot
        --polkadot
        "$DEPENDENT_REPO"
        "$GITHUB_PR_TOKEN"

check-dependent-cumulus:
  <<: *check-dependent-project
  variables:
    DEPENDENT_REPO: cumulus

306
307
308
309
310
check-transaction-versions:
  stage:                           build
  <<:                              *rules-test
  <<:                              *docker-env
  <<:                              *vault-secrets
Denis_P's avatar
Denis_P committed
311
  image:                           node:15
312
313
314
315
316
317
  needs:
    - job:                         test-build-linux-stable
      artifacts:                   true
  before_script:
    - apt-get -y update; apt-get -y install jq lsof
    - npm install --ignore-scripts -g @polkadot/metadata-cmp
Denis_P's avatar
Denis_P committed
318
319
    # Set git config
    - git config remote.origin.url "https://github.com/paritytech/polkadot.git"
320
321
322
323
    - git fetch origin release
  script:
    - scripts/gitlab/check_extrinsics_ordering.sh

324
generate-impl-guide:
325
326
327
  stage:                           build
  <<:                              *rules-test
  <<:                              *docker-env
328
  image:
329
    name: michaelfbryan/mdbook-docker-image:v0.4.4
330
331
    entrypoint: [""]
  script:
332
    - mdbook build roadmap/implementers-guide
333

Denis_P's avatar
Denis_P committed
334
335
336
337
338
339
340
341
342
343
344
345
346
347
build-rustdoc:
  stage:                           build
  <<:                              *docker-env
  <<:                              *rules-test
  variables:
    SKIP_WASM_BUILD:               1
  artifacts:
    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
    when:                          on_success
    expire_in:                     7 days
    paths:
    - ./crate-docs/
  script:
    # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
Denis_P's avatar
Denis_P committed
348
    - time cargo doc --workspace --verbose
Denis_P's avatar
Denis_P committed
349
350
351
352
353
354
355
    - rm -f ./target/doc/.lock
    - mv ./target/doc ./crate-docs
    # FIXME: remove me after CI image gets nonroot
    - chown -R nonroot:nonroot ./crate-docs
    - echo "<meta http-equiv=refresh content=0;url=polkadot_service/index.html>" > ./crate-docs/index.html
    - sccache -s

Denis_P's avatar
Denis_P committed
356
.build-push-image:                 &build-push-image
357
  <<:                              *kubernetes-env
358
  <<:                              *vault-secrets
359
  image:                           quay.io/buildah/stable
Denis_P's avatar
Denis_P committed
360
  variables:                       &image-variables
361
    GIT_STRATEGY:                  none
Denis_P's avatar
Denis_P committed
362
363
    DOCKER_USER:                   ${PARITYPR_USER}
    DOCKER_PASS:                   ${PARITYPR_PASS}
364
  before_script:                   &check-versions
365
    - test -s ./artifacts/VERSION || exit 1
366
367
368
    - test -s ./artifacts/EXTRATAG || exit 1
    - VERSION="$(cat ./artifacts/VERSION)"
    - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
Denis_P's avatar
Denis_P committed
369
    - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
370
  script:
371
    - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
372
        ( echo "no docker credentials provided"; exit 1 )
373
    - cd ./artifacts
Denis_P's avatar
Denis_P committed
374
    - buildah bud
375
376
377
        --format=docker
        --build-arg VCS_REF="${CI_COMMIT_SHA}"
        --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
Denis_P's avatar
Denis_P committed
378
        --build-arg IMAGE_NAME="${IMAGE_NAME}"
379
        --tag "$IMAGE_NAME:$VERSION"
Denis_P's avatar
Denis_P committed
380
381
        --tag "$IMAGE_NAME:$EXTRATAG"
        --file ${DOCKERFILE} .
382
    # The job will success only on the protected branch
383
384
    - echo "$DOCKER_PASS" |
        buildah login --username "$DOCKER_USER" --password-stdin docker.io
Denis_P's avatar
Denis_P committed
385
    - buildah info
386
387
    - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
    - buildah push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
388
  after_script:
389
    - buildah logout --all
390

Denis_P's avatar
Denis_P committed
391
publish-polkadot-image:
392
  stage:                           build
Denis_P's avatar
Denis_P committed
393
  <<:                              *build-push-image
394
395
  variables:
    <<:                            *image-variables
Alexander Samusev's avatar
Alexander Samusev committed
396
397
    # scripts/dockerfiles/polkadot_injected_debug.Dockerfile
    DOCKERFILE:                    dockerfiles/polkadot_injected_debug.Dockerfile
398
    IMAGE_NAME:                    docker.io/paritypr/synth-wave
399
  rules:
400
401
    # Don't run on releases - this is handled by the Github Action here:
    # .github/workflows/publish-docker-release.yml
402
403
    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
      when: never
404
405
406
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
407
408
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
      variables:
Denis_P's avatar
Denis_P committed
409
        <<:                        *image-variables
410
        IMAGE_NAME:                docker.io/parity/rococo
411
412
        DOCKER_USER:               ${DOCKER_HUB_USER}
        DOCKER_PASS:               ${DOCKER_HUB_PASS}
Denis_P's avatar
Denis_P committed
413
  needs:
Denis_P's avatar
Denis_P committed
414
    - job:                         test-build-linux-stable
Denis_P's avatar
Denis_P committed
415
      artifacts:                   true
Radu Popa's avatar
Radu Popa committed
416
  after_script:
417
    # pass artifacts to the zombienet-tests job
Radu Popa's avatar
Radu Popa committed
418
419
420
421
422
    # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
    - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
    - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
  artifacts:
    reports:
423
      # this artifact is used in zombienet-tests job
Radu Popa's avatar
Radu Popa committed
424
      dotenv: ./artifacts/parachains.env
Denis_P's avatar
Denis_P committed
425

426
427
428
429
publish-adder-collator-image:
  # service image for Simnet
  stage:                           build
  <<:                              *build-push-image
Denis_P's avatar
Denis_P committed
430
  <<:                              *rules-test-and-rococo
431
432
  variables:
    <<:                            *image-variables
Alexander Samusev's avatar
Alexander Samusev committed
433
434
    # scripts/dockerfiles/collator_injected.Dockerfile
    DOCKERFILE:                    dockerfiles/collator_injected.Dockerfile
435
436
437
438
439
    IMAGE_NAME:                    docker.io/paritypr/colander
  needs:
    - job:                         build-adder-collator
      artifacts:                   true
  after_script:
440
    - buildah logout --all
441
    # pass artifacts to the zombienet-tests job
Radu Popa's avatar
Radu Popa committed
442
443
    - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
    - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
444
445
  artifacts:
    reports:
446
      # this artifact is used in zombienet-tests job
447
      dotenv: ./artifacts/collator.env
448

449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
publish-malus-image:
  # service image for Simnet
  stage:                           build
  <<:                              *build-push-image
  variables:
    <<:                            *image-variables
    # scripts/dockerfiles/malus.Dockerfile
    DOCKERFILE:                    dockerfiles/malus.Dockerfile
    IMAGE_NAME:                    docker.io/paritypr/malus
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
  needs:
    - job:                         build-malus
      artifacts:                   true
  after_script:
    - buildah logout "$IMAGE_NAME"
    # pass artifacts to the zombienet-tests job
    - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env
    - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env
  artifacts:
    reports:
      # this artifact is used in zombienet-tests job
      dotenv: ./artifacts/malus.env

Denis_P's avatar
Denis_P committed
475
update_polkadot_weights:           &update-weights
476
477
478
479
480
481
482
  stage:                           build
  when:                            manual
  variables:
    RUNTIME:                       polkadot
  artifacts:
    paths:
      - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
Denis_P's avatar
Denis_P committed
483
484
485
486
487
488
  script:
    - ./scripts/run_benches_for_runtime.sh $RUNTIME
    - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch
  # uses the "shell" executors
  tags:
    - weights
489
490
491
492
493
494
495
496
497
498
499

update_kusama_weights:
  <<:                              *update-weights
  variables:
    RUNTIME:                       kusama

update_westend_weights:
  <<:                              *update-weights
  variables:
    RUNTIME:                       westend

500
501
#### stage:                        publish

Denis_P's avatar
Denis_P committed
502
publish-s3-release:                &publish-s3
503
504
  stage:                           publish
  needs:
Denis_P's avatar
Denis_P committed
505
    - job:                         test-build-linux-stable
506
507
      artifacts:                   true
  <<:                              *kubernetes-env
508
  <<:                              *vault-secrets
Denis_P's avatar
Denis_P committed
509
  image:                           paritytech/awscli:latest
510
511
  variables:
    GIT_STRATEGY:                  none
Denis_P's avatar
Denis_P committed
512
    PREFIX:                        "builds/polkadot/${ARCH}-${DOCKER_OS}"
513
  rules:
Denis_P's avatar
Denis_P committed
514
    # publishing binaries nightly
515
    - if: $CI_PIPELINE_SOURCE == "schedule"
516
517
  before_script:
    - *check-versions
518
  script:
Denis_P's avatar
Denis_P committed
519
520
521
    - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}"
    - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/
    - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}"
522
    - find ./artifacts -type f | while read file; do
Denis_P's avatar
Denis_P committed
523
524
525
526
        name="${file#./artifacts/}";
        aws s3api copy-object
          --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name}
          --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name};
527
      done
528
529
530
    - |
      cat <<-EOM
      |
531
      |  polkadot binary paths:
532
      |
Denis_P's avatar
Denis_P committed
533
534
      |  - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot
      |  - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot
535
536
      |
      EOM
537
  after_script:
Denis_P's avatar
Denis_P committed
538
    - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/
539
        --recursive --human-readable --summarize
540

Denis_P's avatar
Denis_P committed
541
542
543
publish-rustdoc:
  stage:                           publish
  <<:                              *kubernetes-env
544
  <<:                              *vault-secrets
Denis_P's avatar
Denis_P committed
545
546
547
548
549
550
  image:                           paritytech/tools:latest
  variables:
    GIT_DEPTH:                     100
  rules:
    - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME == "master"
Denis_P's avatar
Denis_P committed
551
552
553
554
555
  # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other
  # artifacts from being dowloaded by this job.
  needs:
    - job:                         build-rustdoc
      artifacts:                   true
Denis_P's avatar
Denis_P committed
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
  script:
    - rm -rf /tmp/*
    # Set git config
    - rm -rf .git/config
    - git config user.email "devops-team@parity.io"
    - git config user.name "${GITHUB_USER}"
    - git config remote.origin.url "https://${GITHUB_TOKEN}@github.com/paritytech/polkadot.git"
    - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*"
    - git fetch origin gh-pages
    # Save README and docs
    - cp -r ./crate-docs/ /tmp/doc/
    - cp README.md /tmp/doc/
    - git checkout gh-pages
    # Remove everything and restore generated docs and README
    - rm -rf ./*
    - mv /tmp/doc/* .
    # Upload files
    - git add --all --force
    # `git commit` has an exit code of > 0 if there is nothing to commit.
    # This causes GitLab to exit immediately and marks this job failed.
    # We don't want to mark the entire job failed if there's nothing to
    # publish though, hence the `|| true`.
    - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" ||
        echo "___Nothing to commit___"
    - git push origin gh-pages --force
Denis_P's avatar
Denis_P committed
581
    - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___"
Denis_P's avatar
Denis_P committed
582
583
584
  after_script:
    - rm -rf .git/ ./*

585
586
#### stage:                        deploy

587
588
deploy-polkasync-kusama:
  stage:                           deploy
589
590
591
592
593
  rules:
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
      when: never
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
594
  variables:
595
    POLKADOT_CI_COMMIT_NAME:       "${CI_COMMIT_REF_NAME}"
Denis_P's avatar
Denis_P committed
596
    POLKADOT_CI_COMMIT_REF:        "${CI_COMMIT_SHORT_SHA}"
597
  allow_failure:                   true
598
  trigger:                         "parity/infrastructure/parity-testnet"
599

600
zombienet-tests-parachains-smoke-test:
601
  stage:                           deploy
602
  image:                           "${ZOMBIENET_IMAGE}"
603
  <<:                              *kubernetes-env
604
605
606
607
608
609
610
611
612
613
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
  needs:
    - job:                         publish-polkadot-image
    - job:                         publish-malus-image
    - job:                         publish-adder-collator-image

614
  variables:
615
    GH_DIR:                        'https://github.com/paritytech/polkadot/tree/master/zombienet_tests/parachains'
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645

  before_script:
    - echo "Zombie-net Tests Config"
    - echo "${ZOMBIENET_IMAGE_NAME}"
    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
    - echo "${GH_DIR}"
    - export DEBUG=zombie,zombie::network-node
    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}

  script:
    - /home/nonroot/zombie-net/scripts/run-test-env-manager.sh
        --github-remote-dir="${GH_DIR}"
        --test="0001-parachains-smoke-test.feature"
  allow_failure:                   true
  retry: 2
  tags:
    - zombienet-polkadot-integration-test

zombienet-tests-malus-dispute-valid:
  stage:                           deploy
  image:                           "${ZOMBIENET_IMAGE}"
  <<:                              *kubernetes-env
  rules:
    - if: $CI_PIPELINE_SOURCE == "schedule"
    - if: $CI_COMMIT_REF_NAME == "master"
    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
    - if: $CI_COMMIT_REF_NAME == "rococo-v1"
646
647
  needs:
    - job:                         publish-polkadot-image
648
    - job:                         publish-malus-image
649
    - job:                         publish-adder-collator-image
Radu Popa's avatar
Radu Popa committed
650

651
  variables:
652
    GH_DIR:                        'https://github.com/paritytech/polkadot/tree/master/node/malus/integrationtests'
653

Radu Popa's avatar
Radu Popa committed
654
  before_script:
655
656
    - echo "Zombie-net Tests Config"
    - echo "${ZOMBIENET_IMAGE_NAME}"
657
    - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}"
658
    - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}"
659
    - echo "${GH_DIR}"
660
661
662
663
664
    - export DEBUG=zombie,zombie::network-node
    - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}
    - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG}
    - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}

665
  script:
666
    - /home/nonroot/zombie-net/scripts/run-test-env-manager.sh
667
        --github-remote-dir="${GH_DIR}"
668
        --test="0001-dispute-valid-block.feature"
669
  allow_failure:                   true
Radu Popa's avatar
Radu Popa committed
670
671
  retry: 2
  tags:
672
    - zombienet-polkadot-integration-test