diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..d8db2f757fd634b3ce84d710004edf71bdda19c9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +target +Dockerfile +.dockerignore +.git +.gitignore \ No newline at end of file diff --git a/.github/workflows/fileserver.yml b/.github/workflows/fileserver.yml new file mode 100644 index 0000000000000000000000000000000000000000..909fa17e3e69dc9e1e1fe70c40ef72058f476055 --- /dev/null +++ b/.github/workflows/fileserver.yml @@ -0,0 +1,50 @@ +name: File server build & image publish +run-name: Deploy ${{github.ref}} + +on: + push: + branches: + - main + paths: + - "Cargo.toml" + - "crates/file-server/**" + +env: + PROJECT_ID: "parity-zombienet" + GCR_REGISTRY: "europe-west3-docker.pkg.dev" + GCR_REPOSITORY: "zombienet-public-images" + +jobs: + build_and_push: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup gcloud CLI + uses: google-github-actions/setup-gcloud@v2.0.1 + with: + service_account_key: ${{ secrets.GCP_SA_KEY }} + project_id: ${{ env.PROJECT_ID }} + export_default_credentials: true + + - name: Login to GCP + uses: google-github-actions/auth@v2.0.1 + with: + credentials_json: ${{ secrets.GCP_SA_KEY }} + + - name: Artifact registry authentication + run: | + gcloud auth configure-docker ${{ env.GCR_REGISTRY }} + + - name: Build, tag, and push image to GCP Artifact registry + id: build-image + env: + IMAGE: "${{ env.GCR_REGISTRY }}/${{ env.GCR_PROJECT_ID }}/zombienet-file-server" + + run: | + docker build -t $IMAGE:${{ github.sha }} -f ./crates/file-server/Dockerfile . + docker tag $IMAGE:${{ github.sha }} $IMAGE:latest + docker push --all-tags $IMAGE + echo "image=$IMAGE:${ github.sha }" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 5df2f40067df8d564599aa70333c8ff69af1e02d..4de74d161399745a0ee0ca8e45fc1f924d3e0008 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/provider", "crates/test-runner", "crates/prom-metrics-parser", + "crates/file-server" ] [workspace.package] @@ -28,6 +29,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" toml = "0.7" tokio = "1.28" +tokio-util = "0.7" reqwest = "0.11" regex = "1.8" lazy_static = "1.4" @@ -43,9 +45,14 @@ hex = "0.4" sp-core = "22.0.0" libp2p = { version = "0.52" } subxt = "0.32.0" -subxt-signer = { version = "0.32.0", features = ["subxt"]} +subxt-signer = { version = "0.32.0", features = ["subxt"] } tracing = "0.1.35" pjs-rs = "0.1.2" +axum = { version = "0.7" } +axum-extra = { version = "0.9" } +tower = { version = "0.4" } +tower-http = { version = "0.5" } +tracing-subscriber = { version = "0.3" } # Zombienet workspace crates: support = { package = "zombienet-support", version = "0.1.0-alpha.0", path = "crates/support" } diff --git a/crates/file-server/Cargo.toml b/crates/file-server/Cargo.toml new file mode 100644 index 0000000000000000000000000000000000000000..4734fb3165f5bd4d768a928118f0147dab366c84 --- /dev/null +++ b/crates/file-server/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "zombienet-file-server" +authors.workspace = true +edition.workspace = true +version.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { workspace = true, features = ["multipart"] } +axum-extra = { workspace = true } +tokio = { workspace = true, features = ["full"] } +tokio-util = { workspace = true, features = ["io"] } +tower = { workspace = true, features = ["util"] } +tower-http = { workspace = true, features = ["fs", "trace"] } +futures = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true, features = ["env-filter"] } diff --git a/crates/file-server/Dockerfile b/crates/file-server/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..414faa5e496374c0b3686a4fd2da7496e75c6ba1 --- /dev/null +++ b/crates/file-server/Dockerfile @@ -0,0 +1,20 @@ +# build stage +FROM rust:1.75-alpine as builder + +WORKDIR /tmp + +COPY . . + +RUN apk add musl-dev + +RUN cargo build --release -p zombienet-file-server + +# run stage +FROM alpine:latest + +ENV LISTENING_ADDRESS 127.0.0.1:80 +ENV UPLOADS_DIRECTORY /uploads + +COPY --from=builder /tmp/target/release/zombienet-file-server /usr/local/bin/file-server + +CMD ["file-server"] \ No newline at end of file diff --git a/crates/file-server/src/main.rs b/crates/file-server/src/main.rs new file mode 100644 index 0000000000000000000000000000000000000000..42a86160aca87dcae13362043d1f80cf67188e45 --- /dev/null +++ b/crates/file-server/src/main.rs @@ -0,0 +1,87 @@ +#![allow(clippy::expect_fun_call)] +use std::io; + +use axum::{ + extract::{Path, Request, State}, + http::StatusCode, + routing::post, + Router, +}; +use futures::TryStreamExt; +use tokio::{fs::File, io::BufWriter, net::TcpListener}; +use tokio_util::io::StreamReader; +use tower_http::services::ServeDir; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +#[derive(Clone)] +struct AppState { + uploads_directory: String, +} + +#[tokio::main] +async fn main() { + let address = + std::env::var("LISTENING_ADDRESS").expect("LISTENING_ADDRESS env variable isn't defined"); + let uploads_directory = + std::env::var("UPLOADS_DIRECTORY").expect("UPLOADS_DIRECTORY env variable isn't defined"); + + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer()) + .init(); + + tokio::fs::create_dir_all(&uploads_directory) + .await + .expect(&format!("failed to create '{uploads_directory}' directory")); + + let app = Router::new() + .route( + "/*file_path", + post(upload).get_service(ServeDir::new(&uploads_directory)), + ) + .with_state(AppState { uploads_directory }); + + let listener = TcpListener::bind(&address) + .await + .expect(&format!("failed to listen on {address}")); + tracing::info!("file server started on {}", listener.local_addr().unwrap()); + axum::serve(listener, app).await.unwrap() +} + +async fn upload( + Path(file_path): Path<String>, + State(state): State<AppState>, + request: Request, +) -> Result<(), (StatusCode, String)> { + if !path_is_valid(&file_path) { + return Err((StatusCode::BAD_REQUEST, "Invalid path".to_owned())); + } + + async { + let path = std::path::Path::new(&state.uploads_directory).join(file_path); + + if let Some(parent_dir) = path.parent() { + tokio::fs::create_dir_all(parent_dir).await?; + } + + let stream = request.into_body().into_data_stream(); + let body_with_io_error = stream.map_err(|err| io::Error::new(io::ErrorKind::Other, err)); + let body_reader = StreamReader::new(body_with_io_error); + futures::pin_mut!(body_reader); + + let mut file = BufWriter::new(File::create(&path).await?); + tokio::io::copy(&mut body_reader, &mut file).await?; + + tracing::info!("created file '{}'", path.to_string_lossy()); + + Ok::<_, io::Error>(()) + } + .await + .map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string())) +} + +fn path_is_valid(path: &str) -> bool { + let path = std::path::Path::new(path); + let mut components = path.components().peekable(); + + components.all(|component| matches!(component, std::path::Component::Normal(_))) +} diff --git a/crates/support/src/process/fake.rs b/crates/support/src/process/fake.rs index d11e4a2092066f6a852c390abe9a6e31309c23e1..cf16726a7cfaed1f50a9b9812dccc93ce957ed8e 100644 --- a/crates/support/src/process/fake.rs +++ b/crates/support/src/process/fake.rs @@ -205,7 +205,7 @@ impl FakeProcessManager { .await .processes .values() - .map(Arc::clone) + .cloned() .collect() }