From 33a0c7dcd3c4377db338ed380ab33c88caeed4bf Mon Sep 17 00:00:00 2001 From: fdb-hiroshima <35889323+fdb-hiroshima@users.noreply.github.com> Date: Tue, 30 Apr 2019 23:30:13 +0200 Subject: [PATCH 01/13] reduce reqwest timeout to 5s (#557) --- plume-common/src/activity_pub/inbox.rs | 5 ++++- plume-common/src/activity_pub/mod.rs | 27 +++++++++++++++----------- plume-models/src/users.rs | 14 +++++++++---- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/plume-common/src/activity_pub/inbox.rs b/plume-common/src/activity_pub/inbox.rs index 9dcc8ddd..9c714614 100644 --- a/plume-common/src/activity_pub/inbox.rs +++ b/plume-common/src/activity_pub/inbox.rs @@ -279,7 +279,10 @@ pub trait FromId: Sized { /// Dereferences an ID fn deref(id: &str) -> Result, Self::Error)> { - reqwest::Client::new() + reqwest::ClientBuilder::new() + .connect_timeout(Some(std::time::Duration::from_secs(5))) + .build() + .map_err(|_| (None, InboxError::DerefError.into()))? .get(id) .header( ACCEPT, diff --git a/plume-common/src/activity_pub/mod.rs b/plume-common/src/activity_pub/mod.rs index 520f5026..c08bf06a 100644 --- a/plume-common/src/activity_pub/mod.rs +++ b/plume-common/src/activity_pub/mod.rs @@ -1,6 +1,6 @@ use activitypub::{Activity, Link, Object}; use array_tool::vec::Uniq; -use reqwest::Client; +use reqwest::ClientBuilder; use rocket::{ http::Status, request::{FromRequest, Request}, @@ -134,16 +134,21 @@ where let body = signed.to_string(); let mut headers = request::headers(); headers.insert("Digest", request::Digest::digest(&body)); - let res = Client::new() - .post(&inbox) - .headers(headers.clone()) - .header( - "Signature", - request::signature(sender, &headers) - .expect("activity_pub::broadcast: request signature error"), - ) - .body(body) - .send(); + let res = ClientBuilder::new() + .connect_timeout(Some(std::time::Duration::from_secs(5))) + .build() + .and_then(|client| { + client + .post(&inbox) + .headers(headers.clone()) + .header( + "Signature", + request::signature(sender, &headers) + .expect("activity_pub::broadcast: request signature error"), + ) + .body(body) + .send() + }); match res { Ok(mut r) => { println!("Successfully sent activity to inbox ({})", inbox); diff --git a/plume-models/src/users.rs b/plume-models/src/users.rs index 098da31b..75305de4 100644 --- a/plume-models/src/users.rs +++ b/plume-models/src/users.rs @@ -23,7 +23,7 @@ use plume_common::activity_pub::{ use plume_common::utils; use reqwest::{ header::{HeaderValue, ACCEPT}, - Client, + ClientBuilder, }; use rocket::{ outcome::IntoOutcome, @@ -267,7 +267,9 @@ impl User { } fn fetch(url: &str) -> Result { - let mut res = Client::new() + let mut res = ClientBuilder::new() + .connect_timeout(Some(std::time::Duration::from_secs(5))) + .build()? .get(url) .header( ACCEPT, @@ -369,7 +371,9 @@ impl User { } pub fn fetch_outbox(&self) -> Result> { - let mut res = Client::new() + let mut res = ClientBuilder::new() + .connect_timeout(Some(std::time::Duration::from_secs(5))) + .build()? .get(&self.outbox_url[..]) .header( ACCEPT, @@ -392,7 +396,9 @@ impl User { } pub fn fetch_followers_ids(&self) -> Result> { - let mut res = Client::new() + let mut res = ClientBuilder::new() + .connect_timeout(Some(std::time::Duration::from_secs(5))) + .build()? .get(&self.followers_endpoint[..]) .header( ACCEPT, -- 2.45.3 From f44254705854e551b5e6ad6c06d5a081e4030c07 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Wed, 1 May 2019 21:32:52 +0200 Subject: [PATCH 02/13] rewrite circleci config with better syntax and try using grcov for coverage --- .circleci/config.yml | 486 ++++++++----------- .circleci/images/plume-buildenv/Dockerfile | 7 +- .circleci/images/plume-buildenv/cargo_config | 2 +- script/run_unit_test.sh | 15 - script/upload_coverage.sh | 3 - 5 files changed, 204 insertions(+), 309 deletions(-) delete mode 100755 script/run_unit_test.sh delete mode 100755 script/upload_coverage.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index d229cfa5..4e463812 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,314 +1,230 @@ version: 2.1 -aliases: - - &plume-docker - image: plumeorg/plume-buildenv:v0.0.5 - - &defaults +executors: + default: + parameters: + postgres: + type: boolean + default: false + selenium: + type: boolean + default: false docker: - - *plume-docker + - image: plumeorg/plume-buildenv:v0.0.6 + - image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<><<^parameters.postgres>>alpine:latest<> + environment: + POSTGRES_USER: postgres + POSTGRES_DB: plume + - image: <<#parameters.postgres>>elgalu/selenium:latest<><<^parameters.postgres>>alpine:latest<> working_directory: ~/projects/Plume - - &postgresql - docker: - - *plume-docker - - image: circleci/postgres:9.6-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_DB: plume - working_directory: ~/projects/Plume - - &selenium - docker: - - *plume-docker - - image: elgalu/selenium:latest - working_directory: ~/projects/Plume - - &postgresql_selenium - docker: - - *plume-docker - - image: circleci/postgres:9.6-alpine - environment: - POSTGRES_USER: postgres - POSTGRES_DB: plume - - image: elgalu/selenium:latest - working_directory: ~/projects/Plume - - - &attach_workspace - attach_workspace: - at: ~/projects/Plume/ - - &persist_to_workspace - persist_to_workspace: - root: ~/projects/Plume/ - paths: - - ./ - - - &env_postgresql environment: - MIGRATION_DIRECTORY: migrations/postgres - FEATURES: postgres - DATABASE_URL: postgres://postgres@localhost/plume - RUST_TEST_THREADS: 1 - - &env_sqlite - environment: - MIGRATION_DIRECTORY: migrations/sqlite - FEATURES: sqlite - DATABASE_URL: plume.sqlite3 RUST_TEST_THREADS: 1 + FEATURES: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + DATABASE_URL: <<#parameters.postgres>>postgres://postgres@localhost/plume<><<^parameters.postgres>>plume.sqlite<> - - &restore_cache - restore_cache: - keys: - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache - save_cache: - key: v2-plume-notest-{{ checksum "Cargo.lock" }} - paths: + +commands: + restore_env: + description: checkout and pull cache + parameters: + cache: + type: enum + default: none + enum: ["none", "clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"] + steps: + - checkout + - run: git config --global --remove-section url."ssh://git@github.com" + - restore_cache: + keys: + - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }} + - v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-master + + cache: + description: push cache + parameters: + cache: + type: enum + enum: ["clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"] + steps: + - save_cache: + key: v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }} + paths: - ~/.cargo/ - ./target - - &restore_cache_web - restore_cache: - keys: - - v2-plume-web-{{ checksum "Cargo.lock" }} - - v2-plume-web - - &save_cache_web - save_cache: - key: v2-plume-web-{{ checksum "Cargo.lock" }} - paths: - - ~/.cargo/ - - ./target - - - &restore_cache_cli - restore_cache: - keys: - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_cli - save_cache: - key: v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli - paths: - - ~/.cargo/ - - target - - - &restore_cache_release - restore_cache: - keys: - - v2-plume-release-{{ checksum "Cargo.lock" }} - - v2-plume-release - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_release - save_cache: - key: v2-plume-release-{{ checksum "Cargo.lock" }} - paths: - - ~/.cargo - - target/release - - target/wasm32-unknown-unknown/release - - - &restore_cache_plume_dead_code - restore_cache: - keys: - - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume - - v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }} - - v2-plume-test-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }} - - v2-plume-notest-{{ checksum "Cargo.lock" }} - - v2-plume-notest - - &save_cache_plume_dead_code - save_cache: - key: v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume - paths: - - ~/.cargo/ - - target - - - &test_cli + clippy: + description: run cargo clippy + parameters: + package: + type: string + default: plume + no_feature: + type: boolean + default: false steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_cli - - run: - name: clippy - command: cargo clippy --no-default-features --features="${FEATURES}" --release -p plume-cli -- -D warnings - - *save_cache_cli - - &test_unit - steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_plume_dead_code - - run: - name: clippy - command: cargo clippy --no-default-features --features="${FEATURES}" --release -- -D warnings - - run: - name: compile test - command: cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run || cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run - - run: - name: run test - command: ./script/run_unit_test.sh - - run: - name: upload coverage - command: ./script/upload_coverage.sh unit - - *save_cache_plume_dead_code + - run: cargo clippy <<#parameters.no_feature>>--no-default-features --features="${FEATURES}"<> --release -p <> -- -D warnings - - &test_browser + run_with_coverage: + description: run command with environment for coverage + parameters: + cmd: + type: string steps: - - *attach_workspace - - run: - name: Set cache key - command: echo "$FEATURES" > /FEATURES - - *restore_cache_plume_dead_code - - run: - name: install server - command: cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force || cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force - - run: - name: install plm - command: cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force || cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force - - run: - name: run test - command: ./script/run_browser_test.sh - environment: - BROWSER: firefox - - run: - name: upload coverage - command: ./script/upload_coverage.sh integration - - *save_cache_plume_dead_code + - run: | + export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads" + export CARGO_INCREMENTAL=0 + << parameters.cmd >> - - &release + upload_coverage: + description: merge coverage files and upload to codecov.io + parameters: + type: + type: string steps: - - *attach_workspace - - *restore_cache_release - - run: - name: build frontend - command: cargo web deploy -p plume-front --release - - run: - name: build server - command: cargo build --release --no-default-features --features="${FEATURES}" || cargo build --release --no-default-features --features="${FEATURES}" - - run: - name: build plm - command: cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli || cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli - - *save_cache_release - - run: script/generate_artifact.sh - - run: script/upload_test_environment.sh - - store_artifacts: - path: plume.tar.gz - destination: plume.tar.gz + - run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'` + - run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info + - run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <> + + build: + description: build a package + parameters: + package: + type: string + default: plume + release: + type: boolean + default: false + steps: + - run: | + cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> || + cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> jobs: - download_deps: - <<: *defaults + cargo fmt: + executor: + name: default steps: - - checkout - - *attach_workspace - - *restore_cache - - run: git config --global --remove-section url."ssh://git@github.com" - - run: cargo fetch - - *save_cache - - *persist_to_workspace + - restore_env + - run: cargo fmt --all -- --check - cargo_fmt: - <<: *defaults + clippy: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> steps: - - *attach_workspace - - run: cargo fmt --all -- --check + - restore_env: + cache: clippy + - clippy + - clippy: + package: plume-cli + - clippy: + package: plume-front + no_feature: true + - cache: + cache: clippy - build_web: - <<: *defaults + unit: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> steps: - - *attach_workspace - - *restore_cache_web - - run: cargo clippy -p plume-front -- -D warnings - - run: cargo web deploy -p plume-front - - *save_cache_web - - *persist_to_workspace + - restore_env: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run_with_coverage: + cmd: | + cargo test --all --exclude plume-front --exclude plume-macro --no-run; + cargo test --all --exclude plume-front --exclude plume-macro; + - upload_coverage: + type: unit + - cache: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - test_cli_postgresql: - <<: *postgresql - <<: *env_postgresql - <<: *test_cli - - test_cli_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *test_cli - - test_unit_postgresql: - <<: *postgresql - <<: *env_postgresql - <<: *test_unit - - test_unit_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *test_unit - - test_browser_postgresql: - <<: *postgresql_selenium - <<: *env_postgresql - <<: *test_browser - - test_browser_sqlite: - <<: *selenium - <<: *env_sqlite - <<: *test_browser - all_ok: - docker: - - image: alpine:3.7 + integration: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> + selenium: true steps: - - run: /bin/true - release_postgresql: - <<: *defaults - <<: *env_postgresql - <<: *release - release_sqlite: - <<: *defaults - <<: *env_sqlite - <<: *release + - restore_env: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: cargo web deploy -p plume-front + - run_with_coverage: + cmd: | + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . || + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . || + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . + - run_with_coverage: + cmd: | + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli || + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli || + cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli + - run: + name: run test + command: ./script/run_browser_test.sh + environment: + BROWSER: firefox + - upload_coverage: + type: integration + - cache: + cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + + release: + parameters: + postgres: + type: boolean + executor: + name: default + postgres: << parameters.postgres >> + steps: + - restore_env: + cache: release-<<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: cargo web deploy -p plume-front --release + - build: + package: plume + release: true + - build: + package: plume-cli + release: true + - cache: + cache: release-<<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> + - run: ./script/generate_artifact.sh + - unless: + condition: << parameters.postgres >> + steps: + - run: ./script/upload_test_environment.sh + - store_artifacts: + path: plume.tar.gz + destination: plume.tar.gz + workflows: version: 2 - build_and_test: + build and test: jobs: - - download_deps - - cargo_fmt: - requires: - - download_deps - - build_web: - requires: - - download_deps - - test_cli_postgresql: - requires: - - download_deps - - test_cli_sqlite: - requires: - - download_deps - - test_unit_postgresql: - requires: - - download_deps - - test_unit_sqlite: - requires: - - download_deps - - test_browser_postgresql: - requires: - - build_web - - test_browser_sqlite: - requires: - - build_web - - all_ok: - requires: - - cargo_fmt - - test_cli_postgresql - - test_cli_sqlite - - test_unit_postgresql - - test_unit_sqlite - - test_browser_postgresql - - test_browser_sqlite - - release_postgresql: - requires: - - all_ok - - release_sqlite: - requires: - - all_ok + - cargo fmt + - clippy: + postgres: false + - clippy: + postgres: true + - unit: + postgres: false + - unit: + postgres: true + - integration: + postgres: false + - integration: + postgres: true + - release: + postgres: false + - release: + postgres: true diff --git a/.circleci/images/plume-buildenv/Dockerfile b/.circleci/images/plume-buildenv/Dockerfile index 4ab51d12..a870cfba 100644 --- a/.circleci/images/plume-buildenv/Dockerfile +++ b/.circleci/images/plume-buildenv/Dockerfile @@ -13,13 +13,10 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0 #compile some deps RUN cargo install cargo-web &&\ + cargo install grcov &&\ + strip /root/.cargo/bin/* &&\ rm -fr ~/.cargo/registry -#install coverage tools -RUN curl -L https://github.com/SimonKagstrom/kcov/archive/master.tar.gz | tar xz &&\ - mkdir -p kcov-master/build && cd kcov-master/build && cmake .. && make &&\ - make install && cd ../.. && rm -rf kcov-master - #set some compilation parametters COPY cargo_config /root/.cargo/config diff --git a/.circleci/images/plume-buildenv/cargo_config b/.circleci/images/plume-buildenv/cargo_config index 10d9285c..889261b1 100644 --- a/.circleci/images/plume-buildenv/cargo_config +++ b/.circleci/images/plume-buildenv/cargo_config @@ -1,3 +1,3 @@ [target.x86_64-unknown-linux-gnu] # link dead code for coverage, attempt to reduce linking memory usage to not get killed -rustflags = ["-Clink-dead-code", "-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"] +rustflags = ["-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"] diff --git a/script/run_unit_test.sh b/script/run_unit_test.sh deleted file mode 100755 index e3ac6657..00000000 --- a/script/run_unit_test.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -set -eo pipefail -for file in target/debug/*-*[^\.d]; do - if [[ -x "$file" ]] - then - filename=$(basename $file) - if [[ $filename =~ ^plume_macro ]]; then - rm $file - continue - fi - mkdir -p "target/cov/$filename" - kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file" - rm $file - fi -done diff --git a/script/upload_coverage.sh b/script/upload_coverage.sh deleted file mode 100755 index b4175297..00000000 --- a/script/upload_coverage.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -eo pipefail -bash <(curl -s https://codecov.io/bash) -F $1 -- 2.45.3 From b26722d829c4a56c1b0a02054832ab2982ec50b2 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Wed, 1 May 2019 23:15:58 +0200 Subject: [PATCH 03/13] attempt to use less memory and to rerun if memory lack --- .circleci/config.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4e463812..c510eba2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -71,7 +71,7 @@ commands: type: string steps: - run: | - export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads" + export RUSTFLAGS="-Zprofile -Zfewer-names -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads -Clink-arg=-Xlinker -Clink-arg=--no-keep-memory -Clink-arg=-Xlinker -Clink-arg=--reduce-memory-overheads" export CARGO_INCREMENTAL=0 << parameters.cmd >> @@ -139,7 +139,9 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cargo test --all --exclude plume-front --exclude plume-macro --no-run; + cargo test --all --exclude plume-front --exclude plume-macro --no-run || + cargo test --all --exclude plume-front --exclude plume-macro --no-run || + cargo test --all --exclude plume-front --exclude plume-macro --no-run cargo test --all --exclude plume-front --exclude plume-macro; - upload_coverage: type: unit @@ -165,9 +167,9 @@ jobs: cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . - run_with_coverage: cmd: | - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli || - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli || - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path plume-cli + cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli || + cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli || + cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli - run: name: run test command: ./script/run_browser_test.sh -- 2.45.3 From 3cbf7a7dfb37b981d1275b21e2d5ed58f61c48b0 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Wed, 1 May 2019 23:48:30 +0200 Subject: [PATCH 04/13] actually remove use of kcov --- script/run_browser_test.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/script/run_browser_test.sh b/script/run_browser_test.sh index 5e93fddf..b3df8adc 100755 --- a/script/run_browser_test.sh +++ b/script/run_browser_test.sh @@ -3,16 +3,12 @@ set -eo pipefail export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -mkdir -p "target/cov/plume" -mkdir -p "target/cov/plm" -plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm' +plm migration run +plm migration redo +plm instance new -d plume-test.local -n plume-test +plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' -$plm migration run -$plm migration redo -$plm instance new -d plume-test.local -n plume-test -$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password' - -kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume & +plume & caddy -conf /Caddyfile & until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null @@ -22,4 +18,4 @@ python3 -m unittest *.py kill -SIGINT %1 kill -SIGKILL %2 -wait +sleep 15 -- 2.45.3 From e13d76dd44de593a85534a682ca26f4d369ed4be Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 00:05:07 +0200 Subject: [PATCH 05/13] use the right var --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c510eba2..2e840a7e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ executors: environment: POSTGRES_USER: postgres POSTGRES_DB: plume - - image: <<#parameters.postgres>>elgalu/selenium:latest<><<^parameters.postgres>>alpine:latest<> + - image: <<#parameters.selenium>>elgalu/selenium:latest<><<^parameters.selenium>>alpine:latest<> working_directory: ~/projects/Plume environment: RUST_TEST_THREADS: 1 -- 2.45.3 From a544f36e2629cc5b4deb35d33194a62513cbc7d4 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 01:16:38 +0200 Subject: [PATCH 06/13] try looping --- .circleci/config.yml | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2e840a7e..357b120e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,9 +96,11 @@ commands: default: false steps: - run: | - cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> || - cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> - + cmd="cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <>" + for i in {1..16}; do + $cmd && exit 0 + done + exit 1 jobs: cargo fmt: @@ -139,10 +141,11 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cargo test --all --exclude plume-front --exclude plume-macro --no-run || - cargo test --all --exclude plume-front --exclude plume-macro --no-run || - cargo test --all --exclude plume-front --exclude plume-macro --no-run - cargo test --all --exclude plume-front --exclude plume-macro; + cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run" + for i in {1..16}; do + $cmd && break + done + cargo test --all --exclude plume-front --exclude plume-macro - upload_coverage: type: unit - cache: @@ -162,14 +165,18 @@ jobs: - run: cargo web deploy -p plume-front - run_with_coverage: cmd: | - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . || - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . || - cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . + cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path ." + for i in {1..16}; do + $cmd && exit 0 + done + exit 1 - run_with_coverage: cmd: | - cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli || - cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli || - cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli + cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli" + for i in {1..16}; do + $cmd && exit 0 + done + exit 1 - run: name: run test command: ./script/run_browser_test.sh -- 2.45.3 From 362ed1d49b842d35d6a8ed4051a7bddd0d57ab98 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 01:34:21 +0200 Subject: [PATCH 07/13] run test serialized --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 357b120e..f7ec080e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -145,7 +145,7 @@ jobs: for i in {1..16}; do $cmd && break done - cargo test --all --exclude plume-front --exclude plume-macro + cargo test --all --exclude plume-front --exclude plume-macro -- --test-threads=1 - upload_coverage: type: unit - cache: -- 2.45.3 From 10f4f578172bcffa1e403d928ef86222e779edf5 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 08:57:41 +0200 Subject: [PATCH 08/13] use feature where we should and make macro path independant --- .circleci/config.yml | 6 +++--- plume-macro/src/lib.rs | 11 ++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f7ec080e..ceaf4cb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,7 +62,7 @@ commands: type: boolean default: false steps: - - run: cargo clippy <<#parameters.no_feature>>--no-default-features --features="${FEATURES}"<> --release -p <> -- -D warnings + - run: cargo clippy <<^parameters.no_feature>>--no-default-features --features="${FEATURES}"<> --release -p <> -- -D warnings run_with_coverage: description: run command with environment for coverage @@ -141,11 +141,11 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run" + cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=\"${FEATURES}\"" for i in {1..16}; do $cmd && break done - cargo test --all --exclude plume-front --exclude plume-macro -- --test-threads=1 + cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -- --test-threads=1 - upload_coverage: type: unit - cache: diff --git a/plume-macro/src/lib.rs b/plume-macro/src/lib.rs index 8ac082da..dcd62514 100644 --- a/plume-macro/src/lib.rs +++ b/plume-macro/src/lib.rs @@ -8,6 +8,7 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use std::fs::{read_dir, File}; use std::io::Read; +use std::path::Path; use std::str::FromStr; #[proc_macro] @@ -20,7 +21,15 @@ pub fn import_migrations(input: TokenStream) -> TokenStream { } else { "migrations" }; - let mut files = read_dir(migration_dir) + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .find(|path| { + path.join(migration_dir).is_dir() || + path.join(".git").exists() + }) + .expect("migrations dir not found") + .join(migration_dir); + let mut files = read_dir(path) .unwrap() .map(|dir| dir.unwrap()) .filter(|dir| dir.file_type().unwrap().is_dir()) -- 2.45.3 From 5cd6cae979abbd7836c4f22614c00dad8b314e1c Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 09:02:44 +0200 Subject: [PATCH 09/13] cargo fmt and remove some '"' --- .circleci/config.yml | 2 +- plume-macro/src/lib.rs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ceaf4cb4..d4701e82 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,7 +141,7 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=\"${FEATURES}\"" + cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES}" for i in {1..16}; do $cmd && break done diff --git a/plume-macro/src/lib.rs b/plume-macro/src/lib.rs index dcd62514..a1966f44 100644 --- a/plume-macro/src/lib.rs +++ b/plume-macro/src/lib.rs @@ -23,10 +23,7 @@ pub fn import_migrations(input: TokenStream) -> TokenStream { }; let path = Path::new(env!("CARGO_MANIFEST_DIR")) .ancestors() - .find(|path| { - path.join(migration_dir).is_dir() || - path.join(".git").exists() - }) + .find(|path| path.join(migration_dir).is_dir() || path.join(".git").exists()) .expect("migrations dir not found") .join(migration_dir); let mut files = read_dir(path) -- 2.45.3 From 70a97de75f0f129cdfef21e2e4c7e2346c731ffe Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 10:23:17 +0200 Subject: [PATCH 10/13] add zip to dependancies --- .circleci/config.yml | 2 +- .circleci/images/plume-buildenv/Dockerfile | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d4701e82..0cfe2fad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ executors: type: boolean default: false docker: - - image: plumeorg/plume-buildenv:v0.0.6 + - image: plumeorg/plume-buildenv:v0.0.7 - image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<><<^parameters.postgres>>alpine:latest<> environment: POSTGRES_USER: postgres diff --git a/.circleci/images/plume-buildenv/Dockerfile b/.circleci/images/plume-buildenv/Dockerfile index a870cfba..47b037f1 100644 --- a/.circleci/images/plume-buildenv/Dockerfile +++ b/.circleci/images/plume-buildenv/Dockerfile @@ -3,8 +3,9 @@ ENV PATH="/root/.cargo/bin:${PATH}" #install native/circleci/build dependancies RUN apt update &&\ - apt install -y git ssh tar gzip ca-certificates &&\ - apt install -y binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip + apt install -y --no-install-recommends git ssh tar gzip ca-certificates &&\ + apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip &&\ + rm -rf /var/lib/apt/lists/* #install and configure rust RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\ -- 2.45.3 From 3f775557b48390a540e08421b7529e034e89315b Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 13:00:42 +0200 Subject: [PATCH 11/13] reduce number of used thread after first failure --- .circleci/config.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0cfe2fad..856e7240 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -96,9 +96,9 @@ commands: default: false steps: - run: | - cmd="cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <>" - for i in {1..16}; do - $cmd && exit 0 + cmd="cargo build <<#parameters.release>>--release<> --no-default-features --features="${FEATURES}" -p <> -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 done exit 1 @@ -141,11 +141,11 @@ jobs: cache: <<#parameters.postgres>>postgres<><<^parameters.postgres>>sqlite<> - run_with_coverage: cmd: | - cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES}" - for i in {1..16}; do - $cmd && break + cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j" + for i in 36 4 2 1 1; do + $cmd $i && break done - cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -- --test-threads=1 + cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -j1 -- --test-threads=1 - upload_coverage: type: unit - cache: @@ -165,16 +165,16 @@ jobs: - run: cargo web deploy -p plume-front - run_with_coverage: cmd: | - cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path ." - for i in {1..16}; do - $cmd && exit 0 + cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 done exit 1 - run_with_coverage: cmd: | - cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli" - for i in {1..16}; do - $cmd && exit 0 + cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli -j" + for i in 36 4 2 1 1; do + $cmd $i && exit 0 done exit 1 - run: -- 2.45.3 From 97c433930e3feba77676e4e5ac58f61fae9d8e41 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Thu, 2 May 2019 22:46:02 +0200 Subject: [PATCH 12/13] push translations to crowdin on commit to master --- .circleci/config.yml | 12 ++++++++++++ .circleci/images/plume-buildenv/Dockerfile | 12 ++++++++++-- crowdin.yml | 2 ++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 856e7240..eb74379c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -215,6 +215,13 @@ jobs: path: plume.tar.gz destination: plume.tar.gz + push translations: + executor: + name: default + steps: + - restore_env: + cache: none + - run: crowdin upload workflows: version: 2 @@ -237,3 +244,8 @@ workflows: postgres: false - release: postgres: true + - push translations: + filters: + branches: + only: + - /^master/ diff --git a/.circleci/images/plume-buildenv/Dockerfile b/.circleci/images/plume-buildenv/Dockerfile index 47b037f1..f962e181 100644 --- a/.circleci/images/plume-buildenv/Dockerfile +++ b/.circleci/images/plume-buildenv/Dockerfile @@ -3,8 +3,8 @@ ENV PATH="/root/.cargo/bin:${PATH}" #install native/circleci/build dependancies RUN apt update &&\ - apt install -y --no-install-recommends git ssh tar gzip ca-certificates &&\ - apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip &&\ + apt install -y --no-install-recommends git ssh tar gzip ca-certificates default-jre&&\ + apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip unzip &&\ rm -rf /var/lib/apt/lists/* #install and configure rust @@ -27,3 +27,11 @@ RUN pip3 install selenium #install and configure caddy RUN curl https://getcaddy.com | bash -s personal COPY Caddyfile /Caddyfile + +#install crowdin +RUN mkdir /crowdin && cd /crowdin &&\ + curl -O https://downloads.crowdin.com/cli/v2/crowdin-cli.zip &&\ + unzip crowdin-cli.zip && rm crowdin-cli.zip &&\ + cd * && mv crowdin-cli.jar /usr/local/bin && cd && rm -rf /crowdin &&\ + /bin/echo -e '#!/bin/sh\njava -jar /usr/local/bin/crowdin-cli.jar $@' > /usr/local/bin/crowdin &&\ + chmod +x /usr/local/bin/crowdin diff --git a/crowdin.yml b/crowdin.yml index c1319cb5..075babd8 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -1,3 +1,5 @@ +"project_identifier": "plume" +"api_key_env": CROWDIN_API_KEY files: - source: /po/plume/plume.pot translation: /po/plume/%two_letters_code%.po -- 2.45.3 From 319c7f6d0146e2822ae46324a67bc198f1f7b7c5 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Fri, 3 May 2019 11:02:50 +0200 Subject: [PATCH 13/13] don't cache result of coverage --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index eb74379c..00b77c19 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -84,6 +84,8 @@ commands: - run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'` - run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info - run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <> + - run: find . -name 'plume*.gc*' -delete -o -name 'plm*.gc*' -delete + - run: rm ccov.zip lcov.info build: description: build a package -- 2.45.3