Plume/Plume
9
18
Fork 24
Code Issues 180 Pull requests 8 Releases 8 Wiki Activity

rewrite circleci config #558

Merged
Plume_migration_agent merged 12 commits from rework-ci into master 2019-05-04 13:35:22 +00:00
Conversation 1 Commits 12 Files changed 8 +253 -322
8 changed files with 253 additions and 322 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

509
.circleci/config.yml
View file

@ -1,314 +1,253 @@
version: 2.1
aliases:
- &plume-docker
image: plumeorg/plume-buildenv:v0.0.5
- &defaults
executors:
default:
parameters:
postgres:
type: boolean
default: false
selenium:
type: boolean
default: false
docker:
- *plume-docker
- image: plumeorg/plume-buildenv:v0.0.7
- image: <<#parameters.postgres>>circleci/postgres:9.6-alpine<</parameters.postgres>><<^parameters.postgres>>alpine:latest<</parameters.postgres>>
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
- image: <<#parameters.selenium>>elgalu/selenium:latest<</parameters.selenium>><<^parameters.selenium>>alpine:latest<</parameters.selenium>>
working_directory: ~/projects/Plume
- &postgresql
docker:
- *plume-docker
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
working_directory: ~/projects/Plume
- &selenium
docker:
- *plume-docker
- image: elgalu/selenium:latest
working_directory: ~/projects/Plume
- &postgresql_selenium
docker:
- *plume-docker
- image: circleci/postgres:9.6-alpine
environment:
POSTGRES_USER: postgres
POSTGRES_DB: plume
- image: elgalu/selenium:latest
working_directory: ~/projects/Plume
- &attach_workspace
attach_workspace:
at: ~/projects/Plume/
- &persist_to_workspace
persist_to_workspace:
root: ~/projects/Plume/
paths:
- ./
- &env_postgresql
environment:
MIGRATION_DIRECTORY: migrations/postgres
FEATURES: postgres
DATABASE_URL: postgres://postgres@localhost/plume
RUST_TEST_THREADS: 1
- &env_sqlite
environment:
MIGRATION_DIRECTORY: migrations/sqlite
FEATURES: sqlite
DATABASE_URL: plume.sqlite3
RUST_TEST_THREADS: 1
FEATURES: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
DATABASE_URL: <<#parameters.postgres>>postgres://postgres@localhost/plume<</parameters.postgres>><<^parameters.postgres>>plume.sqlite<</parameters.postgres>>
- &restore_cache
restore_cache:
keys:
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache
save_cache:
key: v2-plume-notest-{{ checksum "Cargo.lock" }}
paths:
commands:
restore_env:
description: checkout and pull cache
parameters:
cache:
type: enum
default: none
enum: ["none", "clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
steps:
- checkout
- run: git config --global --remove-section url."ssh://git@github.com"
- restore_cache:
keys:
- v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
- v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-master
cache:
description: push cache
parameters:
cache:
type: enum
enum: ["clippy", "postgres", "sqlite", "release-postgres", "release-sqlite"]
steps:
- save_cache:
key: v0-<< parameters.cache >>-{{ checksum "Cargo.lock" }}-{{ .Branch }}
paths:
- ~/.cargo/
- ./target
- &restore_cache_web
restore_cache:
keys:
- v2-plume-web-{{ checksum "Cargo.lock" }}
- v2-plume-web
- &save_cache_web
save_cache:
key: v2-plume-web-{{ checksum "Cargo.lock" }}
paths:
- ~/.cargo/
- ./target
- &restore_cache_cli
restore_cache:
keys:
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_cli
save_cache:
key: v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}-cli
paths:
- ~/.cargo/
- target
- &restore_cache_release
restore_cache:
keys:
- v2-plume-release-{{ checksum "Cargo.lock" }}
- v2-plume-release
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_release
save_cache:
key: v2-plume-release-{{ checksum "Cargo.lock" }}
paths:
- ~/.cargo
- target/release
- target/wasm32-unknown-unknown/release
- &restore_cache_plume_dead_code
restore_cache:
keys:
- v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
- v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}
- v2-plume-test-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}-{{ checksum "/FEATURES" }}
- v2-plume-notest-{{ checksum "Cargo.lock" }}
- v2-plume-notest
- &save_cache_plume_dead_code
save_cache:
key: v2-plume-test-{{ checksum "/FEATURES" }}-{{ checksum "Cargo.lock" }}-plume
paths:
- ~/.cargo/
- target
- &test_cli
clippy:
description: run cargo clippy
parameters:
package:
type: string
default: plume
no_feature:
type: boolean
default: false
steps:
- *attach_workspace
- run:
name: Set cache key
command: echo "$FEATURES" > /FEATURES
- *restore_cache_cli
- run:
name: clippy
command: cargo clippy --no-default-features --features="${FEATURES}" --release -p plume-cli -- -D warnings
- *save_cache_cli
- &test_unit
steps:
- *attach_workspace
- run:
name: Set cache key
command: echo "$FEATURES" > /FEATURES
- *restore_cache_plume_dead_code
- run:
name: clippy
command: cargo clippy --no-default-features --features="${FEATURES}" --release -- -D warnings
- run:
name: compile test
command: cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run || cargo test --no-default-features --features="${FEATURES}" --all --exclude plume-front --no-run
- run:
name: run test
command: ./script/run_unit_test.sh
- run:
name: upload coverage
command: ./script/upload_coverage.sh unit
- *save_cache_plume_dead_code
- run: cargo clippy <<^parameters.no_feature>>--no-default-features --features="${FEATURES}"<</parameters.no_feature>> --release -p <<parameters.package>> -- -D warnings
- &test_browser
run_with_coverage:
description: run command with environment for coverage
parameters:
cmd:
type: string
steps:
- *attach_workspace
- run:
name: Set cache key
command: echo "$FEATURES" > /FEATURES
- *restore_cache_plume_dead_code
- run:
name: install server
command: cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force || cargo install --debug --no-default-features --features="${FEATURES}",test --path . --force
- run:
name: install plm
command: cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force || cargo install --debug --no-default-features --features="${FEATURES}" --path plume-cli --force
- run:
name: run test
command: ./script/run_browser_test.sh
environment:
BROWSER: firefox
- run:
name: upload coverage
command: ./script/upload_coverage.sh integration
- *save_cache_plume_dead_code
- run: |
export RUSTFLAGS="-Zprofile -Zfewer-names -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Zno-landing-pads -Clink-arg=-Xlinker -Clink-arg=--no-keep-memory -Clink-arg=-Xlinker -Clink-arg=--reduce-memory-overheads"
export CARGO_INCREMENTAL=0
<< parameters.cmd >>
- &release
upload_coverage:
description: merge coverage files and upload to codecov.io
parameters:
type:
type: string
steps:
- *attach_workspace
- *restore_cache_release
- run:
name: build frontend
command: cargo web deploy -p plume-front --release
- run:
name: build server
command: cargo build --release --no-default-features --features="${FEATURES}" || cargo build --release --no-default-features --features="${FEATURES}"
- run:
name: build plm
command: cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli || cargo build --release --no-default-features --features="${FEATURES}" -p plume-cli
- *save_cache_release
- run: script/generate_artifact.sh
- run: script/upload_test_environment.sh
- store_artifacts:
path: plume.tar.gz
destination: plume.tar.gz
- run: zip -0 ccov.zip `find . -name 'plume*.gc*' -o -name 'plm*.gc*'`
- run: grcov ccov.zip -s . -t lcov --llvm --branch --ignore-not-existing --ignore-dir '/*' -o lcov.info
- run: bash <(curl -s https://codecov.io/bash) -f lcov.info -F <<parameters.type>>
- run: find . -name 'plume*.gc*' -delete -o -name 'plm*.gc*' -delete
- run: rm ccov.zip lcov.info
build:
description: build a package
parameters:
package:
type: string
default: plume
igalic commented 2019-05-03 14:21:35 +00:00 (Migrated from github.com)
Review
Copy link

that seems excessive

that seems excessive
elegaanz commented 2019-05-03 14:26:18 +00:00 (Migrated from github.com)
Review
Copy link

Yeah, but don't worry, this was an old version, now we only repeat it a few times:

for i in 36 4 2 1 1; do
   ...

where i is the number of threads of the command.

It is to avoid job failures because of OOM errors.

Yeah, but don't worry, this was an old version, now we only repeat it a few times: ```bash for i in 36 4 2 1 1; do ... ``` where `i` is the number of threads of the command. It is to avoid job failures because of OOM errors.
trinity-1686a commented 2019-05-03 14:27:11 +00:00
Review
Copy link

I think you're reviewing an older commit

I think you're reviewing an older commit
release:
type: boolean
default: false
steps:
- run: |
cmd="cargo build <<#parameters.release>>--release<</parameters.release>> --no-default-features --features="${FEATURES}" -p <<parameters.package>> -j"
for i in 36 4 2 1 1; do
$cmd $i && exit 0
done
exit 1
jobs:
download_deps:
<<: *defaults
cargo fmt:
executor:
name: default
steps:
- checkout
- *attach_workspace
- *restore_cache
- run: git config --global --remove-section url."ssh://git@github.com"
- run: cargo fetch
- *save_cache
- *persist_to_workspace
- restore_env
- run: cargo fmt --all -- --check
cargo_fmt:
<<: *defaults
clippy:
parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps:
- *attach_workspace
- run: cargo fmt --all -- --check
- restore_env:
cache: clippy
- clippy
- clippy:
package: plume-cli
- clippy:
package: plume-front
no_feature: true
- cache:
cache: clippy
build_web:
<<: *defaults
unit:
parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps:
- *attach_workspace
- *restore_cache_web
- run: cargo clippy -p plume-front -- -D warnings
- run: cargo web deploy -p plume-front
- *save_cache_web
- *persist_to_workspace
- restore_env:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run_with_coverage:
cmd: |
cmd="cargo test --all --exclude plume-front --exclude plume-macro --no-run --no-default-features --features=${FEATURES} -j"
for i in 36 4 2 1 1; do
$cmd $i && break
done
cargo test --all --exclude plume-front --exclude plume-macro --no-default-features --features="${FEATURES}" -j1 -- --test-threads=1
- upload_coverage:
type: unit
- cache:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
test_cli_postgresql:
<<: *postgresql
<<: *env_postgresql
<<: *test_cli
test_cli_sqlite:
<<: *defaults
<<: *env_sqlite
<<: *test_cli
test_unit_postgresql:
<<: *postgresql
<<: *env_postgresql
<<: *test_unit
test_unit_sqlite:
<<: *defaults
<<: *env_sqlite
<<: *test_unit
test_browser_postgresql:
<<: *postgresql_selenium
<<: *env_postgresql
<<: *test_browser
test_browser_sqlite:
<<: *selenium
<<: *env_sqlite
<<: *test_browser
all_ok:
docker:
- image: alpine:3.7
integration:
parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
selenium: true
steps:
- run: /bin/true
release_postgresql:
<<: *defaults
<<: *env_postgresql
<<: *release
release_sqlite:
<<: *defaults
<<: *env_sqlite
<<: *release
- restore_env:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: cargo web deploy -p plume-front
- run_with_coverage:
cmd: |
cmd="cargo install --debug --no-default-features --features="${FEATURES}",test --force --path . -j"
for i in 36 4 2 1 1; do
$cmd $i && exit 0
done
exit 1
- run_with_coverage:
cmd: |
cmd="cargo install --debug --no-default-features --features="${FEATURES}" --force --path plume-cli -j"
for i in 36 4 2 1 1; do
$cmd $i && exit 0
done
exit 1
- run:
name: run test
command: ./script/run_browser_test.sh
environment:
BROWSER: firefox
- upload_coverage:
type: integration
- cache:
cache: <<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
release:
parameters:
postgres:
type: boolean
executor:
name: default
postgres: << parameters.postgres >>
steps:
- restore_env:
cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: cargo web deploy -p plume-front --release
- build:
package: plume
release: true
- build:
package: plume-cli
release: true
- cache:
cache: release-<<#parameters.postgres>>postgres<</ parameters.postgres>><<^parameters.postgres>>sqlite<</parameters.postgres>>
- run: ./script/generate_artifact.sh
- unless:
condition: << parameters.postgres >>
steps:
- run: ./script/upload_test_environment.sh
- store_artifacts:
path: plume.tar.gz
destination: plume.tar.gz
push translations:
executor:
name: default
steps:
- restore_env:
cache: none
- run: crowdin upload
workflows:
version: 2
build_and_test:
build and test:
jobs:
- download_deps
- cargo_fmt:
requires:
- download_deps
- build_web:
requires:
- download_deps
- test_cli_postgresql:
requires:
- download_deps
- test_cli_sqlite:
requires:
- download_deps
- test_unit_postgresql:
requires:
- download_deps
- test_unit_sqlite:
requires:
- download_deps
- test_browser_postgresql:
requires:
- build_web
- test_browser_sqlite:
requires:
- build_web
- all_ok:
requires:
- cargo_fmt
- test_cli_postgresql
- test_cli_sqlite
- test_unit_postgresql
- test_unit_sqlite
- test_browser_postgresql
- test_browser_sqlite
- release_postgresql:
requires:
- all_ok
- release_sqlite:
requires:
- all_ok
- cargo fmt
- clippy:
postgres: false
- clippy:
postgres: true
- unit:
postgres: false
- unit:
postgres: true
- integration:
postgres: false
- integration:
postgres: true
- release:
postgres: false
- release:
postgres: true
- push translations:
filters:
branches:
only:
- /^master/

20
.circleci/images/plume-buildenv/Dockerfile
View file

@ -3,8 +3,9 @@ ENV PATH="/root/.cargo/bin:${PATH}"
#install native/circleci/build dependancies
RUN apt update &&\
apt install -y git ssh tar gzip ca-certificates &&\
apt install -y binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip
apt install -y --no-install-recommends git ssh tar gzip ca-certificates default-jre&&\
apt install -y --no-install-recommends binutils-dev build-essential cmake curl gcc gettext git libcurl4-openssl-dev libdw-dev libelf-dev libiberty-dev libpq-dev libsqlite3-dev libssl-dev make openssl pkg-config postgresql postgresql-contrib python zlib1g-dev python3-pip zip unzip &&\
rm -rf /var/lib/apt/lists/*
#install and configure rust
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-03-23 -y &&\
@ -13,13 +14,10 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0
#compile some deps
RUN cargo install cargo-web &&\
cargo install grcov &&\
strip /root/.cargo/bin/* &&\
rm -fr ~/.cargo/registry
#install coverage tools
RUN curl -L https://github.com/SimonKagstrom/kcov/archive/master.tar.gz | tar xz &&\
mkdir -p kcov-master/build && cd kcov-master/build && cmake .. && make &&\
make install && cd ../.. && rm -rf kcov-master
#set some compilation parametters
COPY cargo_config /root/.cargo/config
@ -29,3 +27,11 @@ RUN pip3 install selenium
#install and configure caddy
RUN curl https://getcaddy.com | bash -s personal
COPY Caddyfile /Caddyfile
#install crowdin
RUN mkdir /crowdin && cd /crowdin &&\
curl -O https://downloads.crowdin.com/cli/v2/crowdin-cli.zip &&\
unzip crowdin-cli.zip && rm crowdin-cli.zip &&\
cd * && mv crowdin-cli.jar /usr/local/bin && cd && rm -rf /crowdin &&\
/bin/echo -e '#!/bin/sh\njava -jar /usr/local/bin/crowdin-cli.jar $@' > /usr/local/bin/crowdin &&\
chmod +x /usr/local/bin/crowdin

2
.circleci/images/plume-buildenv/cargo_config
View file

@ -1,3 +1,3 @@
[target.x86_64-unknown-linux-gnu]
# link dead code for coverage, attempt to reduce linking memory usage to not get killed
rustflags = ["-Clink-dead-code", "-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"]
rustflags = ["-Clink-args=-Xlinker --no-keep-memory -Xlinker --reduce-memory-overheads"]

2
crowdin.yml
View file

@ -1,3 +1,5 @@
"project_identifier": "plume"
"api_key_env": CROWDIN_API_KEY
files:
- source: /po/plume/plume.pot
translation: /po/plume/%two_letters_code%.po

8
plume-macro/src/lib.rs
View file

@ -8,6 +8,7 @@ use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use std::fs::{read_dir, File};
use std::io::Read;
use std::path::Path;
use std::str::FromStr;
#[proc_macro]
@ -20,7 +21,12 @@ pub fn import_migrations(input: TokenStream) -> TokenStream {
} else {
"migrations"
};
let mut files = read_dir(migration_dir)
let path = Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
.find(|path| path.join(migration_dir).is_dir() || path.join(".git").exists())
.expect("migrations dir not found")
.join(migration_dir);
let mut files = read_dir(path)
.unwrap()
.map(|dir| dir.unwrap())
.filter(|dir| dir.file_type().unwrap().is_dir())

16
script/run_browser_test.sh
View file

@ -3,16 +3,12 @@ set -eo pipefail
export ROCKET_SECRET_KEY="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
mkdir -p "target/cov/plume"
mkdir -p "target/cov/plm"
plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm'
plm migration run
plm migration redo
plm instance new -d plume-test.local -n plume-test
plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
$plm migration run
$plm migration redo
$plm instance new -d plume-test.local -n plume-test
$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume &
plume &
caddy -conf /Caddyfile &
until curl http://localhost:7878/test/health -f; do sleep 1; done 2>/dev/null >/dev/null
@ -22,4 +18,4 @@ python3 -m unittest *.py
kill -SIGINT %1
kill -SIGKILL %2
wait
sleep 15

15
script/run_unit_test.sh
View file

@ -1,15 +0,0 @@
#!/bin/bash
set -eo pipefail
for file in target/debug/*-*[^\.d]; do
if [[ -x "$file" ]]
then
filename=$(basename $file)
if [[ $filename =~ ^plume_macro ]]; then
rm $file
continue
fi
mkdir -p "target/cov/$filename"
kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file"
rm $file
fi
done

3
script/upload_coverage.sh
View file

@ -1,3 +0,0 @@
#!/bin/bash
set -eo pipefail
bash <(curl -s https://codecov.io/bash) -F $1