import migrations and don't require diesel_cli for admins #555

Merged
Plume_migration_agent merged 9 commits from integrated-migrations into master 5 years ago

@ -13,7 +13,6 @@ RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain nightly-2019-0
#compile some deps
RUN cargo install cargo-web &&\
cargo install diesel_cli --no-default-features --features postgres,sqlite --version '=1.3.0' &&\
rm -fr ~/.cargo/registry
#install coverage tools

47
Cargo.lock generated

@ -615,7 +615,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"devise_core 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@ -625,7 +625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bitflags 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -649,7 +649,7 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -809,7 +809,7 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1063,7 +1063,7 @@ dependencies = [
"mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"markup5ever 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -1724,7 +1724,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -1870,6 +1870,15 @@ dependencies = [
"stdweb 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "plume-macro"
version = "0.1.0"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "plume-models"
version = "0.3.0"
@ -1885,10 +1894,12 @@ dependencies = [
"guid-create 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"itertools 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"migrations_internals 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"openssl 0.10.19 (registry+https://github.com/rust-lang/crates.io-index)",
"plume-api 0.3.0",
"plume-common 0.3.0",
"plume-macro 0.1.0",
"reqwest 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)",
@ -1999,7 +2010,7 @@ dependencies = [
[[package]]
name = "quote"
version = "0.6.11"
version = "0.6.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2213,7 +2224,7 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2284,7 +2295,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"devise 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"indexmap 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_http 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2506,7 +2517,7 @@ version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2601,7 +2612,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2614,7 +2625,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"base-x 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_derive 1.0.89 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.39 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2654,7 +2665,7 @@ dependencies = [
"phf_generator 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)",
"phf_shared 0.7.24 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"string_cache_shared 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2704,7 +2715,7 @@ version = "0.15.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -2722,7 +2733,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
@ -3122,7 +3133,7 @@ dependencies = [
"if_chain 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"proc-macro2 0.4.27 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.27 (registry+https://github.com/rust-lang/crates.io-index)",
"validator 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
@ -3473,7 +3484,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a"
"checksum quote 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1eca14c727ad12702eb4b6bfb5a232287dcf8385cb8ca83a3eeaf6519c44c408"
"checksum quote 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9949cfe66888ffe1d53e6ec9d9f3b70714083854be20fd5e271b232a017401e8"
"checksum quote 0.6.11 (registry+https://github.com/rust-lang/crates.io-index)" = "cdd8e04bd9c52e0342b406469d494fcb033be4bdbe5c606016defbb1681411e1"
"checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db"
"checksum r2d2 0.8.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5d746fc8a0dab19ccea7ff73ad535854e90ddb3b4b8cdce953dd5cd0b2e7bd22"
"checksum rand 0.3.23 (registry+https://github.com/rust-lang/crates.io-index)" = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c"
"checksum rand 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"

@ -78,4 +78,4 @@ debug-mailer = []
test = []
[workspace]
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front"]
members = ["plume-api", "plume-cli", "plume-models", "plume-common", "plume-front", "plume-macro"]

@ -18,7 +18,6 @@ RUN chmod a+x ./wasm-deps.sh && sleep 1 && ./wasm-deps.sh
WORKDIR /app
COPY Cargo.toml Cargo.lock rust-toolchain ./
RUN cargo install diesel_cli --no-default-features --features postgres --version '=1.3.0'
RUN cargo install cargo-web
COPY . .
@ -40,7 +39,6 @@ WORKDIR /app
COPY --from=builder /app /app
COPY --from=builder /usr/local/cargo/bin/plm /bin/
COPY --from=builder /usr/local/cargo/bin/plume /bin/
COPY --from=builder /usr/local/cargo/bin/diesel /bin/
CMD ["plume"]

@ -0,0 +1,6 @@
-- This file should undo anything in `up.sql`
--#!|_conn, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! std::fs::remove_dir_all(pb).map_err(Error::from)
--#!}

@ -0,0 +1,10 @@
-- Your SQL goes here
--#!|conn: &Connection, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! let searcher = super::search::Searcher::create(&pb)?;
--#! searcher.fill(conn)?;
--#! searcher.commit();
--#! Ok(())
--#!}

@ -0,0 +1,6 @@
-- This file should undo anything in `up.sql`
--#!|_conn, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! std::fs::remove_dir_all(pb).map_err(Error::from)
--#!}

@ -0,0 +1,10 @@
-- Your SQL goes here
--#!|conn: &Connection, path: &Path| {
--#! let mut pb = path.to_path_buf();
--#! pb.push("search_index");
--#! let searcher = super::search::Searcher::create(&pb)?;
--#! searcher.fill(conn)?;
--#! searcher.commit();
--#! Ok(())
--#!}

@ -10,6 +10,7 @@ use plume_models::{Connection as Conn, CONFIG};
use std::io::{self, prelude::*};
mod instance;
mod migration;
mod search;
mod users;
@ -19,8 +20,9 @@ fn main() {
.version(env!("CARGO_PKG_VERSION"))
.about("Collection of tools to manage your Plume instance.")
.subcommand(instance::command())
.subcommand(users::command())
.subcommand(search::command());
.subcommand(migration::command())
.subcommand(search::command())
.subcommand(users::command());
let matches = app.clone().get_matches();
dotenv::dotenv().ok();
@ -30,12 +32,15 @@ fn main() {
("instance", Some(args)) => {
instance::run(args, &conn.expect("Couldn't connect to the database."))
}
("users", Some(args)) => {
users::run(args, &conn.expect("Couldn't connect to the database."))
("migration", Some(args)) => {
migration::run(args, &conn.expect("Couldn't connect to the database."))
}
("search", Some(args)) => {
search::run(args, &conn.expect("Couldn't connect to the database."))
}
("users", Some(args)) => {
users::run(args, &conn.expect("Couldn't connect to the database."))
}
_ => app.print_help().expect("Couldn't print help"),
};
}

@ -0,0 +1,59 @@
use clap::{App, Arg, ArgMatches, SubCommand};
use plume_models::{migrations::IMPORTED_MIGRATIONS, Connection};
use std::path::Path;
pub fn command<'a, 'b>() -> App<'a, 'b> {
SubCommand::with_name("migration")
.about("Manage migrations")
.subcommand(
SubCommand::with_name("run")
.arg(
Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(false)
.help("Path to Plume's working directory"),
)
.about("Run migrations"),
)
.subcommand(
SubCommand::with_name("redo")
.arg(
Arg::with_name("path")
.short("p")
.long("path")
.takes_value(true)
.required(false)
.help("Path to Plume's working directory"),
)
.about("Rerun latest migration"),
)
}
pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let conn = conn;
match args.subcommand() {
("run", Some(x)) => run_(x, conn),
("redo", Some(x)) => redo(x, conn),
("", None) => command().print_help().unwrap(),
_ => println!("Unknown subcommand"),
}
}
fn run_<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let path = args.value_of("path").unwrap_or(".");
IMPORTED_MIGRATIONS
.run_pending_migrations(conn, Path::new(path))
.expect("Failed to run migrations")
}
fn redo<'a>(args: &ArgMatches<'a>, conn: &Connection) {
let path = args.value_of("path").unwrap_or(".");
IMPORTED_MIGRATIONS
.rerun_last_migration(conn, Path::new(path))
.expect("Failed to rerun migrations")
}

@ -1,7 +1,6 @@
use clap::{App, Arg, ArgMatches, SubCommand};
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use plume_models::{posts::Post, schema::posts, search::Searcher, Connection, CONFIG};
use plume_models::{search::Searcher, Connection, CONFIG};
use std::fs::{read_dir, remove_file};
use std::io::ErrorKind;
use std::path::Path;
@ -98,18 +97,7 @@ fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searche
let path = Path::new(path).join("search_index");
let searcher = searcher.unwrap_or_else(|| Searcher::open(&path).unwrap());
let posts = posts::table
.filter(posts::published.eq(true))
.load::<Post>(conn)
.expect("Post::get_recents: loading error");
let len = posts.len();
for (i, post) in posts.iter().enumerate() {
println!("Importing {}/{} : {}", i + 1, len, post.title);
searcher
.update_document(conn, &post)
.expect("Couldn't import post");
}
searcher.fill(conn).expect("Couldn't import post");
println!("Commiting result");
searcher.commit();
}

@ -0,0 +1,21 @@
[package]
name = "plume-macro"
version = "0.1.0"
authors = ["Trinity Pointard <trinity.pointard@insa-rennes.fr>"]
edition = "2018"
description = "Plume procedural macros"
license = "AGPLv3"
[lib]
proc-macro = true
[dependencies]
proc-macro2 = "0.4"
quote = "0.6.12"
syn = "0.11.4"
[features]
default = []
postgres = []
sqlite = []

@ -0,0 +1,139 @@
#![recursion_limit = "128"]
extern crate proc_macro;
#[macro_use]
extern crate quote;
extern crate syn;
use proc_macro::TokenStream;
use proc_macro2::TokenStream as TokenStream2;
use std::fs::{read_dir, File};
use std::io::Read;
use std::str::FromStr;
#[proc_macro]
pub fn import_migrations(input: TokenStream) -> TokenStream {
assert!(input.is_empty());
let migration_dir = if cfg!(feature = "postgres") {
"migrations/postgres"
} else if cfg!(feature = "sqlite") {
"migrations/sqlite"
} else {
"migrations"
};
let mut files = read_dir(migration_dir)
.unwrap()
.map(|dir| dir.unwrap())
.filter(|dir| dir.file_type().unwrap().is_dir())
.map(|dir| dir.path())
.collect::<Vec<_>>();
files.sort_unstable();
let migrations = files
.into_iter()
.map(|path| {
let mut up = path.clone();
let mut down = path.clone();
up.push("up.sql");
down.push("down.sql");
let mut up_sql = String::new();
let mut down_sql = String::new();
File::open(up).unwrap().read_to_string(&mut up_sql).unwrap();
File::open(down)
.unwrap()
.read_to_string(&mut down_sql)
.unwrap();
let name = path
.file_name()
.unwrap()
.to_str()
.unwrap()
.chars()
.filter(char::is_ascii_digit)
.take(14)
igalic commented 5 years ago (Migrated from github.com)
Review

this seems like a strange thing to take()
what is this whole expression supposed to do?

this seems like a strange thing to `take()` what is this whole expression supposed to do?
Review

diesel store whether migrations were run by storing a 14 digit code. For 2019-03-06-115158_blog_images, it is translated to 20190306115158. To do that I get an iterator over the folder name, filter only digits, and take the first 14 ones (if there are more, they are ignored). See here for take documentation

diesel store whether migrations were run by storing a 14 digit code. For `2019-03-06-115158_blog_images`, it is translated to `20190306115158`. To do that I get an iterator over the folder name, filter only digits, and take the first 14 ones (if there are more, they are ignored). See [here](https://doc.rust-lang.org/std/iter/trait.Iterator.html#method.take) for `take` documentation
.collect::<String>();
(name, up_sql, down_sql)
})
.collect::<Vec<_>>();
let migrations_name = migrations.iter().map(|m| &m.0).collect::<Vec<_>>();
let migrations_up = migrations
.iter()
.map(|m| m.1.as_str())
.map(file_to_migration)
.collect::<Vec<_>>();
let migrations_down = migrations
.iter()
.map(|m| m.2.as_str())
.map(file_to_migration)
.collect::<Vec<_>>();
/*
enum Action {
Sql(&'static str),
Function(&'static Fn(&Connection, &Path) -> Result<()>)
}*/
quote!(
ImportedMigrations(
&[#(ComplexMigration{name: #migrations_name, up: #migrations_up, down: #migrations_down}),*]
)
).into()
}
fn file_to_migration(file: &str) -> TokenStream2 {
let mut sql = true;
let mut acc = String::new();
let mut actions = vec![];
for line in file.lines() {
if sql {
if line.starts_with("--#!") {
if !acc.trim().is_empty() {
actions.push(quote!(Action::Sql(#acc)));
}
sql = false;
acc = line[4..].to_string();
acc.push('\n');
} else if line.starts_with("--") {
continue;
} else {
acc.push_str(line);
acc.push('\n');
}
} else {
if line.starts_with("--#!") {
acc.push_str(&line[4..]);
acc.push('\n');
} else if line.starts_with("--") {
continue;
} else {
let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into());
actions.push(quote!(Action::Function(&#func)));
sql = true;
acc = line.to_string();
acc.push('\n');
}
}
}
if !acc.trim().is_empty() {
if sql {
actions.push(quote!(Action::Sql(#acc)));
} else {
let func: TokenStream2 = trampoline(TokenStream::from_str(&acc).unwrap().into());
actions.push(quote!(Action::Function(&#func)));
}
}
quote!(
&[#(#actions),*]
)
}
/// Build a trampoline to allow reference to closure from const context
fn trampoline(closure: TokenStream2) -> TokenStream2 {
quote! {
{
fn trampoline<'a, 'b>(conn: &'a Connection, path: &'b Path) -> Result<()> {
(#closure)(conn, path)
}
trampoline
}
}
}

@ -13,6 +13,7 @@ guid-create = "0.1"
heck = "0.3.0"
itertools = "0.8.0"
lazy_static = "*"
migrations_internals= "1.4.0"
openssl = "0.10.15"
rocket = "0.4.0"
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }
@ -40,9 +41,12 @@ path = "../plume-api"
[dependencies.plume-common]
path = "../plume-common"
[dependencies.plume-macro]
path = "../plume-macro"
[dev-dependencies]
diesel_migrations = "1.3.0"
[features]
postgres = ["diesel/postgres"]
sqlite = ["diesel/sqlite"]
postgres = ["diesel/postgres", "plume-macro/postgres"]
sqlite = ["diesel/sqlite", "plume-macro/sqlite"]

@ -1,6 +1,7 @@
#![feature(try_trait)]
#![feature(never_type)]
#![feature(custom_attribute)]
#![feature(proc_macro_hygiene)]
extern crate activitypub;
extern crate ammonia;
@ -15,9 +16,12 @@ extern crate heck;
extern crate itertools;
#[macro_use]
extern crate lazy_static;
extern crate migrations_internals;
extern crate openssl;
extern crate plume_api;
extern crate plume_common;
#[macro_use]
extern crate plume_macro;
extern crate reqwest;
extern crate rocket;
extern crate rocket_i18n;
@ -33,10 +37,6 @@ extern crate url;
extern crate webfinger;
extern crate whatlang;
#[cfg(test)]
#[macro_use]
extern crate diesel_migrations;
use plume_common::activity_pub::inbox::InboxError;
#[cfg(not(any(feature = "sqlite", feature = "postgres")))]
@ -305,18 +305,15 @@ mod tests {
use diesel::r2d2::ConnectionManager;
#[cfg(feature = "sqlite")]
use diesel::{dsl::sql_query, RunQueryDsl};
use migrations::IMPORTED_MIGRATIONS;
use plume_common::utils::random_hex;
use scheduled_thread_pool::ScheduledThreadPool;
use search;
use std::env::temp_dir;
use std::sync::Arc;
use Connection as Conn;
use CONFIG;
#[cfg(feature = "sqlite")]
embed_migrations!("../migrations/sqlite");
#[cfg(feature = "postgres")]
embed_migrations!("../migrations/postgres");
#[macro_export]
macro_rules! part_eq {
( $x:expr, $y:expr, [$( $var:ident ),*] ) => {
@ -338,7 +335,10 @@ mod tests {
.connection_customizer(Box::new(db_conn::PragmaForeignKey))
.build(ConnectionManager::<Conn>::new(CONFIG.database_url.as_str()))
.unwrap();
embedded_migrations::run(&*pool.get().unwrap()).expect("Migrations error");
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
IMPORTED_MIGRATIONS
.run_pending_migrations(&pool.get().unwrap(), &dir)
.expect("Migrations error");
pool
};
}
@ -368,6 +368,7 @@ pub mod instance;
pub mod likes;
pub mod medias;
pub mod mentions;
pub mod migrations;
pub mod notifications;
pub mod plume_rocket;
pub mod post_authors;

@ -0,0 +1,122 @@
use Connection;
use Error;
use Result;
use diesel::connection::{Connection as Conn, SimpleConnection};
use migrations_internals::{setup_database, MigrationConnection};
use std::path::Path;
#[allow(dead_code)] //variants might not be constructed if not required by current migrations
enum Action {
Sql(&'static str),
Function(&'static Fn(&Connection, &Path) -> Result<()>),
}
impl Action {
fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
match self {
Action::Sql(sql) => conn.batch_execute(sql).map_err(Error::from),
Action::Function(f) => f(conn, path),
}
}
}
struct ComplexMigration {
name: &'static str,
up: &'static [Action],
down: &'static [Action],
}
impl ComplexMigration {
fn run(&self, conn: &Connection, path: &Path) -> Result<()> {
println!("Running migration {}", self.name);
for step in self.up {
step.run(conn, path)?
}
Ok(())
}
fn revert(&self, conn: &Connection, path: &Path) -> Result<()> {
println!("Reverting migration {}", self.name);
for step in self.down {
step.run(conn, path)?
}
Ok(())
}
}
pub struct ImportedMigrations(&'static [ComplexMigration]);
impl ImportedMigrations {
pub fn run_pending_migrations(&self, conn: &Connection, path: &Path) -> Result<()> {
elegaanz commented 5 years ago (Migrated from github.com)
Review

Should these dbg! be here?

Should these `dbg!` be here?
Review

no :x

no :x
use diesel::dsl::sql;
use diesel::sql_types::Bool;
use diesel::{select, RunQueryDsl};
#[cfg(feature = "postgres")]
let schema_exists: bool = select(sql::<Bool>(
"EXISTS \
(SELECT 1 \
FROM information_schema.tables \
WHERE table_name = '__diesel_schema_migrations')",
))
.get_result(conn)?;
#[cfg(feature = "sqlite")]
let schema_exists: bool = select(sql::<Bool>(
"EXISTS \
(SELECT 1 \
FROM sqlite_master \
WHERE type = 'table' \
AND name = '__diesel_schema_migrations')",
))
.get_result(conn)?;
if !schema_exists {
setup_database(conn)?;
}
let latest_migration = conn.latest_run_migration_version()?;
let latest_id = if let Some(migration) = latest_migration {
self.0
.binary_search_by_key(&migration.as_str(), |mig| mig.name)
.map(|id| id + 1)
.map_err(|_| Error::NotFound)?
} else {
0
};
let to_run = &self.0[latest_id..];
for migration in to_run {
conn.transaction(|| {
migration.run(conn, path)?;
conn.insert_new_migration(migration.name)
.map_err(Error::from)
})?;
}
Ok(())
}
pub fn is_pending(&self, conn: &Connection) -> Result<bool> {
let latest_migration = conn.latest_run_migration_version()?;
if let Some(migration) = latest_migration {
Ok(self.0.last().expect("no migrations found").name != migration)
} else {
Ok(true)
}
}
pub fn rerun_last_migration(&self, conn: &Connection, path: &Path) -> Result<()> {
let latest_migration = conn.latest_run_migration_version()?;
let id = latest_migration
.and_then(|m| self.0.binary_search_by_key(&m.as_str(), |m| m.name).ok())?;
let migration = &self.0[id];
conn.transaction(|| {
migration.revert(conn, path)?;
migration.run(conn, path)
})
}
}
pub const IMPORTED_MIGRATIONS: ImportedMigrations = {
import_migrations! {}
};

@ -1,9 +1,11 @@
use instance::Instance;
use posts::Post;
use schema::posts;
use tags::Tag;
use Connection;
use chrono::Datelike;
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use itertools::Itertools;
use std::{cmp, fs::create_dir_all, path::Path, sync::Mutex};
use tantivy::{
@ -222,6 +224,16 @@ impl Searcher {
.collect()
}
pub fn fill(&self, conn: &Connection) -> Result<()> {
for post in posts::table
.filter(posts::published.eq(true))
.load::<Post>(conn)?
{
self.update_document(conn, &post)?
}
Ok(())
}
pub fn commit(&self) {
let mut writer = self.writer.lock().unwrap();
writer.as_mut().unwrap().commit().unwrap();

@ -1,22 +1,21 @@
extern crate diesel;
#[macro_use]
extern crate diesel_migrations;
extern crate plume_common;
extern crate plume_models;
use diesel::Connection;
use plume_common::utils::random_hex;
use plume_models::migrations::IMPORTED_MIGRATIONS;
use plume_models::{Connection as Conn, CONFIG};
#[cfg(feature = "sqlite")]
embed_migrations!("../migrations/sqlite");
#[cfg(feature = "postgres")]
embed_migrations!("../migrations/postgres");
use std::env::temp_dir;
fn db() -> Conn {
let conn =
Conn::establish(CONFIG.database_url.as_str()).expect("Couldn't connect to the database");
embedded_migrations::run(&conn).expect("Couldn't run migrations");
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
IMPORTED_MIGRATIONS
.run_pending_migrations(&conn, &dir)
.expect("Couldn't run migrations");
conn
}

@ -1,6 +1,5 @@
#!/bin/bash
mkdir bin
cp target/release/{plume,plm} bin
cp "$(which diesel)" bin
strip -s bin/*
tar -cvzf plume.tar.gz bin/ static/ migrations/$FEATURES

@ -7,11 +7,10 @@ mkdir -p "target/cov/plume"
mkdir -p "target/cov/plm"
plm='kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plm plm'
diesel migration run
diesel migration redo
$plm migration run
$plm migration redo
$plm instance new -d plume-test.local -n plume-test
$plm users new -n admin -N 'Admin' -e 'email@exemple.com' -p 'password'
$plm search init
kcov --exclude-pattern=/.cargo,/usr/lib --verify target/cov/plume plume &
caddy -conf /Caddyfile &

@ -4,6 +4,10 @@ for file in target/debug/*-*[^\.d]; do
if [[ -x "$file" ]]
then
filename=$(basename $file)
if [[ $filename =~ ^plume_macro ]]; then
rm $file
continue
fi
mkdir -p "target/cov/$filename"
kcov --exclude-pattern=/.cargo,/usr/lib --verify "target/cov/$filename" "$file"
rm $file

@ -42,6 +42,7 @@ extern crate webfinger;
use diesel::r2d2::ConnectionManager;
use plume_models::{
db_conn::{DbPool, PragmaForeignKey},
migrations::IMPORTED_MIGRATIONS,
search::{Searcher as UnmanagedSearcher, SearcherError},
Connection, Error, CONFIG,
};
@ -84,6 +85,22 @@ fn init_pool() -> Option<DbPool> {
fn main() {
let dbpool = init_pool().expect("main: database pool initialization error");
if IMPORTED_MIGRATIONS
.is_pending(&dbpool.get().unwrap())
.unwrap_or(true)
{
panic!(
r#"
It appear your database migration does not run the migration required
by this version of Plume. To fix this, you can run migrations via
this command:
plm migration run
Then try to restart Plume.
"#
)
}
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
// we want a fast exit here, so
#[allow(clippy::match_wild_err_arm)]

Loading…
Cancel
Save