diff --git a/Cargo.lock b/Cargo.lock index 84a06f19..5debe77e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,6 +544,21 @@ dependencies = [ "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "config" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "nom 5.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "yaml-rust 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "conv" version = "0.3.3" @@ -709,6 +724,14 @@ name = "custom_derive" version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "dashmap" +version = "4.0.0-rc6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "data-encoding" version = "2.1.2" @@ -1890,6 +1913,15 @@ dependencies = [ "safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "linked-hash-map" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "linked-hash-map" version = "0.5.3" @@ -2261,6 +2293,14 @@ dependencies = [ "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "num-traits" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "num-traits" version = "0.2.12" @@ -2549,6 +2589,7 @@ dependencies = [ "plume-api 0.4.0", "plume-common 0.4.0", "plume-models 0.4.0", + "riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_contrib 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=29910f2829e7e590a540da3804336577b48c7b31)", @@ -2660,6 +2701,7 @@ dependencies = [ "plume-common 0.4.0", "plume-macro 0.4.0", "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", + "riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)", "scheduled-thread-pool 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3127,6 +3169,36 @@ dependencies = [ "winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "riker" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "chrono 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", + "config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", + "dashmap 4.0.0-rc6 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "riker-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "riker-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ring" version = "0.13.5" @@ -3254,6 +3326,11 @@ dependencies = [ "nom 5.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "rust-ini" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "rust-stemmers" version = "1.2.0" @@ -3355,6 +3432,11 @@ name = "semver-parser" version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "serde" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "serde" version = "1.0.114" @@ -3363,6 +3445,18 @@ dependencies = [ "serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde-hjson" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_derive" version = "1.0.114" @@ -3394,6 +3488,14 @@ dependencies = [ "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_test" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_urlencoded" version = "0.5.5" @@ -3468,6 +3570,32 @@ name = "slab" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "slog" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "slog-scope" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "slog-stdlog" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)", + "slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)", + "slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "smallvec" version = "0.6.13" @@ -4128,6 +4256,14 @@ dependencies = [ "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "toml" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tower-service" version = "0.3.0" @@ -4656,6 +4792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum cloudabi 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" "checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59" "checksum combine 4.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b8e5ef862b2df927249f4e2bdc29c1bd13a33105f900884b0c32acdf32aff584" +"checksum config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3" "checksum conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299" "checksum cookie 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5795cda0897252e34380a27baf884c53aa7ad9990329cdad96d4c5d027015d44" "checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5" @@ -4673,6 +4810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" "checksum ctrlc 3.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "54dedab740bc412d514cfbc4a1d9d5d16fed02c4b14a7be129003c07fdc33b9b" "checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9" +"checksum dashmap 4.0.0-rc6 (registry+https://github.com/rust-lang/crates.io-index)" = "308a6703be2d759cb5fb7b80a23547fe73a8d5ebf70d3a4ca7f0ef4c0bfc2265" "checksum data-encoding 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4f47ca1860a761136924ddd2422ba77b2ea54fe8cc75b9040804a0d9d32ad97" "checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum debug-builders 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f5d8e3d14cabcb2a8a59d7147289173c6ada77a0bc526f6b85078f941c0cf12" @@ -4799,6 +4937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum lindera-ipadic-builder 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a617422d3b9fa3bcaef470423f6b686aa8954e7081bd148ced8d8cf3c08925b6" "checksum lindera-tantivy 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b1bd76e0573b2f28cfeb16f2ee50f8cc6ea14a1e307f328be1d9744053a97633" "checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9" +"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd" "checksum linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a" "checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" "checksum lock_api 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" @@ -4839,6 +4978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd" "checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b" "checksum num-rational 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31" "checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" "checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum object 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" @@ -4918,6 +5058,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" "checksum reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680" "checksum reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "f88643aea3c1343c804950d7bf983bd2067f5ab59db6d613a08e05572f2714ab" +"checksum riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b07140216b4c980a887315331822cc63f6d22f824b23dbff1751aee655a365" +"checksum riker-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2a8e8f71c9e7980a596c39c7e3537ea8563054526e15712a610ac97a02dba15" "checksum ring 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2c4db68a2e35f3497146b7e4563df7d4773a2433230c5e4b448328e31740458a" "checksum rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6130967b369cfb8411b0b73e96fcba1229c32a9cc6f295d144f879bfced13c6e" "checksum rocket_codegen 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "cb852e6da168fb948a8f2b798ba2e2f0e4fc860eae0efa9cf2bf0f5466bb0425" @@ -4928,6 +5070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rpassword 4.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "99371657d3c8e4d816fb6221db98fa408242b0b53bac08f8676a41f8554fe99f" "checksum rsass 0.9.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4534cc03040beacd2668621815f26fe57e5b7cfe085790f98e5e87c1612316" "checksum ructe 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c85620b8046f88a870d93d90fa56904dec76cc79139bfcc22e71e87f0cd2169f" +"checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2" "checksum rust-stemmers 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" "checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" "checksum rustc-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" @@ -4942,10 +5085,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum security-framework-sys 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405" "checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" "checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +"checksum serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8" "checksum serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3" +"checksum serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8" "checksum serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e" "checksum serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3" "checksum serde_qs 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d43eef44996bbe16e99ac720e1577eefa16f7b76b5172165c98ced20ae9903e1" +"checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5" "checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a" "checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" "checksum sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" @@ -4955,6 +5101,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +"checksum slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cc9c640a4adbfbcc11ffb95efe5aa7af7309e002adab54b185507dbf2377b99" +"checksum slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c44c89dd8b0ae4537d1ae318353eaf7840b4869c536e31c41e963d1ea523ee6" +"checksum slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "be4d87903baf655da2d82bc3ac3f7ef43868c58bf712b3a661fda72009304c23" "checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" "checksum smallvec 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f" "checksum snap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da73c8f77aebc0e40c300b93f0a5f1bece7a248a36eee287d4e095f35c7b7d6e" @@ -5016,6 +5165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tokio-uds 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0" "checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" "checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" +"checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" "checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" "checksum tracing 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "c2e2a2de6b0d5cbb13fc21193a2296888eaab62b6044479aafb3c54c01c29fcd" "checksum tracing-core 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "94ae75f0d28ae10786f3b1895c55fe72e79928fd5ccdebb5438c75e93fec178f" diff --git a/Cargo.toml b/Cargo.toml index 3c16059b..582a1ce9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ heck = "0.3.0" lettre = "0.9.2" lettre_email = "0.9.2" num_cpus = "1.10" +riker = "0.4" rocket = "0.4.5" rocket_contrib = { version = "0.4.5", features = ["json"] } rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" } diff --git a/DESIGN.md b/DESIGN.md new file mode 100644 index 00000000..49d3383b --- /dev/null +++ b/DESIGN.md @@ -0,0 +1,101 @@ +# Refactoring + +This is document describes the Design Goals and Refactoring Strategies of our ongoing efforts to improve the architecture, and with it the stability and performance of Plume. + +## Current Issues + +Let's look at the current architecture's problems, and the attempts we've made at resolving them. + +### PlumeRocket + +This data structure was [introduced by yours truly](https://github.com/Plume-org/Plume/pull/462) with the intent to please Clippy, reduce the number of arguments passed around (the main thing Clippy complained about). +We also tried reduce the amount of bytes being passed around, by using references. + +At the time, this seemed like a good idea. +Right now, it's the main source of our problems. + +This `struct` bundles `DbConn`, which makes it difficult migrate Plume to `async` Rocket: + +Passing around a giant `struct` as `ref` in `async` world, means that different owners are waiting for it to be `.await`ed, so they can access them. +This makes working with such a `struct` very unwieldy, if not impossible sometimes. + +### DbConn, Searcher and Post + +`DbConn` and `Searcher` are deeply bundled via `Post`. +`Searcher` is called every time a `Post` is added/updated/deleted. +It needs access to `DbConn` to fill the data that `Post` does not have. + +While removing one or the other from `PlumeRocket` is possible, complications still arise with `AsObject`: + +Posts's `AsObject` APIs are called every time a Post is added/updated/deleted. +It builds on `PlumeRocket` as main `Context`, and so we'd have to touch every API if we split either `DbConn` or `Searcher` out of `PlumeRocket` + +## Solution Attempts and their Problems + +in the past couple of weeks, we've made the following attepts to at least partially dissolve `PlumeRocket` + +- [plume-model: refactor Searcher to have its own DbPool](https://git.joinplu.me/Plume/Plume/pulls/809) +- [WIP: Experiment: extract Searcher into an Actor](https://git.joinplu.me/Plume/Plume/pulls/807) +- [extract DbConn from PlumeRocket](https://git.joinplu.me/Plume/Plume/pulls/805) + +As soon as we attempted to delete out one of the members from `PlumeRocket`, compiles would fail all over the place, meaning we'd have to touch almost every single function's *signature* that uses `PlumeRocket`. +This then means we'd have to touch every single function that in turn use those functions! +That is a lot of broken code, before we've even started refactoring. + +## Strategy + +Despite ambitions to use an [Actor System (Riker)](https://riker.rs/), it is not magnitude of the ambitions, but the size of the steps we've taken. +So, given past failures we devise a strategy. +We want to replace each of the systems in `PlumeRocket` with an `Actor`, accessed by a single reference to the `ActorSystem`. +This way we don't have to touch every single function's parameters that `PlumeRocket` flows thru, while the code is still in motion. + +### Actors + +in [#807](https://git.joinplu.me/Plume/Plume/pulls/807), we've already made our first attempt at extracting `Searcher` into a Riker `Actor`. +Here, just like in [#809](https://git.joinplu.me/Plume/Plume/pulls/809), we've already given `Searcher` its own `DbPool`. + +Why? + +### DbPool instead of DbConn + +In our previous attempts at this code, we've realized that `DbPool`, being wrapped in an `Arc` is very cheap to `.clone()`. +This means that every `Actor` that needs a `DbConn`, could get its own `DbPool`. +We also realized that `DbPool` acts like an `Actor` in its own right: + +- `DbPool` has a `.get()` message +- when that message is sent, it responds with a `DbConn` +- if the pool is exhausted, it does not! + +Thus, we conclude: +We want to `.clone()` a `DbPool` for every Actor that we extract from `PlumeRocket` that needs it. + +### Workers + +In [#799](https://git.joinplu.me/Plume/Plume/issues/799#issuecomment-4402), we've identified the following `workers`: + +> Here is the list of things the worker is used for: +> +> - sending activities to other instances (just needs the activity and a list of recipients) +> - rotating user keypair when they delete a post (after 10 minutes), which requires the DB +> - fetching posts for a blog/users, either because it is new or because it has not been updated in 24 hours (depends on the DB too, and on the searcher) + +For the first type of worker, we'll have to make *repeated* network requests. +There's a [Riker issue](https://github.com/riker-rs/riker/issues/130) asking how to best do that (with an answer.) + +The two workers that need access to the database, should get their own `DbPool`. +Being part of the same `ActorSystem`, the last type of worker will be able to access `Searcher` thru messages. + +### Request Path vs DbConn vs async + +For the Rocket Request path, we want to wrap `DbPool` in an `Actor` of its own: + +Then, in the `RequestGuard` we'd [ask](https://riker.rs/patterns/#ask) for a `DbConn`. +This brings us on-par with Rocket's current [`#[database]`](https://github.com/SergioBenitez/Rocket/pull/1375) approach, that puts database connection access into `spawn_blocking()`. +However, unlike `#[database]`, Riker's `ask()` responds with a Future, which would mean that in `async` functions we could simply `.await` it! + +## The long road + +Once we've refactored the main systems in `PlumeRocket` into their own Actors, and once we're down to only the `ActorSystem` being the main component of `PlumeRocket`, we can finally shed the husk. +That means that we *do* go and touch all the functions that take `PlumeRocket` and only give them access to the things they need. + +This is also a good chance to look at functions that are too long, or are doing to much, and mark them for refactoring. diff --git a/plume-cli/src/main.rs b/plume-cli/src/main.rs index 8dc11692..4828fbba 100644 --- a/plume-cli/src/main.rs +++ b/plume-cli/src/main.rs @@ -1,8 +1,7 @@ use dotenv; use clap::App; -use diesel::Connection; -use plume_models::{instance::Instance, Connection as Conn, CONFIG}; +use plume_models::{db_conn::init_pool, instance::Instance}; use std::io::{self, prelude::*}; mod instance; @@ -26,22 +25,27 @@ fn main() { Err(ref e) if e.not_found() => eprintln!("no .env was found"), e => e.map(|_| ()).unwrap(), } - let conn = Conn::establish(CONFIG.database_url.as_str()); - let _ = conn.as_ref().map(|conn| Instance::cache_local(conn)); + let db_pool = init_pool() + .expect("Couldn't create a database pool, please check DATABASE_URL in your .env"); + let _ = db_pool + .get() + .as_ref() + .map(|conn| Instance::cache_local(conn)); match matches.subcommand() { - ("instance", Some(args)) => { - instance::run(args, &conn.expect("Couldn't connect to the database.")) - } - ("migration", Some(args)) => { - migration::run(args, &conn.expect("Couldn't connect to the database.")) - } - ("search", Some(args)) => { - search::run(args, &conn.expect("Couldn't connect to the database.")) - } - ("users", Some(args)) => { - users::run(args, &conn.expect("Couldn't connect to the database.")) - } + ("instance", Some(args)) => instance::run( + args, + &db_pool.get().expect("Couldn't connect to the database."), + ), + ("migration", Some(args)) => migration::run( + args, + &db_pool.get().expect("Couldn't connect to the database."), + ), + ("search", Some(args)) => search::run(args, db_pool), + ("users", Some(args)) => users::run( + args, + &db_pool.get().expect("Couldn't connect to the database."), + ), _ => app.print_help().expect("Couldn't print help"), }; } diff --git a/plume-cli/src/search.rs b/plume-cli/src/search.rs index 7afa05d9..48c8a3c4 100644 --- a/plume-cli/src/search.rs +++ b/plume-cli/src/search.rs @@ -1,6 +1,6 @@ use clap::{App, Arg, ArgMatches, SubCommand}; -use plume_models::{search::Searcher, Connection, CONFIG}; +use plume_models::{db_conn::DbPool, search::Searcher, CONFIG}; use std::fs::{read_dir, remove_file}; use std::io::ErrorKind; use std::path::Path; @@ -52,7 +52,7 @@ pub fn command<'a, 'b>() -> App<'a, 'b> { ) } -pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) { +pub fn run<'a>(args: &ArgMatches<'a>, conn: DbPool) { let conn = conn; match args.subcommand() { ("init", Some(x)) => init(x, conn), @@ -63,7 +63,7 @@ pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) { } } -fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) { +fn init<'a>(args: &ArgMatches<'a>, db_pool: DbPool) { let path = args .value_of("path") .map(|p| Path::new(p).join("search_index")) @@ -82,8 +82,8 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) { } }; if can_do || force { - let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap(); - refill(args, conn, Some(searcher)); + let searcher = Searcher::create(&path, db_pool.clone(), &CONFIG.search_tokenizers).unwrap(); + refill(args, db_pool, Some(searcher)); } else { eprintln!( "Can't create new index, {} exist and is not empty", @@ -92,16 +92,16 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) { } } -fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option) { +fn refill<'a>(args: &ArgMatches<'a>, db_pool: DbPool, searcher: Option) { let path = args.value_of("path"); let path = match path { Some(path) => Path::new(path).join("search_index"), None => Path::new(&CONFIG.search_index).to_path_buf(), }; - let searcher = - searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap()); + let searcher = searcher + .unwrap_or_else(|| Searcher::open(&path, db_pool, &CONFIG.search_tokenizers).unwrap()); - searcher.fill(conn).expect("Couldn't import post"); + searcher.fill().expect("Couldn't import post"); println!("Commiting result"); searcher.commit(); } diff --git a/plume-models/Cargo.toml b/plume-models/Cargo.toml index 9c368681..7f892d06 100644 --- a/plume-models/Cargo.toml +++ b/plume-models/Cargo.toml @@ -32,6 +32,7 @@ shrinkwraprs = "0.2.1" diesel-derive-newtype = "0.1.2" glob = "0.3.0" lindera-tantivy = { version = "0.1.3", optional = true } +riker = "0.4" [dependencies.chrono] features = ["serde"] diff --git a/plume-models/plume.db-journal b/plume-models/plume.db-journal deleted file mode 100644 index 2a40935d..00000000 Binary files a/plume-models/plume.db-journal and /dev/null differ diff --git a/plume-models/plume_tests.sqlite-journal b/plume-models/plume_tests.sqlite-journal deleted file mode 100644 index 624b437f..00000000 Binary files a/plume-models/plume_tests.sqlite-journal and /dev/null differ diff --git a/plume-models/src/db_conn.rs b/plume-models/src/db_conn.rs index 5e461b18..dbff1661 100644 --- a/plume-models/src/db_conn.rs +++ b/plume-models/src/db_conn.rs @@ -1,4 +1,4 @@ -use crate::Connection; +use crate::{instance::Instance, Connection, CONFIG}; use diesel::r2d2::{ ConnectionManager, CustomizeConnection, Error as ConnError, Pool, PooledConnection, }; @@ -13,6 +13,20 @@ use std::ops::Deref; pub type DbPool = Pool>; +/// Initializes a database pool. +pub fn init_pool() -> Option { + let manager = ConnectionManager::::new(CONFIG.database_url.as_str()); + let mut builder = DbPool::builder() + .connection_customizer(Box::new(PragmaForeignKey)) + .min_idle(CONFIG.db_min_idle); + if let Some(max_size) = CONFIG.db_max_size { + builder = builder.max_size(max_size); + }; + let pool = builder.build(manager).ok()?; + Instance::cache_local(&pool.get().unwrap()); + Some(pool) +} + // From rocket documentation // Connection request guard type: a wrapper around an r2d2 pooled connection. diff --git a/plume-models/src/lib.rs b/plume-models/src/lib.rs index 67bf51ad..7245ea06 100644 --- a/plume-models/src/lib.rs +++ b/plume-models/src/lib.rs @@ -17,6 +17,8 @@ extern crate serde_json; #[macro_use] extern crate tantivy; +extern crate riker; + use plume_common::activity_pub::inbox::InboxError; #[cfg(not(any(feature = "sqlite", feature = "postgres")))] @@ -40,6 +42,7 @@ pub enum Error { Io(std::io::Error), MissingApProperty, NotFound, + DbPool, Request, SerDe, Search(search::SearcherError), @@ -303,6 +306,10 @@ mod tests { db_conn::DbConn((*DB_POOL).get().unwrap()) } + pub fn pool<'a>() -> db_conn::DbPool { + (*DB_POOL).clone() + } + lazy_static! { static ref DB_POOL: db_conn::DbPool = { let pool = db_conn::DbPool::builder() @@ -323,6 +330,9 @@ mod tests { searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)), worker: Arc::new(ScheduledThreadPool::new(2)), user: None, + actors: Arc::new( + riker::actors::ActorSystem::new().expect("Couldn't create test actor system"), + ), } } } diff --git a/plume-models/src/plume_rocket.rs b/plume-models/src/plume_rocket.rs index 0a2b3cec..a07f04db 100644 --- a/plume-models/src/plume_rocket.rs +++ b/plume-models/src/plume_rocket.rs @@ -3,6 +3,7 @@ pub use self::module::PlumeRocket; #[cfg(not(test))] mod module { use crate::{db_conn::DbConn, search, users}; + use riker::actors::ActorSystem; use rocket::{ request::{self, FlashMessage, FromRequest, Request}, Outcome, State, @@ -18,6 +19,7 @@ mod module { pub searcher: Arc, pub worker: Arc, pub flash_msg: Option<(String, String)>, + pub actors: Arc, } impl<'a, 'r> FromRequest<'a, 'r> for PlumeRocket { @@ -30,6 +32,7 @@ mod module { let worker = request.guard::<'_, State<'_, Arc>>()?; let searcher = request.guard::<'_, State<'_, Arc>>()?; let flash_msg = request.guard::>().succeeded(); + let actors = request.guard::<'_, State<'_, Arc>>()?; Outcome::Success(PlumeRocket { conn, intl, @@ -37,6 +40,7 @@ mod module { flash_msg: flash_msg.map(|f| (f.name().into(), f.msg().into())), worker: worker.clone(), searcher: searcher.clone(), + actors: actors.clone(), }) } } @@ -45,6 +49,7 @@ mod module { #[cfg(test)] mod module { use crate::{db_conn::DbConn, search, users}; + use riker::actors::ActorSystem; use rocket::{ request::{self, FromRequest, Request}, Outcome, State, @@ -58,6 +63,7 @@ mod module { pub user: Option, pub searcher: Arc, pub worker: Arc, + pub actors: Arc, } impl<'a, 'r> FromRequest<'a, 'r> for PlumeRocket { @@ -68,11 +74,13 @@ mod module { let user = request.guard::().succeeded(); let worker = request.guard::<'_, State<'_, Arc>>()?; let searcher = request.guard::<'_, State<'_, Arc>>()?; + let actors = request.guard::<'_, State<'_, Arc>>()?; Outcome::Success(PlumeRocket { conn, user, worker: worker.clone(), searcher: searcher.clone(), + actors: actors.clone(), }) } } diff --git a/plume-models/src/posts.rs b/plume-models/src/posts.rs index 73c0c915..49cb2c41 100644 --- a/plume-models/src/posts.rs +++ b/plume-models/src/posts.rs @@ -1,7 +1,7 @@ use crate::{ ap_url, blogs::Blog, instance::Instance, medias::Media, mentions::Mention, post_authors::*, - safe_string::SafeString, schema::posts, search::Searcher, tags::*, timeline::*, users::User, - Connection, Error, PlumeRocket, Result, CONFIG, + safe_string::SafeString, schema::posts, search::Searcher, search::UpdateDocument, tags::*, + timeline::*, users::User, Connection, Error, PlumeRocket, Result, CONFIG, }; use activitypub::{ activity::{Create, Delete, Update}, @@ -19,12 +19,14 @@ use plume_common::{ }, utils::md_to_html, }; + +use riker::actors::*; use serde_json; use std::collections::HashSet; pub type LicensedArticle = CustomObject; -#[derive(Queryable, Identifiable, Clone, AsChangeset)] +#[derive(Debug, Queryable, Identifiable, Clone, AsChangeset)] #[changeset_options(treat_none_as_null = "true")] pub struct Post { pub id: i32, @@ -78,14 +80,13 @@ impl Post { let _: Post = post.save_changes(conn)?; } - searcher.add_document(conn, &post)?; + searcher.add_document(&post)?; Ok(post) } - pub fn update(&self, conn: &Connection, searcher: &Searcher) -> Result { + pub fn update(&self, conn: &Connection) -> Result { diesel::update(self).set(self).execute(conn)?; let post = Self::get(conn, self.id)?; - searcher.update_document(conn, &post)?; Ok(post) } @@ -728,7 +729,7 @@ impl AsObject for PostUpdate { fn activity(self, c: &PlumeRocket, actor: User, _id: &str) -> Result<()> { let conn = &*c.conn; - let searcher = &c.searcher; + let searcher_actor = &c.actors.select("searcher-actor").unwrap(); let mut post = Post::from_id(c, &self.ap_url, None).map_err(|(_, e)| e)?; if !post.is_author(conn, actor.id)? { @@ -791,7 +792,8 @@ impl AsObject for PostUpdate { post.update_hashtags(conn, hashtags)?; } - post.update(conn, searcher)?; + post.update(conn)?; + searcher_actor.try_tell(UpdateDocument(post.clone()), None); Ok(()) } } diff --git a/plume-models/src/search/mod.rs b/plume-models/src/search/mod.rs index 83b9bf62..ef0b9992 100644 --- a/plume-models/src/search/mod.rs +++ b/plume-models/src/search/mod.rs @@ -7,7 +7,7 @@ pub use self::tokenizer::TokenizerKind; #[cfg(test)] pub(crate) mod tests { - use super::{Query, Searcher, TokenizerKind}; + use super::{Query, Searcher}; use diesel::Connection; use plume_common::utils::random_hex; use std::env::temp_dir; @@ -20,15 +20,16 @@ pub(crate) mod tests { posts::{NewPost, Post}, safe_string::SafeString, tests::db, + tests::pool, CONFIG, }; pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher { let dir = temp_dir().join(&format!("plume-test-{}", random_hex())); if dir.exists() { - Searcher::open(&dir, tokenizers) + Searcher::open(&dir, pool(), tokenizers) } else { - Searcher::create(&dir, tokenizers) + Searcher::create(&dir, pool(), tokenizers) } .unwrap() } @@ -103,20 +104,20 @@ pub(crate) mod tests { fn open() { let dir = temp_dir().join(format!("plume-test-{}", random_hex())); { - Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap(); + Searcher::create(&dir, pool(), &CONFIG.search_tokenizers).unwrap(); } - Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); + Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).unwrap(); } #[test] fn create() { let dir = temp_dir().join(format!("plume-test-{}", random_hex())); - assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err()); + assert!(Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).is_err()); { - Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap(); + Searcher::create(&dir, pool(), &CONFIG.search_tokenizers).unwrap(); } - Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created + Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).unwrap(); //verify it's well created } #[test] @@ -158,26 +159,26 @@ pub(crate) mod tests { searcher.commit(); assert_eq!( - searcher.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))[0].id, + searcher.search_document(Query::from_str(&title).unwrap(), (0, 1))[0].id, post.id ); let newtitle = random_hex()[..8].to_owned(); post.title = newtitle.clone(); - post.update(conn, &searcher).unwrap(); + post.update(conn).unwrap(); searcher.commit(); assert_eq!( - searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))[0].id, + searcher.search_document(Query::from_str(&newtitle).unwrap(), (0, 1))[0].id, post.id ); assert!(searcher - .search_document(conn, Query::from_str(&title).unwrap(), (0, 1)) + .search_document(Query::from_str(&title).unwrap(), (0, 1)) .is_empty()); post.delete(conn, &searcher).unwrap(); searcher.commit(); assert!(searcher - .search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1)) + .search_document(Query::from_str(&newtitle).unwrap(), (0, 1)) .is_empty()); Ok(()) }); diff --git a/plume-models/src/search/searcher.rs b/plume-models/src/search/searcher.rs index 34a45b0b..d0d367be 100644 --- a/plume-models/src/search/searcher.rs +++ b/plume-models/src/search/searcher.rs @@ -1,11 +1,12 @@ use crate::{ - config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts, - search::query::PlumeQuery, tags::Tag, Connection, Result, + config::SearchTokenizerConfig, db_conn::DbPool, instance::Instance, posts::Post, schema::posts, + search::query::PlumeQuery, tags::Tag, Error, Result, CONFIG, }; -use chrono::Datelike; +use chrono::{Datelike, Utc}; use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl}; use itertools::Itertools; -use std::{cmp, fs::create_dir_all, io, path::Path, sync::Mutex}; +use riker::actors::*; +use std::{cmp, fs::create_dir_all, io, path::Path, sync::Arc, sync::Mutex}; use tantivy::{ collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter, ReloadPolicy, TantivyError, Term, @@ -25,9 +26,153 @@ pub struct Searcher { index: Index, reader: IndexReader, writer: Mutex>, + dbpool: DbPool, +} + +#[derive(Clone, Debug)] +pub struct AddDocument(Post); + +#[derive(Clone, Debug)] +pub struct UpdateDocument(pub Post); + +#[derive(Clone, Debug)] +pub struct DeleteDocument(Post); + +#[actor(AddDocument, UpdateDocument)] +pub struct SearcherActor(Searcher); + +impl Actor for SearcherActor { + type Msg = SearcherActorMsg; + + // forwards Msg to the respective Receive implementation + fn recv(&mut self, ctx: &Context, msg: Self::Msg, sender: Sender) { + self.receive(ctx, msg, sender); + } +} + +impl Receive for SearcherActor { + type Msg = SearcherActorMsg; + + fn receive(&mut self, _ctx: &Context, msg: AddDocument, _sender: Sender) { + let _ = self.0.add_document(&msg.0); + } +} + +impl Receive for SearcherActor { + type Msg = SearcherActorMsg; + + fn receive(&mut self, _ctx: &Context, msg: UpdateDocument, _sender: Sender) { + let _ = self.0.update_document(&msg.0); + } +} + +impl ActorFactoryArgs> for SearcherActor { + fn create_args(searcher: Arc) -> Self { + SearcherActor(Arc::try_unwrap(searcher).ok().unwrap()) + } } impl Searcher { + /// Initializes a new `Searcher`, ready to be used by + /// Plume. + /// + /// The main task of this function is to try everything + /// to get a valid `Searcher`: + /// + /// - first it tries to open the search index normally (using the options from `CONFIG`) + /// - if it fails, it makes a back-up of the index files, deletes the original ones, + /// and recreate the whole index. It removes the backup only if the re-creation + /// succeeds. + /// + /// # Panics + /// + /// This function panics if it needs to create a backup and it can't, or if it fails + /// to recreate the search index. + /// + /// After that, it can also panic if there are still errors remaining. + /// + /// The panic messages are normally explicit enough for a human to + /// understand how to fix the issue when they see it. + pub fn new(db_pool: DbPool) -> Self { + // We try to open the index a first time + let searcher = match Self::open( + &CONFIG.search_index, + db_pool.clone(), + &CONFIG.search_tokenizers, + ) { + // The index may be corrupted, inexistent or use an older format. + // In this case, we can easily recover by deleting and re-creating it. + Err(Error::Search(SearcherError::InvalidIndexDataError)) => { + if Self::create( + &CONFIG.search_index, + db_pool.clone(), + &CONFIG.search_tokenizers, + ) + .is_err() + { + let current_path = Path::new(&CONFIG.search_index); + let backup_path = + format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); + let backup_path = Path::new(&backup_path); + std::fs::rename(current_path, backup_path) + .expect("Error while backing up search index directory for re-creation"); + if Self::create( + &CONFIG.search_index, + db_pool.clone(), + &CONFIG.search_tokenizers, + ) + .is_ok() + { + if std::fs::remove_dir_all(backup_path).is_err() { + eprintln!( + "error on removing backup directory: {}. it remains", + backup_path.display() + ); + } + } else { + panic!("Error while re-creating search index in new index format. Remove search index and run `plm search init` manually."); + } + } + Self::open(&CONFIG.search_index, db_pool, &CONFIG.search_tokenizers) + } + // If it opened successfully or if it was another kind of + // error (that we don't know how to handle), don't do anything more + other => other, + }; + + // At this point, if there are still errors, we just panic + #[allow(clippy::match_wild_err_arm)] + match searcher { + Err(Error::Search(e)) => match e { + SearcherError::WriteLockAcquisitionError => panic!( + r#" +Your search index is locked. Plume can't start. To fix this issue +make sure no other Plume instance is started, and run: + + plm search unlock + +Then try to restart Plume. +"# + ), + SearcherError::IndexOpeningError => panic!( + r#" +Plume was unable to open the search index. If you created the index +before, make sure to run Plume in the same directory it was created in, or +to set SEARCH_INDEX accordingly. If you did not yet create the search +index, run this command: + + plm search init + +Then try to restart Plume +"# + ), + e => Err(e).unwrap(), + }, + Err(_) => panic!("Unexpected error while opening search index"), + Ok(s) => s, + } + } + pub fn schema() -> Schema { let tag_indexing = TextOptions::default().set_indexing_options( TextFieldIndexing::default() @@ -67,7 +212,11 @@ impl Searcher { schema_builder.build() } - pub fn create(path: &dyn AsRef, tokenizers: &SearchTokenizerConfig) -> Result { + pub fn create( + path: &dyn AsRef, + dbpool: DbPool, + tokenizers: &SearchTokenizerConfig, + ) -> Result { let schema = Self::schema(); create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?; @@ -95,10 +244,15 @@ impl Searcher { .try_into() .map_err(|_| SearcherError::IndexCreationError)?, index, + dbpool, }) } - pub fn open(path: &dyn AsRef, tokenizers: &SearchTokenizerConfig) -> Result { + pub fn open( + path: &dyn AsRef, + dbpool: DbPool, + tokenizers: &SearchTokenizerConfig, + ) -> Result { let mut index = Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?) .map_err(|_| SearcherError::IndexOpeningError)?; @@ -150,10 +304,11 @@ impl Searcher { } })?, index, + dbpool, }) } - pub fn add_document(&self, conn: &Connection, post: &Post) -> Result<()> { + pub fn add_document(&self, post: &Post) -> Result<()> { if !post.published { return Ok(()); } @@ -175,15 +330,19 @@ impl Searcher { let lang = schema.get_field("lang").unwrap(); let license = schema.get_field("license").unwrap(); + let conn = match self.dbpool.get() { + Ok(c) => c, + Err(_) => return Err(Error::DbPool), + }; let mut writer = self.writer.lock().unwrap(); let writer = writer.as_mut().unwrap(); writer.add_document(doc!( post_id => i64::from(post.id), - author => post.get_authors(conn)?.into_iter().map(|u| u.fqn).join(" "), + author => post.get_authors(&conn)?.into_iter().map(|u| u.fqn).join(" "), creation_date => i64::from(post.creation_date.num_days_from_ce()), - instance => Instance::get(conn, post.get_blog(conn)?.instance_id)?.public_domain, - tag => Tag::for_post(conn, post.id)?.into_iter().map(|t| t.tag).join(" "), - blog_name => post.get_blog(conn)?.title, + instance => Instance::get(&conn, post.get_blog(&conn)?.instance_id)?.public_domain, + tag => Tag::for_post(&conn, post.id)?.into_iter().map(|t| t.tag).join(" "), + blog_name => post.get_blog(&conn)?.title, content => post.content.get().clone(), subtitle => post.subtitle.clone(), title => post.title.clone(), @@ -203,17 +362,12 @@ impl Searcher { writer.delete_term(doc_id); } - pub fn update_document(&self, conn: &Connection, post: &Post) -> Result<()> { + pub fn update_document(&self, post: &Post) -> Result<()> { self.delete_document(post); - self.add_document(conn, post) + self.add_document(post) } - pub fn search_document( - &self, - conn: &Connection, - query: PlumeQuery, - (min, max): (i32, i32), - ) -> Vec { + pub fn search_document(&self, query: PlumeQuery, (min, max): (i32, i32)) -> Vec { let schema = self.index.schema(); let post_id = schema.get_field("post_id").unwrap(); @@ -222,24 +376,33 @@ impl Searcher { let searcher = self.reader.searcher(); let res = searcher.search(&query.into_query(), &collector).unwrap(); + let conn = match self.dbpool.get() { + Ok(c) => c, + Err(_) => return Vec::new(), + }; + res.get(min as usize..) .unwrap_or(&[]) .iter() .filter_map(|(_, doc_add)| { let doc = searcher.doc(*doc_add).ok()?; let id = doc.get_first(post_id)?; - Post::get(conn, id.i64_value() as i32).ok() + Post::get(&conn, id.i64_value() as i32).ok() //borrow checker don't want me to use filter_map or and_then here }) .collect() } - pub fn fill(&self, conn: &Connection) -> Result<()> { + pub fn fill(&self) -> Result<()> { + let conn = match self.dbpool.get() { + Ok(c) => c, + Err(_) => return Err(Error::DbPool), + }; for post in posts::table .filter(posts::published.eq(true)) - .load::(conn)? + .load::(&conn)? { - self.update_document(conn, &post)? + self.update_document(&post)? } Ok(()) } diff --git a/src/main.rs b/src/main.rs index 4ae2f88f..9010f16b 100755 --- a/src/main.rs +++ b/src/main.rs @@ -10,20 +10,16 @@ extern crate serde_json; #[macro_use] extern crate validator_derive; -use chrono::Utc; +extern crate riker; + use clap::App; -use diesel::r2d2::ConnectionManager; use plume_models::{ - db_conn::{DbPool, PragmaForeignKey}, - instance::Instance, - migrations::IMPORTED_MIGRATIONS, - search::{Searcher as UnmanagedSearcher, SearcherError}, - Connection, Error, CONFIG, + db_conn::init_pool, migrations::IMPORTED_MIGRATIONS, search::Searcher, search::SearcherActor, + CONFIG, }; +use riker::actors::*; use rocket_csrf::CsrfFairingBuilder; use scheduled_thread_pool::ScheduledThreadPool; -use std::fs; -use std::path::Path; use std::process::exit; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -48,26 +44,6 @@ include!(concat!(env!("OUT_DIR"), "/templates.rs")); compile_i18n!(); -/// Initializes a database pool. -fn init_pool() -> Option { - match dotenv::dotenv() { - Ok(path) => println!("Configuration read from {}", path.display()), - Err(ref e) if e.not_found() => eprintln!("no .env was found"), - e => e.map(|_| ()).unwrap(), - } - - let manager = ConnectionManager::::new(CONFIG.database_url.as_str()); - let mut builder = DbPool::builder() - .connection_customizer(Box::new(PragmaForeignKey)) - .min_idle(CONFIG.db_min_idle); - if let Some(max_size) = CONFIG.db_max_size { - builder = builder.max_size(max_size); - }; - let pool = builder.build(manager).ok()?; - Instance::cache_local(&pool.get().unwrap()); - Some(pool) -} - fn main() { App::new("Plume") .bin_name("plume") @@ -82,6 +58,13 @@ and https://docs.joinplu.me/installation/init for more info. "#, ) .get_matches(); + + match dotenv::dotenv() { + Ok(path) => println!("Configuration read from {}", path.display()), + Err(ref e) if e.not_found() => eprintln!("no .env was found"), + e => e.map(|_| ()).unwrap(), + } + let dbpool = init_pool().expect("main: database pool initialization error"); if IMPORTED_MIGRATIONS .is_pending(&dbpool.get().unwrap()) @@ -100,59 +83,12 @@ Then try to restart Plume. ) } let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get()); - // we want a fast exit here, so - let mut open_searcher = - UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); - if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher { - if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() { - let current_path = Path::new(&CONFIG.search_index); - let backup_path = format!("{}.{}", ¤t_path.display(), Utc::now().timestamp()); - let backup_path = Path::new(&backup_path); - fs::rename(current_path, backup_path) - .expect("main: error on backing up search index directory for recreating"); - if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() { - if fs::remove_dir_all(backup_path).is_err() { - eprintln!( - "error on removing backup directory: {}. it remains", - backup_path.display() - ); - } - } else { - panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually"); - } - } - open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers); - } - #[allow(clippy::match_wild_err_arm)] - let searcher = match open_searcher { - Err(Error::Search(e)) => match e { - SearcherError::WriteLockAcquisitionError => panic!( - r#" -Your search index is locked. Plume can't start. To fix this issue -make sure no other Plume instance is started, and run: - - plm search unlock + let searcher = Arc::new(Searcher::new(dbpool.clone())); -Then try to restart Plume. -"# - ), - SearcherError::IndexOpeningError => panic!( - r#" -Plume was unable to open the search index. If you created the index -before, make sure to run Plume in the same directory it was created in, or -to set SEARCH_INDEX accordingly. If you did not yet create the search -index, run this command: - - plm search init - -Then try to restart Plume -"# - ), - e => Err(e).unwrap(), - }, - Err(_) => panic!("Unexpected error while opening search index"), - Ok(s) => Arc::new(s), - }; + let sys = SystemBuilder::new().name("plume").create().unwrap(); + let _ = sys + .actor_of_args::("searcher-actor", searcher.clone()) + .unwrap(); let commiter = searcher.clone(); workpool.execute_with_fixed_delay( @@ -299,6 +235,7 @@ Then try to restart Plume .manage(dbpool) .manage(Arc::new(workpool)) .manage(searcher) + .manage(Arc::new(sys)) .manage(include_i18n!()) .attach( CsrfFairingBuilder::new() diff --git a/src/routes/posts.rs b/src/routes/posts.rs index e6b10be8..fa63fcc8 100644 --- a/src/routes/posts.rs +++ b/src/routes/posts.rs @@ -1,5 +1,6 @@ use chrono::Utc; use heck::{CamelCase, KebabCase}; +use riker::actors::*; use rocket::request::LenientForm; use rocket::response::{Flash, Redirect}; use rocket_i18n::I18n; @@ -26,6 +27,7 @@ use plume_models::{ post_authors::*, posts::*, safe_string::SafeString, + search::UpdateDocument, tags::*, timeline::*, users::User, @@ -297,8 +299,6 @@ pub fn update( post.source = form.content.clone(); post.license = form.license.clone(); post.cover_id = form.cover; - post.update(&*conn, &rockets.searcher) - .expect("post::update: update error"); if post.published { post.update_mentions( @@ -351,6 +351,9 @@ pub fn update( } } + let searcher_actor = rockets.actors.select("searcher-actor").unwrap(); + searcher_actor.try_tell(UpdateDocument(post.clone()), None); + Flash::success( Redirect::to(uri!(details: blog = blog, slug = new_slug, responding_to = _)), i18n!(intl, "Your article has been updated."), diff --git a/src/routes/search.rs b/src/routes/search.rs index 0341bf8f..5def8263 100644 --- a/src/routes/search.rs +++ b/src/routes/search.rs @@ -51,7 +51,6 @@ macro_rules! param_to_query { #[get("/search?")] pub fn search(query: Option>, rockets: PlumeRocket) -> Ructe { - let conn = &*rockets.conn; let query = query.map(Form::into_inner).unwrap_or_default(); let page = query.page.unwrap_or_default(); let mut parsed_query = @@ -72,7 +71,7 @@ pub fn search(query: Option>, rockets: PlumeRocket) -> Ructe { } else { let res = rockets .searcher - .search_document(&conn, parsed_query, page.limits()); + .search_document(parsed_query, page.limits()); let next_page = if res.is_empty() { 0 } else { page.0 + 1 }; render!(search::result( &rockets.to_context(),