#813 WIP: Extract DbConn, Searcher & Worker into Riker Actors

Ouvert
igalic veut fusionner 18 révision(s) depuis igalic/Plume:refactor/extract-actors vers main
  1. +150
    -0
      Cargo.lock
  2. +1
    -0
      Cargo.toml
  3. +101
    -0
      DESIGN.md
  4. +20
    -16
      plume-cli/src/main.rs
  5. +9
    -9
      plume-cli/src/search.rs
  6. +1
    -0
      plume-models/Cargo.toml
  7. BIN
      plume-models/plume.db-journal
  8. BIN
      plume-models/plume_tests.sqlite-journal
  9. +15
    -1
      plume-models/src/db_conn.rs
  10. +10
    -0
      plume-models/src/lib.rs
  11. +8
    -0
      plume-models/src/plume_rocket.rs
  12. +10
    -8
      plume-models/src/posts.rs
  13. +14
    -13
      plume-models/src/search/mod.rs
  14. +186
    -23
      plume-models/src/search/searcher.rs
  15. +18
    -81
      src/main.rs
  16. +5
    -2
      src/routes/posts.rs
  17. +1
    -2
      src/routes/search.rs

+ 150
- 0
Cargo.lock Voir le fichier

@@ -544,6 +544,21 @@ dependencies = [
"memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "config"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"nom 5.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
"serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
"yaml-rust 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "conv"
version = "0.3.3"
@@ -709,6 +724,14 @@ name = "custom_derive"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "dashmap"
version = "4.0.0-rc6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "data-encoding"
version = "2.1.2"
@@ -1890,6 +1913,15 @@ dependencies = [
"safemem 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "linked-hash-map"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "linked-hash-map"
version = "0.5.3"
@@ -2261,6 +2293,14 @@ dependencies = [
"num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "num-traits"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "num-traits"
version = "0.2.12"
@@ -2549,6 +2589,7 @@ dependencies = [
"plume-api 0.4.0",
"plume-common 0.4.0",
"plume-models 0.4.0",
"riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_contrib 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_csrf 0.1.0 (git+https://github.com/fdb-hiroshima/rocket_csrf?rev=29910f2829e7e590a540da3804336577b48c7b31)",
@@ -2660,6 +2701,7 @@ dependencies = [
"plume-common 0.4.0",
"plume-macro 0.4.0",
"reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)",
"riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
"rocket_i18n 0.4.0 (git+https://github.com/Plume-org/rocket_i18n?rev=e922afa7c366038b3433278c03b1456b346074f2)",
"scheduled-thread-pool 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -3127,6 +3169,36 @@ dependencies = [
"winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "riker"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"chrono 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)",
"config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
"dashmap 4.0.0-rc6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-utils 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"riker-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"uuid 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "riker-macros"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "ring"
version = "0.13.5"
@@ -3254,6 +3326,11 @@ dependencies = [
"nom 5.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "rust-ini"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "rust-stemmers"
version = "1.2.0"
@@ -3355,6 +3432,11 @@ name = "semver-parser"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "serde"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "serde"
version = "1.0.114"
@@ -3363,6 +3445,18 @@ dependencies = [
"serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "serde-hjson"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)",
"regex 1.3.9 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "serde_derive"
version = "1.0.114"
@@ -3394,6 +3488,14 @@ dependencies = [
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "serde_test"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "serde_urlencoded"
version = "0.5.5"
@@ -3468,6 +3570,32 @@ name = "slab"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "slog"
version = "2.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

[[package]]
name = "slog-scope"
version = "4.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "slog-stdlog"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"crossbeam 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)",
"slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "smallvec"
version = "0.6.13"
@@ -4128,6 +4256,14 @@ dependencies = [
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "toml"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)",
]

[[package]]
name = "tower-service"
version = "0.3.0"
@@ -4656,6 +4792,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum cloudabi 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467"
"checksum colored 1.9.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ffc801dacf156c5854b9df4f425a626539c3a6ef7893cc0c5084a23f0b6c59"
"checksum combine 4.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b8e5ef862b2df927249f4e2bdc29c1bd13a33105f900884b0c32acdf32aff584"
"checksum config 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "19b076e143e1d9538dde65da30f8481c2a6c44040edb8e02b9bf1351edb92ce3"
"checksum conv 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "78ff10625fd0ac447827aa30ea8b861fead473bb60aeb73af6c1c58caf0d1299"
"checksum cookie 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5795cda0897252e34380a27baf884c53aa7ad9990329cdad96d4c5d027015d44"
"checksum cookie 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
@@ -4673,6 +4810,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5"
"checksum ctrlc 3.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "54dedab740bc412d514cfbc4a1d9d5d16fed02c4b14a7be129003c07fdc33b9b"
"checksum custom_derive 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ef8ae57c4978a2acd8b869ce6b9ca1dfe817bff704c220209fdef2c0b75a01b9"
"checksum dashmap 4.0.0-rc6 (registry+https://github.com/rust-lang/crates.io-index)" = "308a6703be2d759cb5fb7b80a23547fe73a8d5ebf70d3a4ca7f0ef4c0bfc2265"
"checksum data-encoding 2.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f4f47ca1860a761136924ddd2422ba77b2ea54fe8cc75b9040804a0d9d32ad97"
"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850"
"checksum debug-builders 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0f5d8e3d14cabcb2a8a59d7147289173c6ada77a0bc526f6b85078f941c0cf12"
@@ -4799,6 +4937,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum lindera-ipadic-builder 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a617422d3b9fa3bcaef470423f6b686aa8954e7081bd148ced8d8cf3c08925b6"
"checksum lindera-tantivy 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b1bd76e0573b2f28cfeb16f2ee50f8cc6ea14a1e307f328be1d9744053a97633"
"checksum line-wrap 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
"checksum linked-hash-map 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6d262045c5b87c0861b3f004610afd0e2c851e2908d08b6c870cbb9d5f494ecd"
"checksum linked-hash-map 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
"checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75"
"checksum lock_api 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c"
@@ -4839,6 +4978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum notify 4.0.15 (registry+https://github.com/rust-lang/crates.io-index)" = "80ae4a7688d1fab81c5bf19c64fc8db920be8d519ce6336ed4e7efe024724dbd"
"checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
"checksum num-rational 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
"checksum num-traits 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "92e5113e9fd4cc14ded8e499429f396a20f98c772a47cc8622a736e1ec843c31"
"checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
"checksum object 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5"
@@ -4918,6 +5058,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum remove_dir_all 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
"checksum reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680"
"checksum reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "f88643aea3c1343c804950d7bf983bd2067f5ab59db6d613a08e05572f2714ab"
"checksum riker 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b07140216b4c980a887315331822cc63f6d22f824b23dbff1751aee655a365"
"checksum riker-macros 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d2a8e8f71c9e7980a596c39c7e3537ea8563054526e15712a610ac97a02dba15"
"checksum ring 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)" = "2c4db68a2e35f3497146b7e4563df7d4773a2433230c5e4b448328e31740458a"
"checksum rocket 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6130967b369cfb8411b0b73e96fcba1229c32a9cc6f295d144f879bfced13c6e"
"checksum rocket_codegen 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "cb852e6da168fb948a8f2b798ba2e2f0e4fc860eae0efa9cf2bf0f5466bb0425"
@@ -4928,6 +5070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum rpassword 4.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "99371657d3c8e4d816fb6221db98fa408242b0b53bac08f8676a41f8554fe99f"
"checksum rsass 0.9.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7f4534cc03040beacd2668621815f26fe57e5b7cfe085790f98e5e87c1612316"
"checksum ructe 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c85620b8046f88a870d93d90fa56904dec76cc79139bfcc22e71e87f0cd2169f"
"checksum rust-ini 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e52c148ef37f8c375d49d5a73aa70713125b7f19095948a923f80afdeb22ec2"
"checksum rust-stemmers 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
"checksum rustc-demangle 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
"checksum rustc-hash 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
@@ -4942,10 +5085,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum security-framework-sys 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "17bf11d99252f512695eb468de5516e5cf75455521e69dfe343f3b74e4748405"
"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
"checksum serde 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "9dad3f759919b92c3068c696c15c3d17238234498bbdcc80f2c469606f948ac8"
"checksum serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
"checksum serde-hjson 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6a3a4e0ea8a88553209f6cc6cfe8724ecad22e1acf372793c27d995290fe74f8"
"checksum serde_derive 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)" = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
"checksum serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)" = "3433e879a558dde8b5e8feb2a04899cf34fdde1fafb894687e52105fc1162ac3"
"checksum serde_qs 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d43eef44996bbe16e99ac720e1577eefa16f7b76b5172165c98ced20ae9903e1"
"checksum serde_test 0.8.23 (registry+https://github.com/rust-lang/crates.io-index)" = "110b3dbdf8607ec493c22d5d947753282f3bae73c0f56d322af1e8c78e4c23d5"
"checksum serde_urlencoded 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "642dd69105886af2efd227f75a520ec9b44a820d65bc133a9131f7d229fd165a"
"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
"checksum sha1 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"
@@ -4955,6 +5101,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41"
"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
"checksum slog 2.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cc9c640a4adbfbcc11ffb95efe5aa7af7309e002adab54b185507dbf2377b99"
"checksum slog-scope 4.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7c44c89dd8b0ae4537d1ae318353eaf7840b4869c536e31c41e963d1ea523ee6"
"checksum slog-stdlog 4.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "be4d87903baf655da2d82bc3ac3f7ef43868c58bf712b3a661fda72009304c23"
"checksum smallvec 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)" = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6"
"checksum smallvec 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3757cb9d89161a2f24e1cf78efa0c1fcff485d18e3f55e0aa3480824ddaa0f3f"
"checksum snap 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "da73c8f77aebc0e40c300b93f0a5f1bece7a248a36eee287d4e095f35c7b7d6e"
@@ -5016,6 +5165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum tokio-uds 0.2.7 (registry+https://github.com/rust-lang/crates.io-index)" = "ab57a4ac4111c8c9dbcf70779f6fc8bc35ae4b2454809febac840ad19bd7e4e0"
"checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f"
"checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a"
"checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
"checksum tracing 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "c2e2a2de6b0d5cbb13fc21193a2296888eaab62b6044479aafb3c54c01c29fcd"
"checksum tracing-core 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "94ae75f0d28ae10786f3b1895c55fe72e79928fd5ccdebb5438c75e93fec178f"


+ 1
- 0
Cargo.toml Voir le fichier

@@ -20,6 +20,7 @@ heck = "0.3.0"
lettre = "0.9.2"
lettre_email = "0.9.2"
num_cpus = "1.10"
riker = "0.4"
rocket = "0.4.5"
rocket_contrib = { version = "0.4.5", features = ["json"] }
rocket_i18n = { git = "https://github.com/Plume-org/rocket_i18n", rev = "e922afa7c366038b3433278c03b1456b346074f2" }


+ 101
- 0
DESIGN.md Voir le fichier

@@ -0,0 +1,101 @@
# Refactoring

This is document describes the Design Goals and Refactoring Strategies of our ongoing efforts to improve the architecture, and with it the stability and performance of Plume.

## Current Issues

Let's look at the current architecture's problems, and the attempts we've made at resolving them.

### PlumeRocket

This data structure was [introduced by yours truly](https://github.com/Plume-org/Plume/pull/462) with the intent to please Clippy, reduce the number of arguments passed around (the main thing Clippy complained about).
We also tried reduce the amount of bytes being passed around, by using references.

At the time, this seemed like a good idea.
Right now, it's the main source of our problems.

This `struct` bundles `DbConn`, which makes it difficult migrate Plume to `async` Rocket:

Passing around a giant `struct` as `ref` in `async` world, means that different owners are waiting for it to be `.await`ed, so they can access them.
This makes working with such a `struct` very unwieldy, if not impossible sometimes.

### DbConn, Searcher and Post

`DbConn` and `Searcher` are deeply bundled via `Post`.
`Searcher` is called every time a `Post` is added/updated/deleted.
It needs access to `DbConn` to fill the data that `Post` does not have.

While removing one or the other from `PlumeRocket` is possible, complications still arise with `AsObject`:

Posts's `AsObject` APIs are called every time a Post is added/updated/deleted.
It builds on `PlumeRocket` as main `Context`, and so we'd have to touch every API if we split either `DbConn` or `Searcher` out of `PlumeRocket`

## Solution Attempts and their Problems

in the past couple of weeks, we've made the following attepts to at least partially dissolve `PlumeRocket`

- [plume-model: refactor Searcher to have its own DbPool](https://git.joinplu.me/Plume/Plume/pulls/809)
- [WIP: Experiment: extract Searcher into an Actor](https://git.joinplu.me/Plume/Plume/pulls/807)
- [extract DbConn from PlumeRocket](https://git.joinplu.me/Plume/Plume/pulls/805)

As soon as we attempted to delete out one of the members from `PlumeRocket`, compiles would fail all over the place, meaning we'd have to touch almost every single function's *signature* that uses `PlumeRocket`.
This then means we'd have to touch every single function that in turn use those functions!
That is a lot of broken code, before we've even started refactoring.

## Strategy

Despite ambitions to use an [Actor System (Riker)](https://riker.rs/), it is not magnitude of the ambitions, but the size of the steps we've taken.
So, given past failures we devise a strategy.
We want to replace each of the systems in `PlumeRocket` with an `Actor`, accessed by a single reference to the `ActorSystem`.
This way we don't have to touch every single function's parameters that `PlumeRocket` flows thru, while the code is still in motion.

### Actors

in [#807](https://git.joinplu.me/Plume/Plume/pulls/807), we've already made our first attempt at extracting `Searcher` into a Riker `Actor`.
Here, just like in [#809](https://git.joinplu.me/Plume/Plume/pulls/809), we've already given `Searcher` its own `DbPool`.

Why?

### DbPool instead of DbConn

In our previous attempts at this code, we've realized that `DbPool`, being wrapped in an `Arc` is very cheap to `.clone()`.
This means that every `Actor` that needs a `DbConn`, could get its own `DbPool`.
We also realized that `DbPool` acts like an `Actor` in its own right:

- `DbPool` has a `.get()` message
- when that message is sent, it responds with a `DbConn`
- if the pool is exhausted, it does not!

Thus, we conclude:
We want to `.clone()` a `DbPool` for every Actor that we extract from `PlumeRocket` that needs it.

### Workers

In [#799](https://git.joinplu.me/Plume/Plume/issues/799#issuecomment-4402), we've identified the following `workers`:

> Here is the list of things the worker is used for:
>
> - sending activities to other instances (just needs the activity and a list of recipients)
> - rotating user keypair when they delete a post (after 10 minutes), which requires the DB
> - fetching posts for a blog/users, either because it is new or because it has not been updated in 24 hours (depends on the DB too, and on the searcher)

For the first type of worker, we'll have to make *repeated* network requests.
There's a [Riker issue](https://github.com/riker-rs/riker/issues/130) asking how to best do that (with an answer.)

The two workers that need access to the database, should get their own `DbPool`.
Being part of the same `ActorSystem`, the last type of worker will be able to access `Searcher` thru messages.

### Request Path vs DbConn vs async

For the Rocket Request path, we want to wrap `DbPool` in an `Actor` of its own:

Then, in the `RequestGuard` we'd [ask](https://riker.rs/patterns/#ask) for a `DbConn`.
This brings us on-par with Rocket's current [`#[database]`](https://github.com/SergioBenitez/Rocket/pull/1375) approach, that puts database connection access into `spawn_blocking()`.
However, unlike `#[database]`, Riker's `ask()` responds with a Future, which would mean that in `async` functions we could simply `.await` it!

## The long road

Once we've refactored the main systems in `PlumeRocket` into their own Actors, and once we're down to only the `ActorSystem` being the main component of `PlumeRocket`, we can finally shed the husk.
That means that we *do* go and touch all the functions that take `PlumeRocket` and only give them access to the things they need.

This is also a good chance to look at functions that are too long, or are doing to much, and mark them for refactoring.

+ 20
- 16
plume-cli/src/main.rs Voir le fichier

@@ -1,8 +1,7 @@
use dotenv;

use clap::App;
use diesel::Connection;
use plume_models::{instance::Instance, Connection as Conn, CONFIG};
use plume_models::{db_conn::init_pool, instance::Instance};
use std::io::{self, prelude::*};

mod instance;
@@ -26,22 +25,27 @@ fn main() {
Err(ref e) if e.not_found() => eprintln!("no .env was found"),
e => e.map(|_| ()).unwrap(),
}
let conn = Conn::establish(CONFIG.database_url.as_str());
let _ = conn.as_ref().map(|conn| Instance::cache_local(conn));
let db_pool = init_pool()
.expect("Couldn't create a database pool, please check DATABASE_URL in your .env");
let _ = db_pool
.get()
.as_ref()
.map(|conn| Instance::cache_local(conn));

match matches.subcommand() {
("instance", Some(args)) => {
instance::run(args, &conn.expect("Couldn't connect to the database."))
}
("migration", Some(args)) => {
migration::run(args, &conn.expect("Couldn't connect to the database."))
}
("search", Some(args)) => {
search::run(args, &conn.expect("Couldn't connect to the database."))
}
("users", Some(args)) => {
users::run(args, &conn.expect("Couldn't connect to the database."))
}
("instance", Some(args)) => instance::run(
args,
&db_pool.get().expect("Couldn't connect to the database."),
),
("migration", Some(args)) => migration::run(
args,
&db_pool.get().expect("Couldn't connect to the database."),
),
("search", Some(args)) => search::run(args, db_pool),
("users", Some(args)) => users::run(
args,
&db_pool.get().expect("Couldn't connect to the database."),
),
_ => app.print_help().expect("Couldn't print help"),
};
}


+ 9
- 9
plume-cli/src/search.rs Voir le fichier

@@ -1,6 +1,6 @@
use clap::{App, Arg, ArgMatches, SubCommand};

use plume_models::{search::Searcher, Connection, CONFIG};
use plume_models::{db_conn::DbPool, search::Searcher, CONFIG};
use std::fs::{read_dir, remove_file};
use std::io::ErrorKind;
use std::path::Path;
@@ -52,7 +52,7 @@ pub fn command<'a, 'b>() -> App<'a, 'b> {
)
}

pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
pub fn run<'a>(args: &ArgMatches<'a>, conn: DbPool) {
let conn = conn;
match args.subcommand() {
("init", Some(x)) => init(x, conn),
@@ -63,7 +63,7 @@ pub fn run<'a>(args: &ArgMatches<'a>, conn: &Connection) {
}
}

fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
fn init<'a>(args: &ArgMatches<'a>, db_pool: DbPool) {
let path = args
.value_of("path")
.map(|p| Path::new(p).join("search_index"))
@@ -82,8 +82,8 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
}
};
if can_do || force {
let searcher = Searcher::create(&path, &CONFIG.search_tokenizers).unwrap();
refill(args, conn, Some(searcher));
let searcher = Searcher::create(&path, db_pool.clone(), &CONFIG.search_tokenizers).unwrap();
refill(args, db_pool, Some(searcher));
} else {
eprintln!(
"Can't create new index, {} exist and is not empty",
@@ -92,16 +92,16 @@ fn init<'a>(args: &ArgMatches<'a>, conn: &Connection) {
}
}

fn refill<'a>(args: &ArgMatches<'a>, conn: &Connection, searcher: Option<Searcher>) {
fn refill<'a>(args: &ArgMatches<'a>, db_pool: DbPool, searcher: Option<Searcher>) {
let path = args.value_of("path");
let path = match path {
Some(path) => Path::new(path).join("search_index"),
None => Path::new(&CONFIG.search_index).to_path_buf(),
};
let searcher =
searcher.unwrap_or_else(|| Searcher::open(&path, &CONFIG.search_tokenizers).unwrap());
let searcher = searcher
.unwrap_or_else(|| Searcher::open(&path, db_pool, &CONFIG.search_tokenizers).unwrap());

searcher.fill(conn).expect("Couldn't import post");
searcher.fill().expect("Couldn't import post");
println!("Commiting result");
searcher.commit();
}


+ 1
- 0
plume-models/Cargo.toml Voir le fichier

@@ -32,6 +32,7 @@ shrinkwraprs = "0.2.1"
diesel-derive-newtype = "0.1.2"
glob = "0.3.0"
lindera-tantivy = { version = "0.1.3", optional = true }
riker = "0.4"

[dependencies.chrono]
features = ["serde"]


BIN
plume-models/plume.db-journal Voir le fichier


BIN
plume-models/plume_tests.sqlite-journal Voir le fichier


+ 15
- 1
plume-models/src/db_conn.rs Voir le fichier

@@ -1,4 +1,4 @@
use crate::Connection;
use crate::{instance::Instance, Connection, CONFIG};
use diesel::r2d2::{
ConnectionManager, CustomizeConnection, Error as ConnError, Pool, PooledConnection,
};
@@ -13,6 +13,20 @@ use std::ops::Deref;

pub type DbPool = Pool<ConnectionManager<Connection>>;

/// Initializes a database pool.
pub fn init_pool() -> Option<DbPool> {
let manager = ConnectionManager::<Connection>::new(CONFIG.database_url.as_str());
let mut builder = DbPool::builder()
.connection_customizer(Box::new(PragmaForeignKey))
.min_idle(CONFIG.db_min_idle);
if let Some(max_size) = CONFIG.db_max_size {
builder = builder.max_size(max_size);
};
let pool = builder.build(manager).ok()?;
Instance::cache_local(&pool.get().unwrap());
Some(pool)
}

// From rocket documentation

// Connection request guard type: a wrapper around an r2d2 pooled connection.


+ 10
- 0
plume-models/src/lib.rs Voir le fichier

@@ -17,6 +17,8 @@ extern crate serde_json;
#[macro_use]
extern crate tantivy;

extern crate riker;

use plume_common::activity_pub::inbox::InboxError;

#[cfg(not(any(feature = "sqlite", feature = "postgres")))]
@@ -40,6 +42,7 @@ pub enum Error {
Io(std::io::Error),
MissingApProperty,
NotFound,
DbPool,
Request,
SerDe,
Search(search::SearcherError),
@@ -303,6 +306,10 @@ mod tests {
db_conn::DbConn((*DB_POOL).get().unwrap())
}

pub fn pool<'a>() -> db_conn::DbPool {
(*DB_POOL).clone()
}

lazy_static! {
static ref DB_POOL: db_conn::DbPool = {
let pool = db_conn::DbPool::builder()
@@ -323,6 +330,9 @@ mod tests {
searcher: Arc::new(search::tests::get_searcher(&CONFIG.search_tokenizers)),
worker: Arc::new(ScheduledThreadPool::new(2)),
user: None,
actors: Arc::new(
riker::actors::ActorSystem::new().expect("Couldn't create test actor system"),
),
}
}
}


+ 8
- 0
plume-models/src/plume_rocket.rs Voir le fichier

@@ -3,6 +3,7 @@ pub use self::module::PlumeRocket;
#[cfg(not(test))]
mod module {
use crate::{db_conn::DbConn, search, users};
use riker::actors::ActorSystem;
use rocket::{
request::{self, FlashMessage, FromRequest, Request},
Outcome, State,
@@ -18,6 +19,7 @@ mod module {
pub searcher: Arc<search::Searcher>,
pub worker: Arc<ScheduledThreadPool>,
pub flash_msg: Option<(String, String)>,
pub actors: Arc<ActorSystem>,
}

impl<'a, 'r> FromRequest<'a, 'r> for PlumeRocket {
@@ -30,6 +32,7 @@ mod module {
let worker = request.guard::<'_, State<'_, Arc<ScheduledThreadPool>>>()?;
let searcher = request.guard::<'_, State<'_, Arc<search::Searcher>>>()?;
let flash_msg = request.guard::<FlashMessage<'_, '_>>().succeeded();
let actors = request.guard::<'_, State<'_, Arc<ActorSystem>>>()?;
Outcome::Success(PlumeRocket {
conn,
intl,
@@ -37,6 +40,7 @@ mod module {
flash_msg: flash_msg.map(|f| (f.name().into(), f.msg().into())),
worker: worker.clone(),
searcher: searcher.clone(),
actors: actors.clone(),
})
}
}
@@ -45,6 +49,7 @@ mod module {
#[cfg(test)]
mod module {
use crate::{db_conn::DbConn, search, users};
use riker::actors::ActorSystem;
use rocket::{
request::{self, FromRequest, Request},
Outcome, State,
@@ -58,6 +63,7 @@ mod module {
pub user: Option<users::User>,
pub searcher: Arc<search::Searcher>,
pub worker: Arc<ScheduledThreadPool>,
pub actors: Arc<ActorSystem>,
}

impl<'a, 'r> FromRequest<'a, 'r> for PlumeRocket {
@@ -68,11 +74,13 @@ mod module {
let user = request.guard::<users::User>().succeeded();
let worker = request.guard::<'_, State<'_, Arc<ScheduledThreadPool>>>()?;
let searcher = request.guard::<'_, State<'_, Arc<search::Searcher>>>()?;
let actors = request.guard::<'_, State<'_, Arc<ActorSystem>>>()?;
Outcome::Success(PlumeRocket {
conn,
user,
worker: worker.clone(),
searcher: searcher.clone(),
actors: actors.clone(),
})
}
}


+ 10
- 8
plume-models/src/posts.rs Voir le fichier

@@ -1,7 +1,7 @@
use crate::{
ap_url, blogs::Blog, instance::Instance, medias::Media, mentions::Mention, post_authors::*,
safe_string::SafeString, schema::posts, search::Searcher, tags::*, timeline::*, users::User,
Connection, Error, PlumeRocket, Result, CONFIG,
safe_string::SafeString, schema::posts, search::Searcher, search::UpdateDocument, tags::*,
timeline::*, users::User, Connection, Error, PlumeRocket, Result, CONFIG,
};
use activitypub::{
activity::{Create, Delete, Update},
@@ -19,12 +19,14 @@ use plume_common::{
},
utils::md_to_html,
};

use riker::actors::*;
use serde_json;
use std::collections::HashSet;

pub type LicensedArticle = CustomObject<Licensed, Article>;

#[derive(Queryable, Identifiable, Clone, AsChangeset)]
#[derive(Debug, Queryable, Identifiable, Clone, AsChangeset)]
#[changeset_options(treat_none_as_null = "true")]
pub struct Post {
pub id: i32,
@@ -78,14 +80,13 @@ impl Post {
let _: Post = post.save_changes(conn)?;
}

searcher.add_document(conn, &post)?;
searcher.add_document(&post)?;
Ok(post)
}

pub fn update(&self, conn: &Connection, searcher: &Searcher) -> Result<Self> {
pub fn update(&self, conn: &Connection) -> Result<Self> {
diesel::update(self).set(self).execute(conn)?;
let post = Self::get(conn, self.id)?;
searcher.update_document(conn, &post)?;
Ok(post)
}

@@ -728,7 +729,7 @@ impl AsObject<User, Update, &PlumeRocket> for PostUpdate {

fn activity(self, c: &PlumeRocket, actor: User, _id: &str) -> Result<()> {
let conn = &*c.conn;
let searcher = &c.searcher;
let searcher_actor = &c.actors.select("searcher-actor").unwrap();
let mut post = Post::from_id(c, &self.ap_url, None).map_err(|(_, e)| e)?;

if !post.is_author(conn, actor.id)? {
@@ -791,7 +792,8 @@ impl AsObject<User, Update, &PlumeRocket> for PostUpdate {
post.update_hashtags(conn, hashtags)?;
}

post.update(conn, searcher)?;
post.update(conn)?;
searcher_actor.try_tell(UpdateDocument(post.clone()), None);
Ok(())
}
}


+ 14
- 13
plume-models/src/search/mod.rs Voir le fichier

@@ -7,7 +7,7 @@ pub use self::tokenizer::TokenizerKind;

#[cfg(test)]
pub(crate) mod tests {
use super::{Query, Searcher, TokenizerKind};
use super::{Query, Searcher};
use diesel::Connection;
use plume_common::utils::random_hex;
use std::env::temp_dir;
@@ -20,15 +20,16 @@ pub(crate) mod tests {
posts::{NewPost, Post},
safe_string::SafeString,
tests::db,
tests::pool,
CONFIG,
};

pub(crate) fn get_searcher(tokenizers: &SearchTokenizerConfig) -> Searcher {
let dir = temp_dir().join(&format!("plume-test-{}", random_hex()));
if dir.exists() {
Searcher::open(&dir, tokenizers)
Searcher::open(&dir, pool(), tokenizers)
} else {
Searcher::create(&dir, tokenizers)
Searcher::create(&dir, pool(), tokenizers)
}
.unwrap()
}
@@ -103,20 +104,20 @@ pub(crate) mod tests {
fn open() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));
{
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::create(&dir, pool(), &CONFIG.search_tokenizers).unwrap();
}
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).unwrap();
}

#[test]
fn create() {
let dir = temp_dir().join(format!("plume-test-{}", random_hex()));

assert!(Searcher::open(&dir, &CONFIG.search_tokenizers).is_err());
assert!(Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).is_err());
{
Searcher::create(&dir, &CONFIG.search_tokenizers).unwrap();
Searcher::create(&dir, pool(), &CONFIG.search_tokenizers).unwrap();
}
Searcher::open(&dir, &CONFIG.search_tokenizers).unwrap(); //verify it's well created
Searcher::open(&dir, pool(), &CONFIG.search_tokenizers).unwrap(); //verify it's well created
}

#[test]
@@ -158,26 +159,26 @@ pub(crate) mod tests {

searcher.commit();
assert_eq!(
searcher.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))[0].id,
searcher.search_document(Query::from_str(&title).unwrap(), (0, 1))[0].id,
post.id
);

let newtitle = random_hex()[..8].to_owned();
post.title = newtitle.clone();
post.update(conn, &searcher).unwrap();
post.update(conn).unwrap();
searcher.commit();
assert_eq!(
searcher.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))[0].id,
searcher.search_document(Query::from_str(&newtitle).unwrap(), (0, 1))[0].id,
post.id
);
assert!(searcher
.search_document(conn, Query::from_str(&title).unwrap(), (0, 1))
.search_document(Query::from_str(&title).unwrap(), (0, 1))
.is_empty());

post.delete(conn, &searcher).unwrap();
searcher.commit();
assert!(searcher
.search_document(conn, Query::from_str(&newtitle).unwrap(), (0, 1))
.search_document(Query::from_str(&newtitle).unwrap(), (0, 1))
.is_empty());
Ok(())
});


+ 186
- 23
plume-models/src/search/searcher.rs Voir le fichier

@@ -1,11 +1,12 @@
use crate::{
config::SearchTokenizerConfig, instance::Instance, posts::Post, schema::posts,
search::query::PlumeQuery, tags::Tag, Connection, Result,
config::SearchTokenizerConfig, db_conn::DbPool, instance::Instance, posts::Post, schema::posts,
search::query::PlumeQuery, tags::Tag, Error, Result, CONFIG,
};
use chrono::Datelike;
use chrono::{Datelike, Utc};
use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl};
use itertools::Itertools;
use std::{cmp, fs::create_dir_all, io, path::Path, sync::Mutex};
use riker::actors::*;
use std::{cmp, fs::create_dir_all, io, path::Path, sync::Arc, sync::Mutex};
use tantivy::{
collector::TopDocs, directory::MmapDirectory, schema::*, Index, IndexReader, IndexWriter,
ReloadPolicy, TantivyError, Term,
@@ -25,9 +26,153 @@ pub struct Searcher {
index: Index,
reader: IndexReader,
writer: Mutex<Option<IndexWriter>>,
dbpool: DbPool,
}

#[derive(Clone, Debug)]
pub struct AddDocument(Post);

#[derive(Clone, Debug)]
pub struct UpdateDocument(pub Post);

#[derive(Clone, Debug)]
pub struct DeleteDocument(Post);

#[actor(AddDocument, UpdateDocument)]
pub struct SearcherActor(Searcher);

impl Actor for SearcherActor {
type Msg = SearcherActorMsg;

// forwards Msg to the respective Receive<T> implementation
fn recv(&mut self, ctx: &Context<Self::Msg>, msg: Self::Msg, sender: Sender) {
self.receive(ctx, msg, sender);
}
}

impl Receive<AddDocument> for SearcherActor {
type Msg = SearcherActorMsg;

fn receive(&mut self, _ctx: &Context<Self::Msg>, msg: AddDocument, _sender: Sender) {
let _ = self.0.add_document(&msg.0);
}
}

impl Receive<UpdateDocument> for SearcherActor {
type Msg = SearcherActorMsg;

fn receive(&mut self, _ctx: &Context<Self::Msg>, msg: UpdateDocument, _sender: Sender) {
let _ = self.0.update_document(&msg.0);
}
}

impl ActorFactoryArgs<Arc<Searcher>> for SearcherActor {
fn create_args(searcher: Arc<Searcher>) -> Self {
SearcherActor(Arc::try_unwrap(searcher).ok().unwrap())
}
}

impl Searcher {
/// Initializes a new `Searcher`, ready to be used by
/// Plume.
///
/// The main task of this function is to try everything
/// to get a valid `Searcher`:
///
/// - first it tries to open the search index normally (using the options from `CONFIG`)
/// - if it fails, it makes a back-up of the index files, deletes the original ones,
/// and recreate the whole index. It removes the backup only if the re-creation
/// succeeds.
///
/// # Panics
///
/// This function panics if it needs to create a backup and it can't, or if it fails
/// to recreate the search index.
///
/// After that, it can also panic if there are still errors remaining.
///
/// The panic messages are normally explicit enough for a human to
/// understand how to fix the issue when they see it.
pub fn new(db_pool: DbPool) -> Self {
// We try to open the index a first time
let searcher = match Self::open(
&CONFIG.search_index,
db_pool.clone(),
&CONFIG.search_tokenizers,
) {
// The index may be corrupted, inexistent or use an older format.
// In this case, we can easily recover by deleting and re-creating it.
Err(Error::Search(SearcherError::InvalidIndexDataError)) => {
if Self::create(
&CONFIG.search_index,
db_pool.clone(),
&CONFIG.search_tokenizers,
)
.is_err()
{
let current_path = Path::new(&CONFIG.search_index);
let backup_path =
format!("{}.{}", &current_path.display(), Utc::now().timestamp());
let backup_path = Path::new(&backup_path);
std::fs::rename(current_path, backup_path)
.expect("Error while backing up search index directory for re-creation");
if Self::create(
&CONFIG.search_index,
db_pool.clone(),
&CONFIG.search_tokenizers,
)
.is_ok()
{
if std::fs::remove_dir_all(backup_path).is_err() {
eprintln!(
"error on removing backup directory: {}. it remains",
backup_path.display()
);
}
} else {
panic!("Error while re-creating search index in new index format. Remove search index and run `plm search init` manually.");
}
}
Self::open(&CONFIG.search_index, db_pool, &CONFIG.search_tokenizers)
}
// If it opened successfully or if it was another kind of
// error (that we don't know how to handle), don't do anything more
other => other,
};

// At this point, if there are still errors, we just panic
#[allow(clippy::match_wild_err_arm)]
match searcher {
Err(Error::Search(e)) => match e {
SearcherError::WriteLockAcquisitionError => panic!(
r#"
Your search index is locked. Plume can't start. To fix this issue
make sure no other Plume instance is started, and run:

plm search unlock

Then try to restart Plume.
"#
),
SearcherError::IndexOpeningError => panic!(
r#"
Plume was unable to open the search index. If you created the index
before, make sure to run Plume in the same directory it was created in, or
to set SEARCH_INDEX accordingly. If you did not yet create the search
index, run this command:

plm search init

Then try to restart Plume
"#
),
e => Err(e).unwrap(),
},
Err(_) => panic!("Unexpected error while opening search index"),
Ok(s) => s,
}
}

pub fn schema() -> Schema {
let tag_indexing = TextOptions::default().set_indexing_options(
TextFieldIndexing::default()
@@ -67,7 +212,11 @@ impl Searcher {
schema_builder.build()
}

pub fn create(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
pub fn create(
path: &dyn AsRef<Path>,
dbpool: DbPool,
tokenizers: &SearchTokenizerConfig,
) -> Result<Self> {
let schema = Self::schema();

create_dir_all(path).map_err(|_| SearcherError::IndexCreationError)?;
@@ -95,10 +244,15 @@ impl Searcher {
.try_into()
.map_err(|_| SearcherError::IndexCreationError)?,
index,
dbpool,
})
}

pub fn open(path: &dyn AsRef<Path>, tokenizers: &SearchTokenizerConfig) -> Result<Self> {
pub fn open(
path: &dyn AsRef<Path>,
dbpool: DbPool,
tokenizers: &SearchTokenizerConfig,
) -> Result<Self> {
let mut index =
Index::open(MmapDirectory::open(path).map_err(|_| SearcherError::IndexOpeningError)?)
.map_err(|_| SearcherError::IndexOpeningError)?;
@@ -150,10 +304,11 @@ impl Searcher {
}
})?,
index,
dbpool,
})
}

pub fn add_document(&self, conn: &Connection, post: &Post) -> Result<()> {
pub fn add_document(&self, post: &Post) -> Result<()> {
if !post.published {
return Ok(());
}
@@ -175,15 +330,19 @@ impl Searcher {
let lang = schema.get_field("lang").unwrap();
let license = schema.get_field("license").unwrap();

let conn = match self.dbpool.get() {
Ok(c) => c,
Err(_) => return Err(Error::DbPool),
};
let mut writer = self.writer.lock().unwrap();
let writer = writer.as_mut().unwrap();
writer.add_document(doc!(
post_id => i64::from(post.id),
author => post.get_authors(conn)?.into_iter().map(|u| u.fqn).join(" "),
author => post.get_authors(&conn)?.into_iter().map(|u| u.fqn).join(" "),
creation_date => i64::from(post.creation_date.num_days_from_ce()),
instance => Instance::get(conn, post.get_blog(conn)?.instance_id)?.public_domain,
tag => Tag::for_post(conn, post.id)?.into_iter().map(|t| t.tag).join(" "),
blog_name => post.get_blog(conn)?.title,
instance => Instance::get(&conn, post.get_blog(&conn)?.instance_id)?.public_domain,
tag => Tag::for_post(&conn, post.id)?.into_iter().map(|t| t.tag).join(" "),
blog_name => post.get_blog(&conn)?.title,
content => post.content.get().clone(),
subtitle => post.subtitle.clone(),
title => post.title.clone(),
@@ -203,17 +362,12 @@ impl Searcher {
writer.delete_term(doc_id);
}

pub fn update_document(&self, conn: &Connection, post: &Post) -> Result<()> {
pub fn update_document(&self, post: &Post) -> Result<()> {
self.delete_document(post);
self.add_document(conn, post)
self.add_document(post)
}

pub fn search_document(
&self,
conn: &Connection,
query: PlumeQuery,
(min, max): (i32, i32),
) -> Vec<Post> {
pub fn search_document(&self, query: PlumeQuery, (min, max): (i32, i32)) -> Vec<Post> {
let schema = self.index.schema();
let post_id = schema.get_field("post_id").unwrap();

@@ -222,24 +376,33 @@ impl Searcher {
let searcher = self.reader.searcher();
let res = searcher.search(&query.into_query(), &collector).unwrap();

let conn = match self.dbpool.get() {
Ok(c) => c,
Err(_) => return Vec::new(),
};

res.get(min as usize..)
.unwrap_or(&[])
.iter()
.filter_map(|(_, doc_add)| {
let doc = searcher.doc(*doc_add).ok()?;
let id = doc.get_first(post_id)?;
Post::get(conn, id.i64_value() as i32).ok()
Post::get(&conn, id.i64_value() as i32).ok()
//borrow checker don't want me to use filter_map or and_then here
})
.collect()
}

pub fn fill(&self, conn: &Connection) -> Result<()> {
pub fn fill(&self) -> Result<()> {
let conn = match self.dbpool.get() {
Ok(c) => c,
Err(_) => return Err(Error::DbPool),
};
for post in posts::table
.filter(posts::published.eq(true))
.load::<Post>(conn)?
.load::<Post>(&conn)?
{
self.update_document(conn, &post)?
self.update_document(&post)?
}
Ok(())
}


+ 18
- 81
src/main.rs Voir le fichier

@@ -10,20 +10,16 @@ extern crate serde_json;
#[macro_use]
extern crate validator_derive;

use chrono::Utc;
extern crate riker;

use clap::App;
use diesel::r2d2::ConnectionManager;
use plume_models::{
db_conn::{DbPool, PragmaForeignKey},
instance::Instance,
migrations::IMPORTED_MIGRATIONS,
search::{Searcher as UnmanagedSearcher, SearcherError},
Connection, Error, CONFIG,
db_conn::init_pool, migrations::IMPORTED_MIGRATIONS, search::Searcher, search::SearcherActor,
CONFIG,
};
use riker::actors::*;
use rocket_csrf::CsrfFairingBuilder;
use scheduled_thread_pool::ScheduledThreadPool;
use std::fs;
use std::path::Path;
use std::process::exit;
use std::sync::{Arc, Mutex};
use std::time::Duration;
@@ -48,26 +44,6 @@ include!(concat!(env!("OUT_DIR"), "/templates.rs"));

compile_i18n!();

/// Initializes a database pool.
fn init_pool() -> Option<DbPool> {
match dotenv::dotenv() {
Ok(path) => println!("Configuration read from {}", path.display()),
Err(ref e) if e.not_found() => eprintln!("no .env was found"),
e => e.map(|_| ()).unwrap(),
}

let manager = ConnectionManager::<Connection>::new(CONFIG.database_url.as_str());
let mut builder = DbPool::builder()
.connection_customizer(Box::new(PragmaForeignKey))
.min_idle(CONFIG.db_min_idle);
if let Some(max_size) = CONFIG.db_max_size {
builder = builder.max_size(max_size);
};
let pool = builder.build(manager).ok()?;
Instance::cache_local(&pool.get().unwrap());
Some(pool)
}

fn main() {
App::new("Plume")
.bin_name("plume")
@@ -82,6 +58,13 @@ and https://docs.joinplu.me/installation/init for more info.
"#,
)
.get_matches();

match dotenv::dotenv() {
Ok(path) => println!("Configuration read from {}", path.display()),
Err(ref e) if e.not_found() => eprintln!("no .env was found"),
e => e.map(|_| ()).unwrap(),
}

let dbpool = init_pool().expect("main: database pool initialization error");
if IMPORTED_MIGRATIONS
.is_pending(&dbpool.get().unwrap())
@@ -100,59 +83,12 @@ Then try to restart Plume.
)
}
let workpool = ScheduledThreadPool::with_name("worker {}", num_cpus::get());
// we want a fast exit here, so
let mut open_searcher =
UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers);
if let Err(Error::Search(SearcherError::InvalidIndexDataError)) = open_searcher {
if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_err() {
let current_path = Path::new(&CONFIG.search_index);
let backup_path = format!("{}.{}", &current_path.display(), Utc::now().timestamp());
let backup_path = Path::new(&backup_path);
fs::rename(current_path, backup_path)
.expect("main: error on backing up search index directory for recreating");
if UnmanagedSearcher::create(&CONFIG.search_index, &CONFIG.search_tokenizers).is_ok() {
if fs::remove_dir_all(backup_path).is_err() {
eprintln!(
"error on removing backup directory: {}. it remains",
backup_path.display()
);
}
} else {
panic!("main: error on recreating search index in new index format. remove search index and run `plm search init` manually");
}
}
open_searcher = UnmanagedSearcher::open(&CONFIG.search_index, &CONFIG.search_tokenizers);
}
#[allow(clippy::match_wild_err_arm)]
let searcher = match open_searcher {
Err(Error::Search(e)) => match e {
SearcherError::WriteLockAcquisitionError => panic!(
r#"
Your search index is locked. Plume can't start. To fix this issue
make sure no other Plume instance is started, and run:

plm search unlock
let searcher = Arc::new(Searcher::new(dbpool.clone()));

Then try to restart Plume.
"#
),
SearcherError::IndexOpeningError => panic!(
r#"
Plume was unable to open the search index. If you created the index
before, make sure to run Plume in the same directory it was created in, or
to set SEARCH_INDEX accordingly. If you did not yet create the search
index, run this command:

plm search init

Then try to restart Plume
"#
),
e => Err(e).unwrap(),
},
Err(_) => panic!("Unexpected error while opening search index"),
Ok(s) => Arc::new(s),
};
let sys = SystemBuilder::new().name("plume").create().unwrap();
let _ = sys
.actor_of_args::<SearcherActor, _>("searcher-actor", searcher.clone())
.unwrap();

let commiter = searcher.clone();
workpool.execute_with_fixed_delay(
@@ -299,6 +235,7 @@ Then try to restart Plume
.manage(dbpool)
.manage(Arc::new(workpool))
.manage(searcher)
.manage(Arc::new(sys))
.manage(include_i18n!())
.attach(
CsrfFairingBuilder::new()


+ 5
- 2
src/routes/posts.rs Voir le fichier

@@ -1,5 +1,6 @@
use chrono::Utc;
use heck::{CamelCase, KebabCase};
use riker::actors::*;
use rocket::request::LenientForm;
use rocket::response::{Flash, Redirect};
use rocket_i18n::I18n;
@@ -26,6 +27,7 @@ use plume_models::{
post_authors::*,
posts::*,
safe_string::SafeString,
search::UpdateDocument,
tags::*,
timeline::*,
users::User,
@@ -297,8 +299,6 @@ pub fn update(
post.source = form.content.clone();
post.license = form.license.clone();
post.cover_id = form.cover;
post.update(&*conn, &rockets.searcher)
.expect("post::update: update error");

if post.published {
post.update_mentions(
@@ -351,6 +351,9 @@ pub fn update(
}
}

let searcher_actor = rockets.actors.select("searcher-actor").unwrap();
searcher_actor.try_tell(UpdateDocument(post.clone()), None);

Flash::success(
Redirect::to(uri!(details: blog = blog, slug = new_slug, responding_to = _)),
i18n!(intl, "Your article has been updated."),


+ 1
- 2
src/routes/search.rs Voir le fichier

@@ -51,7 +51,6 @@ macro_rules! param_to_query {

#[get("/search?<query..>")]
pub fn search(query: Option<Form<SearchQuery>>, rockets: PlumeRocket) -> Ructe {
let conn = &*rockets.conn;
let query = query.map(Form::into_inner).unwrap_or_default();
let page = query.page.unwrap_or_default();
let mut parsed_query =
@@ -72,7 +71,7 @@ pub fn search(query: Option<Form<SearchQuery>>, rockets: PlumeRocket) -> Ructe {
} else {
let res = rockets
.searcher
.search_document(&conn, parsed_query, page.limits());
.search_document(parsed_query, page.limits());
let next_page = if res.is_empty() { 0 } else { page.0 + 1 };
render!(search::result(
&rockets.to_context(),


Chargement…
Annuler
Enregistrer