filtron.sh 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629
  1. #!/usr/bin/env bash
  2. # SPDX-License-Identifier: AGPL-3.0-or-later
  3. # shellcheck disable=SC2001
  4. # shellcheck source=utils/lib.sh
  5. source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
  6. # shellcheck source=utils/lib_go.sh
  7. source "${REPO_ROOT}/utils/lib_go.sh"
  8. # shellcheck source=utils/lib_install.sh
  9. source "${REPO_ROOT}/utils/lib_install.sh"
  10. # ----------------------------------------------------------------------------
  11. # config
  12. # ----------------------------------------------------------------------------
  13. PUBLIC_HOST="${PUBLIC_HOST:-$(echo "$PUBLIC_URL" | sed -e 's/[^/]*\/\/\([^@]*@\)\?\([^:/]*\).*/\2/')}"
  14. FILTRON_URL_PATH="${FILTRON_URL_PATH:-$(echo "${PUBLIC_URL}" \
  15. | sed -e 's,^.*://[^/]*\(/.*\),\1,g')}"
  16. [[ "${FILTRON_URL_PATH}" == "${PUBLIC_URL}" ]] && FILTRON_URL_PATH=/
  17. FILTRON_ETC="/etc/filtron"
  18. FILTRON_RULES="$FILTRON_ETC/rules.json"
  19. FILTRON_RULES_TEMPLATE="${FILTRON_RULES_TEMPLATE:-${REPO_ROOT}/utils/templates/etc/filtron/rules.json}"
  20. FILTRON_API="${FILTRON_API:-127.0.0.1:4005}"
  21. FILTRON_LISTEN="${FILTRON_LISTEN:-127.0.0.1:4004}"
  22. # The filtron target is the SearXNG installation, listenning on server.port at
  23. # server.bind_address. The default of FILTRON_TARGET is taken from the YAML
  24. # configuration, do not change this value without reinstalling the entire
  25. # SearXNG suite including filtron & morty.
  26. FILTRON_TARGET="${SEARXNG_BIND_ADDRESS}:${SEARXNG_PORT}"
  27. SERVICE_NAME="filtron"
  28. SERVICE_USER="${SERVICE_USER:-${SERVICE_NAME}}"
  29. SERVICE_HOME_BASE="${SERVICE_HOME_BASE:-/usr/local}"
  30. SERVICE_HOME="${SERVICE_HOME_BASE}/${SERVICE_USER}"
  31. SERVICE_SYSTEMD_UNIT="${SYSTEMD_UNITS}/${SERVICE_NAME}.service"
  32. # shellcheck disable=SC2034
  33. SERVICE_GROUP="${SERVICE_USER}"
  34. # shellcheck disable=SC2034
  35. SERVICE_GROUP="${SERVICE_USER}"
  36. GO_ENV="${SERVICE_HOME}/.go_env"
  37. GO_VERSION="go1.17.2"
  38. APACHE_FILTRON_SITE="searxng.conf"
  39. NGINX_FILTRON_SITE="searxng.conf"
  40. # shellcheck disable=SC2034
  41. CONFIG_FILES=(
  42. "${FILTRON_RULES}"
  43. "${SERVICE_SYSTEMD_UNIT}"
  44. )
  45. # ----------------------------------------------------------------------------
  46. usage() {
  47. # ----------------------------------------------------------------------------
  48. # shellcheck disable=SC1117
  49. cat <<EOF
  50. usage::
  51. $(basename "$0") shell
  52. $(basename "$0") install [all|user|rules]
  53. $(basename "$0") reinstall all
  54. $(basename "$0") update [filtron]
  55. $(basename "$0") remove [all]
  56. $(basename "$0") activate [service]
  57. $(basename "$0") deactivate [service]
  58. $(basename "$0") inspect [service]
  59. $(basename "$0") option [debug-on|debug-off]
  60. $(basename "$0") apache [install|remove]
  61. $(basename "$0") nginx [install|remove]
  62. shell
  63. start interactive shell from user ${SERVICE_USER}
  64. install / remove
  65. :all: complete setup of filtron service
  66. :user: add/remove service user '$SERVICE_USER' ($SERVICE_HOME)
  67. :rules: reinstall filtron rules $FILTRON_RULES
  68. install
  69. :check: check the filtron installation
  70. reinstall:
  71. :all: runs 'install/remove all'
  72. update filtron
  73. Update filtron installation ($SERVICE_HOME)
  74. activate service
  75. activate and start service daemon (systemd unit)
  76. deactivate service
  77. stop and deactivate service daemon (systemd unit)
  78. inspect service
  79. show service status and log
  80. option
  81. set one of the available options
  82. apache (${PUBLIC_URL})
  83. :install: apache site with a reverse proxy (ProxyPass)
  84. :remove: apache site ${APACHE_FILTRON_SITE}
  85. nginx (${PUBLIC_URL})
  86. :install: nginx site with a reverse proxy (ProxyPass)
  87. :remove: nginx site ${NGINX_FILTRON_SITE}
  88. filtron rules: ${FILTRON_RULES_TEMPLATE}
  89. ---- sourced ${DOT_CONFIG} :
  90. SERVICE_USER : ${SERVICE_USER}
  91. SERVICE_HOME : ${SERVICE_HOME}
  92. FILTRON_TARGET : ${FILTRON_TARGET}
  93. FILTRON_API : ${FILTRON_API}
  94. FILTRON_LISTEN : ${FILTRON_LISTEN}
  95. FILTRON_URL_PATH : ${FILTRON_URL_PATH}
  96. EOF
  97. install_log_searx_instance
  98. [[ -n ${1} ]] && err_msg "$1"
  99. }
  100. main() {
  101. required_commands \
  102. sudo install git wget curl \
  103. || exit
  104. local _usage="unknown or missing $1 command $2"
  105. case $1 in
  106. --getenv) var="$2"; echo "${!var}"; exit 0;;
  107. -h|--help) usage; exit 0;;
  108. shell)
  109. sudo_or_exit
  110. interactive_shell "${SERVICE_USER}"
  111. ;;
  112. inspect)
  113. case $2 in
  114. service)
  115. sudo_or_exit
  116. inspect_service
  117. ;;
  118. *) usage "$_usage"; exit 42;;
  119. esac ;;
  120. reinstall)
  121. rst_title "re-install $SERVICE_NAME" part
  122. sudo_or_exit
  123. case $2 in
  124. all)
  125. remove_all
  126. install_all
  127. ;;
  128. *) usage "$_usage"; exit 42;;
  129. esac ;;
  130. install)
  131. rst_title "$SERVICE_NAME" part
  132. sudo_or_exit
  133. case $2 in
  134. check)
  135. rst_title "Check filtron installation" part
  136. install_check
  137. ;;
  138. all) install_all ;;
  139. user) assert_user ;;
  140. rules)
  141. install_rules
  142. systemd_restart_service "${SERVICE_NAME}"
  143. ;;
  144. *) usage "$_usage"; exit 42;;
  145. esac ;;
  146. update)
  147. sudo_or_exit
  148. case $2 in
  149. filtron) update_filtron ;;
  150. *) usage "$_usage"; exit 42;;
  151. esac ;;
  152. remove)
  153. sudo_or_exit
  154. case $2 in
  155. all) remove_all;;
  156. user) drop_service_account "${SERVICE_USER}" ;;
  157. *) usage "$_usage"; exit 42;;
  158. esac ;;
  159. activate)
  160. sudo_or_exit
  161. case $2 in
  162. service) systemd_activate_service "${SERVICE_NAME}" ;;
  163. *) usage "$_usage"; exit 42;;
  164. esac ;;
  165. deactivate)
  166. sudo_or_exit
  167. case $2 in
  168. service) systemd_deactivate_service "${SERVICE_NAME}" ;;
  169. *) usage "$_usage"; exit 42;;
  170. esac ;;
  171. apache)
  172. sudo_or_exit
  173. case $2 in
  174. install) install_apache_site ;;
  175. remove) remove_apache_site ;;
  176. *) usage "$_usage"; exit 42;;
  177. esac ;;
  178. nginx)
  179. sudo_or_exit
  180. case $2 in
  181. install) install_nginx_site ;;
  182. remove) remove_nginx_site ;;
  183. *) usage "$_usage"; exit 42;;
  184. esac ;;
  185. option)
  186. sudo_or_exit
  187. case $2 in
  188. debug-on) echo; enable_debug ;;
  189. debug-off) echo; disable_debug ;;
  190. *) usage "$_usage"; exit 42;;
  191. esac ;;
  192. doc) rst-doc ;;
  193. *) usage "unknown or missing command $1"; exit 42;;
  194. esac
  195. }
  196. install_all() {
  197. rst_title "Install $SERVICE_NAME (service)"
  198. assert_user
  199. wait_key
  200. go.golang "${GO_VERSION}" "${SERVICE_USER}"
  201. wait_key
  202. install_filtron
  203. install_rules
  204. wait_key
  205. systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
  206. wait_key
  207. echo
  208. if ! service_is_available "http://${FILTRON_LISTEN}" ; then
  209. err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
  210. fi
  211. if apache_is_installed; then
  212. info_msg "Apache is installed on this host."
  213. if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
  214. install_apache_site
  215. fi
  216. elif nginx_is_installed; then
  217. info_msg "nginx is installed on this host."
  218. if ask_yn "Do you want to install a reverse proxy (ProxyPass)" Yn; then
  219. install_nginx_site
  220. fi
  221. fi
  222. if ask_yn "Do you want to inspect the installation?" Ny; then
  223. inspect_service
  224. fi
  225. }
  226. install_check() {
  227. if service_account_is_available "$SERVICE_USER"; then
  228. info_msg "service account $SERVICE_USER available."
  229. else
  230. err_msg "service account $SERVICE_USER not available!"
  231. fi
  232. if go_is_available "$SERVICE_USER"; then
  233. info_msg "~$SERVICE_USER: go is installed"
  234. else
  235. err_msg "~$SERVICE_USER: go is not installed"
  236. fi
  237. if filtron_is_installed; then
  238. info_msg "~$SERVICE_USER: filtron app is installed"
  239. else
  240. err_msg "~$SERVICE_USER: filtron app is not installed!"
  241. fi
  242. if ! service_is_available "http://${FILTRON_API}"; then
  243. err_msg "API not available at: http://${FILTRON_API}"
  244. fi
  245. if ! service_is_available "http://${FILTRON_LISTEN}" ; then
  246. err_msg "Filtron is not listening on: http://${FILTRON_LISTEN}"
  247. fi
  248. if service_is_available "http://${FILTRON_TARGET}" ; then
  249. info_msg "Filtron's target is available at: http://${FILTRON_TARGET}"
  250. fi
  251. if ! service_is_available "${PUBLIC_URL}"; then
  252. warn_msg "Public service at ${PUBLIC_URL} is not available!"
  253. if ! in_container; then
  254. warn_msg "Check if public name is correct and routed or use the public IP from above."
  255. fi
  256. fi
  257. if [[ "${GO_VERSION}" > "$(go_version)" ]]; then
  258. warn_msg "golang ($(go_version)) needs to be $GO_VERSION at least"
  259. warn_msg "you need to reinstall $SERVICE_USER --> $0 reinstall all"
  260. else
  261. info_msg "golang $(go_version) is installed (min needed is: $GO_VERSION)"
  262. fi
  263. if [ -f "${APACHE_SITES_AVAILABLE}/searx.conf" ]; then
  264. warn_msg "old searx.conf apache site exists"
  265. fi
  266. if [ -f "${NGINX_APPS_AVAILABLE}/searx.conf" ]; then
  267. warn_msg "old searx.conf nginx site exists"
  268. fi
  269. }
  270. go_version(){
  271. go.version "${SERVICE_USER}"
  272. }
  273. remove_all() {
  274. rst_title "De-Install $SERVICE_NAME (service)"
  275. rst_para "\
  276. It goes without saying that this script can only be used to remove
  277. installations that were installed with this script."
  278. if ! systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"; then
  279. return 42
  280. fi
  281. drop_service_account "${SERVICE_USER}"
  282. rm -r "$FILTRON_ETC" 2>&1 | prefix_stdout
  283. if service_is_available "${PUBLIC_URL}"; then
  284. MSG="** Don't forget to remove your public site! (${PUBLIC_URL}) **" wait_key 10
  285. fi
  286. }
  287. assert_user() {
  288. rst_title "user $SERVICE_USER" section
  289. echo
  290. tee_stderr 1 <<EOF | bash | prefix_stdout
  291. useradd --shell /bin/bash --system \
  292. --home-dir "$SERVICE_HOME" \
  293. --comment 'Reverse HTTP proxy to filter requests' $SERVICE_USER
  294. mkdir "$SERVICE_HOME"
  295. chown -R "$SERVICE_GROUP:$SERVICE_GROUP" "$SERVICE_HOME"
  296. groups $SERVICE_USER
  297. EOF
  298. SERVICE_HOME="$(sudo -i -u "$SERVICE_USER" echo \$HOME)"
  299. export SERVICE_HOME
  300. echo "export SERVICE_HOME=$SERVICE_HOME"
  301. tee_stderr <<EOF | sudo -i -u "$SERVICE_USER"
  302. touch "$GO_ENV"
  303. grep -qFs -- 'source "$GO_ENV"' ~/.profile || echo 'source "$GO_ENV"' >> ~/.profile
  304. EOF
  305. }
  306. filtron_is_installed() {
  307. [[ -f $SERVICE_HOME/go-apps/bin/filtron ]]
  308. }
  309. install_filtron() {
  310. rst_title "Install filtron in user's ~/go-apps" section
  311. echo
  312. go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
  313. }
  314. update_filtron() {
  315. rst_title "Update filtron" section
  316. echo
  317. go.install github.com/searxng/filtron@latest "${SERVICE_USER}"
  318. }
  319. install_rules() {
  320. rst_title "Install filtron rules"
  321. echo
  322. if [[ ! -f "${FILTRON_RULES}" ]]; then
  323. info_msg "install rules ${FILTRON_RULES_TEMPLATE}"
  324. info_msg " --> ${FILTRON_RULES}"
  325. mkdir -p "$(dirname "${FILTRON_RULES}")"
  326. cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
  327. return
  328. fi
  329. if cmp --silent "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"; then
  330. info_msg "${FILTRON_RULES} is up to date with"
  331. info_msg "${FILTRON_RULES_TEMPLATE}"
  332. return
  333. fi
  334. rst_para "Diff between origin's rules file (+) and current (-):"
  335. echo "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
  336. $DIFF_CMD "${FILTRON_RULES}" "${FILTRON_RULES_TEMPLATE}"
  337. local action
  338. choose_one action "What should happen to the rules file? " \
  339. "keep configuration unchanged" \
  340. "use origin rules" \
  341. "start interactive shell"
  342. case $action in
  343. "keep configuration unchanged")
  344. info_msg "leave rules file unchanged"
  345. ;;
  346. "use origin rules")
  347. backup_file "${FILTRON_RULES}"
  348. info_msg "install origin rules"
  349. cp "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
  350. ;;
  351. "start interactive shell")
  352. backup_file "${FILTRON_RULES}"
  353. echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
  354. sudo -H -i
  355. rst_para 'Diff between new rules file (-) and current (+):'
  356. echo
  357. $DIFF_CMD "${FILTRON_RULES_TEMPLATE}" "${FILTRON_RULES}"
  358. wait_key
  359. ;;
  360. esac
  361. }
  362. inspect_service() {
  363. rst_title "service status & log"
  364. cat <<EOF
  365. sourced ${DOT_CONFIG} :
  366. SERVICE_USER : ${SERVICE_USER}
  367. SERVICE_HOME : ${SERVICE_HOME}
  368. FILTRON_TARGET : ${FILTRON_TARGET}
  369. FILTRON_API : ${FILTRON_API}
  370. FILTRON_LISTEN : ${FILTRON_LISTEN}
  371. FILTRON_URL_PATH : ${FILTRON_URL_PATH}
  372. EOF
  373. install_log_searx_instance
  374. install_check
  375. if in_container; then
  376. lxc_suite_info
  377. else
  378. info_msg "public URL --> ${PUBLIC_URL}"
  379. info_msg "internal URL --> http://${FILTRON_LISTEN}"
  380. fi
  381. local _debug_on
  382. if ask_yn "Enable filtron debug mode?"; then
  383. enable_debug
  384. _debug_on=1
  385. fi
  386. echo
  387. systemctl --no-pager -l status "${SERVICE_NAME}"
  388. echo
  389. info_msg "public URL --> ${PUBLIC_URL}"
  390. # shellcheck disable=SC2059
  391. printf "// use ${_BCyan}CTRL-C${_creset} to stop monitoring the log"
  392. read -r -s -n1 -t 5
  393. echo
  394. while true; do
  395. trap break 2
  396. journalctl -f -u "${SERVICE_NAME}"
  397. done
  398. if [[ $_debug_on == 1 ]]; then
  399. disable_debug
  400. fi
  401. return 0
  402. }
  403. enable_debug() {
  404. info_msg "try to enable debug mode ..."
  405. python <<EOF
  406. import sys, json
  407. debug = {
  408. u'name': u'debug request'
  409. , u'filters': []
  410. , u'interval': 0
  411. , u'limit': 0
  412. , u'actions': [{u'name': u'log'}]
  413. }
  414. with open('$FILTRON_RULES') as rules:
  415. j = json.load(rules)
  416. pos = None
  417. for i in range(len(j)):
  418. if j[i].get('name') == 'debug request':
  419. pos = i
  420. break
  421. if pos is not None:
  422. j[pos] = debug
  423. else:
  424. j.append(debug)
  425. with open('$FILTRON_RULES', 'w') as rules:
  426. json.dump(j, rules, indent=2, sort_keys=True)
  427. EOF
  428. systemctl restart "${SERVICE_NAME}.service"
  429. }
  430. disable_debug() {
  431. info_msg "try to disable debug mode ..."
  432. python <<EOF
  433. import sys, json
  434. with open('$FILTRON_RULES') as rules:
  435. j = json.load(rules)
  436. pos = None
  437. for i in range(len(j)):
  438. if j[i].get('name') == 'debug request':
  439. pos = i
  440. break
  441. if pos is not None:
  442. del j[pos]
  443. with open('$FILTRON_RULES', 'w') as rules:
  444. json.dump(j, rules, indent=2, sort_keys=True)
  445. EOF
  446. systemctl restart "${SERVICE_NAME}.service"
  447. }
  448. install_apache_site() {
  449. rst_title "Install Apache site $APACHE_FILTRON_SITE"
  450. rst_para "\
  451. This installs a reverse proxy (ProxyPass) into apache site (${APACHE_FILTRON_SITE})"
  452. ! apache_is_installed && info_msg "Apache is not installed."
  453. if ! ask_yn "Do you really want to continue?" Yn; then
  454. return
  455. else
  456. install_apache
  457. fi
  458. "${REPO_ROOT}/utils/searx.sh" install uwsgi
  459. apache_install_site --variant=filtron "${APACHE_FILTRON_SITE}"
  460. info_msg "testing public url .."
  461. if ! service_is_available "${PUBLIC_URL}"; then
  462. err_msg "Public service at ${PUBLIC_URL} is not available!"
  463. fi
  464. }
  465. remove_apache_site() {
  466. rst_title "Remove Apache site $APACHE_FILTRON_SITE"
  467. rst_para "\
  468. This removes apache site ${APACHE_FILTRON_SITE}."
  469. ! apache_is_installed && err_msg "Apache is not installed."
  470. if ! ask_yn "Do you really want to continue?" Yn; then
  471. return
  472. fi
  473. apache_remove_site "$APACHE_FILTRON_SITE"
  474. }
  475. install_nginx_site() {
  476. rst_title "Install nginx site $NGINX_FILTRON_SITE"
  477. rst_para "\
  478. This installs a reverse proxy (ProxyPass) into nginx site (${NGINX_FILTRON_SITE})"
  479. ! nginx_is_installed && info_msg "nginx is not installed."
  480. if ! ask_yn "Do you really want to continue?" Yn; then
  481. return
  482. else
  483. install_nginx
  484. fi
  485. "${REPO_ROOT}/utils/searx.sh" install uwsgi
  486. # shellcheck disable=SC2034
  487. SEARX_SRC=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARX_SRC)
  488. # shellcheck disable=SC2034
  489. SEARXNG_URL_PATH=$("${REPO_ROOT}/utils/searx.sh" --getenv SEARXNG_URL_PATH)
  490. nginx_install_app --variant=filtron "${NGINX_FILTRON_SITE}"
  491. info_msg "testing public url .."
  492. if ! service_is_available "${PUBLIC_URL}"; then
  493. err_msg "Public service at ${PUBLIC_URL} is not available!"
  494. fi
  495. }
  496. remove_nginx_site() {
  497. rst_title "Remove nginx site $NGINX_FILTRON_SITE"
  498. rst_para "\
  499. This removes nginx site ${NGINX_FILTRON_SITE}."
  500. ! nginx_is_installed && err_msg "nginx is not installed."
  501. if ! ask_yn "Do you really want to continue?" Yn; then
  502. return
  503. fi
  504. nginx_remove_site "$FILTRON_FILTRON_SITE"
  505. }
  506. rst-doc() {
  507. eval "echo \"$(< "${REPO_ROOT}/docs/build-templates/filtron.rst")\""
  508. echo -e "\n.. START install systemd unit"
  509. cat <<EOF
  510. .. tabs::
  511. .. group-tab:: systemd
  512. .. code:: bash
  513. EOF
  514. eval "echo \"$(< "${TEMPLATES}/${SERVICE_SYSTEMD_UNIT}")\"" | prefix_stdout " "
  515. echo -e "\n.. END install systemd unit"
  516. # for DIST_NAME in ubuntu-20.04 arch fedora centos; do
  517. # (
  518. # DIST_ID=${DIST_NAME%-*}
  519. # DIST_VERS=${DIST_NAME#*-}
  520. # [[ $DIST_VERS =~ $DIST_ID ]] && DIST_VERS=
  521. # # ...
  522. # )
  523. # done
  524. }
  525. # ----------------------------------------------------------------------------
  526. main "$@"
  527. # ----------------------------------------------------------------------------