lib.sh 48 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812
  1. #!/usr/bin/env bash
  2. # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
  3. # SPDX-License-Identifier: AGPL-3.0-or-later
  4. # shellcheck disable=SC2059,SC1117
  5. # ubuntu, debian, arch, fedora, centos ...
  6. DIST_ID=$(source /etc/os-release; echo "$ID");
  7. # shellcheck disable=SC2034
  8. DIST_VERS=$(source /etc/os-release; echo "$VERSION_ID");
  9. # shellcheck disable=SC2034
  10. DIST_VERSION_CODENAME=$(source /etc/os-release; echo "$VERSION_CODENAME");
  11. ADMIN_NAME="${ADMIN_NAME:-$(git config user.name)}"
  12. ADMIN_NAME="${ADMIN_NAME:-$USER}"
  13. ADMIN_EMAIL="${ADMIN_EMAIL:-$(git config user.email)}"
  14. ADMIN_EMAIL="${ADMIN_EMAIL:-$USER@$(hostname)}"
  15. if [[ -z "${REPO_ROOT}" ]]; then
  16. REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")
  17. while [ -h "${REPO_ROOT}" ] ; do
  18. REPO_ROOT=$(readlink "${REPO_ROOT}")
  19. done
  20. REPO_ROOT=$(cd "${REPO_ROOT}/.." && pwd -P )
  21. fi
  22. if [[ -z ${TEMPLATES} ]]; then
  23. TEMPLATES="${REPO_ROOT}/utils/templates"
  24. fi
  25. if [[ -z "$CACHE" ]]; then
  26. CACHE="${REPO_ROOT}/cache"
  27. fi
  28. if [[ -z ${DIFF_CMD} ]]; then
  29. DIFF_CMD="diff -u"
  30. if command -v colordiff >/dev/null; then
  31. DIFF_CMD="colordiff -u"
  32. fi
  33. fi
  34. DOT_CONFIG="${DOT_CONFIG:-${REPO_ROOT}/.config.sh}"
  35. source_dot_config() {
  36. if [[ ! -e "${DOT_CONFIG}" ]]; then
  37. err_msg "configuration does not exists at: ${DOT_CONFIG}"
  38. return 42
  39. fi
  40. # shellcheck disable=SC1090
  41. source "${DOT_CONFIG}"
  42. }
  43. sudo_or_exit() {
  44. # usage: sudo_or_exit
  45. if [ ! "$(id -u)" -eq 0 ]; then
  46. err_msg "this command requires root (sudo) privilege!" >&2
  47. exit 42
  48. fi
  49. }
  50. required_commands() {
  51. # usage: required_commands [cmd1 ...]
  52. local exit_val=0
  53. while [ -n "$1" ]; do
  54. if ! command -v "$1" &>/dev/null; then
  55. err_msg "missing command $1"
  56. exit_val=42
  57. fi
  58. shift
  59. done
  60. return $exit_val
  61. }
  62. # colors
  63. # ------
  64. # shellcheck disable=SC2034
  65. set_terminal_colors() {
  66. # https://en.wikipedia.org/wiki/ANSI_escape_code
  67. # CSI (Control Sequence Introducer) sequences
  68. _show_cursor='\e[?25h'
  69. _hide_cursor='\e[?25l'
  70. # SGR (Select Graphic Rendition) parameters
  71. _creset='\e[0m' # reset all attributes
  72. # original specification only had 8 colors
  73. _colors=8
  74. _Black='\e[0;30m'
  75. _White='\e[1;37m'
  76. _Red='\e[0;31m'
  77. _Green='\e[0;32m'
  78. _Yellow='\e[0;33m'
  79. _Blue='\e[0;94m'
  80. _Violet='\e[0;35m'
  81. _Cyan='\e[0;36m'
  82. _BBlack='\e[1;30m'
  83. _BWhite='\e[1;37m'
  84. _BRed='\e[1;31m'
  85. _BGreen='\e[1;32m'
  86. _BYellow='\e[1;33m'
  87. _BBlue='\e[1;94m'
  88. _BPurple='\e[1;35m'
  89. _BCyan='\e[1;36m'
  90. }
  91. if [ ! -p /dev/stdout ] && [ ! "$TERM" = 'dumb' ] && [ ! "$TERM" = 'unknown' ]; then
  92. set_terminal_colors
  93. fi
  94. # reST
  95. # ----
  96. if command -v fmt >/dev/null; then
  97. export FMT="fmt -u"
  98. else
  99. export FMT="cat"
  100. fi
  101. rst_title() {
  102. # usage: rst_title <header-text> [part|chapter|section]
  103. case ${2-chapter} in
  104. part) printf "\n${_BGreen}${1//?/=}${_creset}\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
  105. chapter) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/=}${_creset}\n";;
  106. section) printf "\n${_BCyan}${1}${_creset}\n${_BGreen}${1//?/-}${_creset}\n";;
  107. *)
  108. err_msg "invalid argument '${2}' in line $(caller)"
  109. return 42
  110. ;;
  111. esac
  112. }
  113. rst_para() {
  114. # usage: RST_INDENT=1 rst_para "lorem ipsum ..."
  115. local prefix=''
  116. if [[ -n $RST_INDENT ]] && [[ $RST_INDENT -gt 0 ]]; then
  117. prefix="$(for i in $(seq 1 "$RST_INDENT"); do printf " "; done)"
  118. echo -en "\n$*\n" | $FMT | prefix_stdout "$prefix"
  119. else
  120. echo -en "\n$*\n" | $FMT
  121. fi
  122. }
  123. die() {
  124. echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[1]}: line ${BASH_LINENO[0]}: ${2-died ${1-1}}" >&2;
  125. exit "${1-1}"
  126. }
  127. die_caller() {
  128. echo -e "${_BRed}ERROR:${_creset} ${BASH_SOURCE[2]}: line ${BASH_LINENO[1]}: ${FUNCNAME[1]}(): ${2-died ${1-1}}" >&2;
  129. exit "${1-1}"
  130. }
  131. err_msg() { echo -e "${_BRed}ERROR:${_creset} $*" >&2; }
  132. warn_msg() { echo -e "${_BBlue}WARN:${_creset} $*" >&2; }
  133. info_msg() { echo -e "${_BYellow}INFO:${_creset} $*" >&2; }
  134. build_msg() {
  135. local tag="$1 "
  136. shift
  137. echo -e "${_Blue}${tag:0:10}${_creset}$*"
  138. }
  139. dump_return() {
  140. # Use this as last command in your function to prompt an ERROR message if
  141. # the exit code is not zero.
  142. local err=$1
  143. [ "$err" -ne "0" ] && err_msg "${FUNCNAME[1]} exit with error ($err)"
  144. return "$err"
  145. }
  146. clean_stdin() {
  147. if [[ $(uname -s) != 'Darwin' ]]; then
  148. while read -r -n1 -t 0.1; do : ; done
  149. fi
  150. }
  151. wait_key(){
  152. # usage: wait_key [<timeout in sec>]
  153. clean_stdin
  154. local _t=$1
  155. local msg="${MSG}"
  156. [[ -z "$msg" ]] && msg="${_Green}** press any [${_BCyan}KEY${_Green}] to continue **${_creset}"
  157. [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
  158. [[ -n $_t ]] && _t="-t $_t"
  159. printf "$msg"
  160. # shellcheck disable=SC2229
  161. # shellcheck disable=SC2086
  162. read -r -s -n1 $_t || true
  163. echo
  164. clean_stdin
  165. }
  166. ask_yn() {
  167. # usage: ask_yn <prompt-text> [Ny|Yn] [<timeout in sec>]
  168. local EXIT_YES=0 # exit status 0 --> successful
  169. local EXIT_NO=1 # exit status 1 --> error code
  170. local _t=$3
  171. [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
  172. [[ -n $_t ]] && _t="-t $_t"
  173. case "${FORCE_SELECTION:-${2}}" in
  174. Y) return ${EXIT_YES} ;;
  175. N) return ${EXIT_NO} ;;
  176. Yn)
  177. local exit_val=${EXIT_YES}
  178. local choice="[${_BGreen}YES${_creset}/no]"
  179. local default="Yes"
  180. ;;
  181. *)
  182. local exit_val=${EXIT_NO}
  183. local choice="[${_BGreen}NO${_creset}/yes]"
  184. local default="No"
  185. ;;
  186. esac
  187. echo
  188. while true; do
  189. clean_stdin
  190. printf "$1 ${choice} "
  191. # shellcheck disable=SC2086,SC2229
  192. read -r -n1 $_t
  193. if [[ -z $REPLY ]]; then
  194. printf "$default\n"; break
  195. elif [[ $REPLY =~ ^[Yy]$ ]]; then
  196. exit_val=${EXIT_YES}
  197. printf "\n"
  198. break
  199. elif [[ $REPLY =~ ^[Nn]$ ]]; then
  200. exit_val=${EXIT_NO}
  201. printf "\n"
  202. break
  203. fi
  204. _t=""
  205. err_msg "invalid choice"
  206. done
  207. clean_stdin
  208. return $exit_val
  209. }
  210. tee_stderr () {
  211. # usage::
  212. # tee_stderr 1 <<EOF | python -i
  213. # print("hello")
  214. # EOF
  215. # ...
  216. # >>> print("hello")
  217. # hello
  218. local _t="0";
  219. if [[ -n $1 ]] ; then _t="$1"; fi
  220. (while read -r line; do
  221. # shellcheck disable=SC2086,SC2229
  222. sleep $_t
  223. echo -e "$line" >&2
  224. echo "$line"
  225. done)
  226. }
  227. prefix_stdout () {
  228. # usage: <cmd> | prefix_stdout [prefix]
  229. local prefix="${_BYellow}-->|${_creset}"
  230. if [[ -n $1 ]] ; then prefix="$1"; fi
  231. # shellcheck disable=SC2162
  232. (while IFS= read line; do
  233. echo -e "${prefix}$line"
  234. done)
  235. # some piped commands hide the cursor, show cursory when the stream ends
  236. echo -en "$_show_cursor"
  237. }
  238. append_line() {
  239. # usage: append_line <line> <file>
  240. #
  241. # Append line if not exists, create file if not exists. E.g::
  242. #
  243. # append_line 'source ~/.foo' ~/bashrc
  244. local LINE=$1
  245. local FILE=$2
  246. grep -qFs -- "$LINE" "$FILE" || echo "$LINE" >> "$FILE"
  247. }
  248. cache_download() {
  249. # usage: cache_download <url> <local-filename>
  250. local exit_value=0
  251. if [[ -n ${SUDO_USER} ]]; then
  252. sudo -u "${SUDO_USER}" mkdir -p "${CACHE}"
  253. else
  254. mkdir -p "${CACHE}"
  255. fi
  256. if [[ -f "${CACHE}/$2" ]] ; then
  257. info_msg "already cached: $1"
  258. info_msg " --> ${CACHE}/$2"
  259. fi
  260. if [[ ! -f "${CACHE}/$2" ]]; then
  261. info_msg "caching: $1"
  262. info_msg " --> ${CACHE}/$2"
  263. if [[ -n ${SUDO_USER} ]]; then
  264. sudo -u "${SUDO_USER}" wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
  265. else
  266. wget --progress=bar -O "${CACHE}/$2" "$1" ; exit_value=$?
  267. fi
  268. if [[ ! $exit_value = 0 ]]; then
  269. err_msg "failed to download: $1"
  270. fi
  271. fi
  272. }
  273. backup_file() {
  274. # usage: backup_file /path/to/file.foo
  275. local stamp
  276. stamp=$(date +"_%Y%m%d_%H%M%S")
  277. info_msg "create backup: ${1}${stamp}"
  278. cp -a "${1}" "${1}${stamp}"
  279. }
  280. choose_one() {
  281. # usage:
  282. #
  283. # DEFAULT_SELECT= 2 \
  284. # choose_one <name> "your selection?" "Coffee" "Coffee with milk"
  285. local default=${DEFAULT_SELECT-1}
  286. local REPLY
  287. local env_name=$1 && shift
  288. local choice=$1;
  289. local max="${#@}"
  290. local _t
  291. [[ -n $FORCE_TIMEOUT ]] && _t=$FORCE_TIMEOUT
  292. [[ -n $_t ]] && _t="-t $_t"
  293. list=("$@")
  294. echo -e "${_BGreen}Menu::${_creset}"
  295. for ((i=1; i<= $((max -1)); i++)); do
  296. if [[ "$i" == "$default" ]]; then
  297. echo -e " ${_BGreen}$i.${_creset}) ${list[$i]} [default]"
  298. else
  299. echo -e " $i.) ${list[$i]}"
  300. fi
  301. done
  302. while true; do
  303. clean_stdin
  304. printf "$1 [${_BGreen}$default${_creset}] "
  305. if (( 10 > max )); then
  306. # shellcheck disable=SC2086,SC2229
  307. read -r -n1 $_t
  308. else
  309. # shellcheck disable=SC2086,SC2229
  310. read -r $_t
  311. fi
  312. # selection fits
  313. [[ $REPLY =~ ^-?[0-9]+$ ]] && (( REPLY > 0 )) && (( REPLY < max )) && break
  314. # take default
  315. [[ -z $REPLY ]] && REPLY=$default && break
  316. _t=""
  317. err_msg "invalid choice"
  318. done
  319. eval "$env_name"='${list[${REPLY}]}'
  320. echo
  321. clean_stdin
  322. }
  323. install_template() {
  324. # usage:
  325. #
  326. # install_template [--no-eval] [--variant=<name>] \
  327. # {file} [{owner} [{group} [{chmod}]]]
  328. #
  329. # E.g. the origin of variant 'raw' of /etc/updatedb.conf is::
  330. #
  331. # ${TEMPLATES}/etc/updatedb.conf:raw
  332. #
  333. # To install variant 'raw' of /etc/updatedb.conf without evaluated
  334. # replacements you can use::
  335. #
  336. # install_template --variant=raw --no-eval \
  337. # /etc/updatedb.conf root root 644
  338. local _reply=""
  339. local do_eval=1
  340. local variant=""
  341. local pos_args=("$0")
  342. for i in "$@"; do
  343. case $i in
  344. --no-eval) do_eval=0; shift ;;
  345. --variant=*) variant=":${i#*=}"; shift ;;
  346. *) pos_args+=("$i") ;;
  347. esac
  348. done
  349. local dst="${pos_args[1]}"
  350. local template_origin="${TEMPLATES}${dst}${variant}"
  351. local template_file="${TEMPLATES}${dst}"
  352. local owner="${pos_args[2]-$(id -un)}"
  353. local group="${pos_args[3]-$(id -gn)}"
  354. local chmod="${pos_args[4]-644}"
  355. info_msg "install (eval=$do_eval): ${dst}"
  356. [[ -n $variant ]] && info_msg "variant --> ${variant}"
  357. if [[ ! -f "${template_origin}" ]] ; then
  358. err_msg "${template_origin} does not exists"
  359. err_msg "... can't install $dst"
  360. wait_key 30
  361. return 42
  362. fi
  363. if [[ "$do_eval" == "1" ]]; then
  364. template_file="${CACHE}${dst}${variant}"
  365. info_msg "BUILD ${template_file}"
  366. info_msg "BUILD using template ${template_origin}"
  367. if [[ -n ${SUDO_USER} ]]; then
  368. sudo -u "${SUDO_USER}" mkdir -p "$(dirname "${template_file}")"
  369. else
  370. mkdir -p "$(dirname "${template_file}")"
  371. fi
  372. # shellcheck disable=SC2086
  373. eval "echo \"$(cat ${template_origin})\"" > "${template_file}"
  374. if [[ -n ${SUDO_USER} ]]; then
  375. chown "${SUDO_USER}:${SUDO_USER}" "${template_file}"
  376. fi
  377. else
  378. template_file=$template_origin
  379. fi
  380. mkdir -p "$(dirname "${dst}")"
  381. if [[ ! -f "${dst}" ]]; then
  382. info_msg "install: ${template_file}"
  383. sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
  384. "${template_file}" "${dst}" | prefix_stdout
  385. return $?
  386. fi
  387. if [[ -f "${dst}" ]] && cmp --silent "${template_file}" "${dst}" ; then
  388. info_msg "file ${dst} already installed"
  389. return 0
  390. fi
  391. info_msg "different file ${dst} already exists on this host"
  392. while true; do
  393. choose_one _reply "choose next step with file $dst" \
  394. "replace file" \
  395. "leave file unchanged" \
  396. "interactive shell" \
  397. "diff files"
  398. case $_reply in
  399. "replace file")
  400. info_msg "install: ${template_file}"
  401. sudo -H install -v -o "${owner}" -g "${group}" -m "${chmod}" \
  402. "${template_file}" "${dst}" | prefix_stdout
  403. break
  404. ;;
  405. "leave file unchanged")
  406. break
  407. ;;
  408. "interactive shell")
  409. echo -e "// edit ${_Red}${dst}${_creset} to your needs"
  410. echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
  411. sudo -H -u "${owner}" -i
  412. $DIFF_CMD "${dst}" "${template_file}"
  413. echo
  414. echo -e "// ${_BBlack}did you edit file ...${_creset}"
  415. echo -en "// ${_Red}${dst}${_creset}"
  416. if ask_yn "//${_BBlack}... to your needs?${_creset}"; then
  417. break
  418. fi
  419. ;;
  420. "diff files")
  421. $DIFF_CMD "${dst}" "${template_file}" | prefix_stdout
  422. esac
  423. done
  424. }
  425. service_is_available() {
  426. # usage: service_is_available <URL>
  427. [[ -z $1 ]] && die_caller 42 "missing argument <URL>"
  428. local URL="$1"
  429. http_code=$(curl -H 'Cache-Control: no-cache' \
  430. --silent -o /dev/null --head --write-out '%{http_code}' --insecure \
  431. "${URL}")
  432. exit_val=$?
  433. if [[ $exit_val = 0 ]]; then
  434. info_msg "got $http_code from ${URL}"
  435. fi
  436. case "$http_code" in
  437. 404|410|423) exit_val=$http_code;;
  438. esac
  439. return "$exit_val"
  440. }
  441. # python
  442. # ------
  443. PY="${PY:=3}"
  444. PYTHON="${PYTHON:=python$PY}"
  445. PY_ENV="${PY_ENV:=local/py${PY}}"
  446. PY_ENV_BIN="${PY_ENV}/bin"
  447. PY_ENV_REQ="${PY_ENV_REQ:=${REPO_ROOT}/requirements*.txt}"
  448. # List of python packages (folders) or modules (files) installed by command:
  449. # pyenv.install
  450. PYOBJECTS="${PYOBJECTS:=.}"
  451. # folder where the python distribution takes place
  452. PYDIST="${PYDIST:=dist}"
  453. # folder where the intermediate build files take place
  454. PYBUILD="${PYBUILD:=build/py${PY}}"
  455. # https://www.python.org/dev/peps/pep-0508/#extras
  456. #PY_SETUP_EXTRAS='[develop,test]'
  457. PY_SETUP_EXTRAS="${PY_SETUP_EXTRAS:=[develop,test]}"
  458. PIP_BOILERPLATE=( pip wheel setuptools )
  459. # shellcheck disable=SC2120
  460. pyenv() {
  461. # usage: pyenv [vtenv_opts ...]
  462. #
  463. # vtenv_opts: see 'pip install --help'
  464. #
  465. # Builds virtualenv with 'requirements*.txt' (PY_ENV_REQ) installed. The
  466. # virtualenv will be reused by validating sha256sum of the requirement
  467. # files.
  468. required_commands \
  469. sha256sum "${PYTHON}" \
  470. || exit
  471. local pip_req=()
  472. if ! pyenv.OK > /dev/null; then
  473. rm -f "${PY_ENV}/${PY_ENV_REQ}.sha256"
  474. pyenv.drop > /dev/null
  475. build_msg PYENV "[virtualenv] installing ${PY_ENV_REQ} into ${PY_ENV}"
  476. "${PYTHON}" -m venv "$@" "${PY_ENV}"
  477. "${PY_ENV_BIN}/python" -m pip install -U "${PIP_BOILERPLATE[@]}"
  478. for i in ${PY_ENV_REQ}; do
  479. pip_req=( "${pip_req[@]}" "-r" "$i" )
  480. done
  481. (
  482. [ "$VERBOSE" = "1" ] && set -x
  483. # shellcheck disable=SC2086
  484. "${PY_ENV_BIN}/python" -m pip install "${pip_req[@]}" \
  485. && sha256sum ${PY_ENV_REQ} > "${PY_ENV}/requirements.sha256"
  486. )
  487. fi
  488. pyenv.OK
  489. }
  490. _pyenv_OK=''
  491. pyenv.OK() {
  492. # probes if pyenv exists and runs the script from pyenv.check
  493. [ "$_pyenv_OK" == "OK" ] && return 0
  494. if [ ! -f "${PY_ENV_BIN}/python" ]; then
  495. build_msg PYENV "[virtualenv] missing ${PY_ENV_BIN}/python"
  496. return 1
  497. fi
  498. if [ ! -f "${PY_ENV}/requirements.sha256" ] \
  499. || ! sha256sum -c "${PY_ENV}/requirements.sha256" > /dev/null 2>&1; then
  500. build_msg PYENV "[virtualenv] requirements.sha256 failed"
  501. sed 's/^/ [virtualenv] - /' <"${PY_ENV}/requirements.sha256"
  502. return 1
  503. fi
  504. if [ "$VERBOSE" = "1" ]; then
  505. pyenv.check \
  506. | "${PY_ENV_BIN}/python" 2>&1 \
  507. | prefix_stdout "${_Blue}PYENV ${_creset}[check] "
  508. else
  509. pyenv.check | "${PY_ENV_BIN}/python" 1>/dev/null
  510. fi
  511. local err=${PIPESTATUS[1]}
  512. if [ "$err" -ne "0" ]; then
  513. build_msg PYENV "[check] python test failed"
  514. return "$err"
  515. fi
  516. [ "$VERBOSE" = "1" ] && build_msg PYENV "OK"
  517. _pyenv_OK="OK"
  518. return 0
  519. }
  520. pyenv.drop() {
  521. build_msg PYENV "[virtualenv] drop ${PY_ENV}"
  522. rm -rf "${PY_ENV}"
  523. _pyenv_OK=''
  524. }
  525. pyenv.check() {
  526. # Prompts a python script with additional checks. Used by pyenv.OK to check
  527. # if virtualenv is ready to install python objects. This function should be
  528. # overwritten by the application script.
  529. local imp=""
  530. for i in "${PIP_BOILERPLATE[@]}"; do
  531. imp="$imp, $i"
  532. done
  533. cat <<EOF
  534. import ${imp#,*}
  535. EOF
  536. }
  537. pyenv.install() {
  538. if ! pyenv.OK; then
  539. py.clean > /dev/null
  540. fi
  541. if ! pyenv.install.OK > /dev/null; then
  542. build_msg PYENV "[install] ${PYOBJECTS}"
  543. if ! pyenv.OK >/dev/null; then
  544. pyenv
  545. fi
  546. for i in ${PYOBJECTS}; do
  547. build_msg PYENV "[install] pip install --use-pep517 --no-build-isolation -e '$i${PY_SETUP_EXTRAS}'"
  548. "${PY_ENV_BIN}/python" -m pip install --use-pep517 --no-build-isolation -e "$i${PY_SETUP_EXTRAS}"
  549. done
  550. fi
  551. pyenv.install.OK
  552. }
  553. _pyenv_install_OK=''
  554. pyenv.install.OK() {
  555. [ "$_pyenv_install_OK" == "OK" ] && return 0
  556. local imp=""
  557. local err=""
  558. if [ "." = "${PYOBJECTS}" ]; then
  559. imp="import $(basename "$(pwd)")"
  560. else
  561. # shellcheck disable=SC2086
  562. for i in ${PYOBJECTS}; do imp="$imp, $i"; done
  563. imp="import ${imp#,*} "
  564. fi
  565. (
  566. [ "$VERBOSE" = "1" ] && set -x
  567. "${PY_ENV_BIN}/python" -c "import sys; sys.path.pop(0); $imp;" 2>/dev/null
  568. )
  569. err=$?
  570. if [ "$err" -ne "0" ]; then
  571. build_msg PYENV "[install] python installation test failed"
  572. return "$err"
  573. fi
  574. build_msg PYENV "[install] OK"
  575. _pyenv_install_OK="OK"
  576. return 0
  577. }
  578. pyenv.uninstall() {
  579. build_msg PYENV "[uninstall] ${PYOBJECTS}"
  580. if [ "." = "${PYOBJECTS}" ]; then
  581. pyenv.cmd python setup.py develop --uninstall 2>&1 \
  582. | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
  583. else
  584. # shellcheck disable=SC2086
  585. pyenv.cmd python -m pip uninstall --yes ${PYOBJECTS} 2>&1 \
  586. | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
  587. fi
  588. }
  589. pyenv.cmd() {
  590. pyenv.install
  591. ( set -e
  592. # shellcheck source=/dev/null
  593. source "${PY_ENV_BIN}/activate"
  594. [ "$VERBOSE" = "1" ] && set -x
  595. "$@"
  596. )
  597. }
  598. pyenv.activate() {
  599. pyenv.install
  600. # shellcheck source=/dev/null
  601. source "${PY_ENV_BIN}/activate"
  602. }
  603. # Sphinx doc
  604. # ----------
  605. GH_PAGES="build/gh-pages"
  606. DOCS_DIST="${DOCS_DIST:=dist/docs}"
  607. DOCS_BUILD="${DOCS_BUILD:=build/docs}"
  608. docs.html() {
  609. build_msg SPHINX "HTML ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"
  610. pyenv.install
  611. docs.prebuild
  612. # shellcheck disable=SC2086
  613. PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-build \
  614. ${SPHINX_VERBOSE} ${SPHINXOPTS} \
  615. -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
  616. dump_return $?
  617. }
  618. docs.live() {
  619. build_msg SPHINX "autobuild ./docs --> file://$(readlink -e "$(pwd)/$DOCS_DIST")"
  620. pyenv.install
  621. docs.prebuild
  622. # shellcheck disable=SC2086
  623. PATH="${PY_ENV_BIN}:${PATH}" pyenv.cmd sphinx-autobuild \
  624. ${SPHINX_VERBOSE} ${SPHINXOPTS} --open-browser --host 0.0.0.0 \
  625. -b html -c ./docs -d "${DOCS_BUILD}/.doctrees" ./docs "${DOCS_DIST}"
  626. dump_return $?
  627. }
  628. docs.clean() {
  629. build_msg CLEAN "docs -- ${DOCS_BUILD} ${DOCS_DIST}"
  630. # shellcheck disable=SC2115
  631. rm -rf "${GH_PAGES}" "${DOCS_BUILD}" "${DOCS_DIST}"
  632. dump_return $?
  633. }
  634. docs.prebuild() {
  635. # Dummy function to run some actions before sphinx-doc build gets started.
  636. # This function needs to be overwritten by the application script.
  637. true
  638. dump_return $?
  639. }
  640. # shellcheck disable=SC2155
  641. docs.gh-pages() {
  642. # The commit history in the gh-pages branch makes no sense, the history only
  643. # inflates the repository unnecessarily. Therefore a *new orphan* branch
  644. # is created each time we deploy on the gh-pages branch.
  645. docs.clean
  646. docs.prebuild
  647. docs.html
  648. [ "$VERBOSE" = "1" ] && set -x
  649. local head="$(git rev-parse HEAD)"
  650. local branch="$(git name-rev --name-only HEAD)"
  651. local remote="$(git config branch."${branch}".remote)"
  652. local remote_url="$(git config remote."${remote}".url)"
  653. build_msg GH-PAGES "prepare folder: ${GH_PAGES}"
  654. build_msg GH-PAGES "remote of the gh-pages branch: ${remote} / ${remote_url}"
  655. build_msg GH-PAGES "current branch: ${branch}"
  656. # prepare the *orphan* gh-pages working tree
  657. (
  658. git worktree remove -f "${GH_PAGES}"
  659. git branch -D gh-pages
  660. ) &> /dev/null || true
  661. git worktree add --no-checkout "${GH_PAGES}" "${remote}/master"
  662. pushd "${GH_PAGES}" &> /dev/null
  663. git checkout --orphan gh-pages
  664. git rm -rfq .
  665. popd &> /dev/null
  666. cp -r "${DOCS_DIST}"/* "${GH_PAGES}"/
  667. touch "${GH_PAGES}/.nojekyll"
  668. cat > "${GH_PAGES}/404.html" <<EOF
  669. <html><head><META http-equiv='refresh' content='0;URL=index.html'></head></html>
  670. EOF
  671. pushd "${GH_PAGES}" &> /dev/null
  672. git add --all .
  673. git commit -q -m "gh-pages build from: ${branch}@${head} (${remote_url})"
  674. git push -f "${remote}" gh-pages
  675. popd &> /dev/null
  676. set +x
  677. build_msg GH-PAGES "deployed"
  678. }
  679. # system accounts
  680. # ---------------
  681. service_account_is_available() {
  682. # usage: service_account_is_available "$SERVICE_USER" && echo "OK"
  683. sudo -i -u "$1" echo \$HOME &>/dev/null
  684. }
  685. drop_service_account() {
  686. # usage: drop_service_account "${SERVICE_USER}"
  687. rst_title "Drop ${1} HOME" section
  688. if ask_yn "Do you really want to drop ${1} home folder?"; then
  689. userdel -r -f "${1}" 2>&1 | prefix_stdout
  690. else
  691. rst_para "Leave HOME folder $(du -sh "${1}") unchanged."
  692. fi
  693. }
  694. interactive_shell(){
  695. # usage: interactive_shell "${SERVICE_USER}"
  696. echo -e "// exit with [${_BCyan}CTRL-D${_creset}]"
  697. sudo -H -u "${1}" -i
  698. }
  699. # systemd
  700. # -------
  701. SYSTEMD_UNITS="${SYSTEMD_UNITS:-/lib/systemd/system}"
  702. systemd_install_service() {
  703. # usage: systemd_install_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
  704. rst_title "Install System-D Unit ${1}" section
  705. echo
  706. install_template "${2}" root root 644
  707. wait_key
  708. systemd_activate_service "${1}"
  709. }
  710. systemd_remove_service() {
  711. # usage: systemd_remove_service "${SERVICE_NAME}" "${SERVICE_SYSTEMD_UNIT}"
  712. if ! ask_yn "Do you really want to deinstall systemd unit ${1}?"; then
  713. return 42
  714. fi
  715. systemd_deactivate_service "${1}"
  716. rm "${2}" 2>&1 | prefix_stdout
  717. }
  718. systemd_activate_service() {
  719. # usage: systemd_activate_service "${SERVICE_NAME}"
  720. rst_title "Activate ${1} (service)" section
  721. echo
  722. tee_stderr <<EOF | bash 2>&1
  723. systemctl enable ${1}.service
  724. systemctl restart ${1}.service
  725. EOF
  726. tee_stderr <<EOF | bash 2>&1
  727. systemctl status --no-pager ${1}.service
  728. EOF
  729. }
  730. systemd_deactivate_service() {
  731. # usage: systemd_deactivate_service "${SERVICE_NAME}"
  732. rst_title "De-Activate ${1} (service)" section
  733. echo
  734. tee_stderr <<EOF | bash 2>&1 | prefix_stdout
  735. systemctl stop ${1}.service
  736. systemctl disable ${1}.service
  737. EOF
  738. }
  739. systemd_restart_service() {
  740. # usage: systemd_restart_service "${SERVICE_NAME}"
  741. rst_title "Restart ${1} (service)" section
  742. echo
  743. tee_stderr <<EOF | bash 2>&1
  744. systemctl restart ${1}.service
  745. EOF
  746. tee_stderr <<EOF | bash 2>&1
  747. systemctl status --no-pager ${1}.service
  748. EOF
  749. }
  750. # nginx
  751. # -----
  752. nginx_distro_setup() {
  753. # shellcheck disable=SC2034
  754. NGINX_DEFAULT_SERVER=/etc/nginx/nginx.conf
  755. # Including *location* directives from a dedicated config-folder into the
  756. # server directive is, what fedora and centos (already) does.
  757. NGINX_APPS_ENABLED="/etc/nginx/default.d"
  758. # We add a apps-available folder and linking configurations into the
  759. # NGINX_APPS_ENABLED folder. See also nginx_include_apps_enabled().
  760. NGINX_APPS_AVAILABLE="/etc/nginx/default.apps-available"
  761. case $DIST_ID-$DIST_VERS in
  762. ubuntu-*|debian-*)
  763. NGINX_PACKAGES="nginx"
  764. NGINX_DEFAULT_SERVER=/etc/nginx/sites-available/default
  765. ;;
  766. arch-*)
  767. NGINX_PACKAGES="nginx-mainline"
  768. ;;
  769. fedora-*|centos-7)
  770. NGINX_PACKAGES="nginx"
  771. ;;
  772. *)
  773. err_msg "$DIST_ID-$DIST_VERS: nginx not yet implemented"
  774. ;;
  775. esac
  776. }
  777. install_nginx(){
  778. info_msg "installing nginx ..."
  779. pkg_install "${NGINX_PACKAGES}"
  780. case $DIST_ID-$DIST_VERS in
  781. arch-*|fedora-*|centos-7)
  782. systemctl enable nginx
  783. systemctl start nginx
  784. ;;
  785. esac
  786. }
  787. nginx_is_installed() {
  788. command -v nginx &>/dev/null
  789. }
  790. nginx_reload() {
  791. info_msg "reload nginx .."
  792. echo
  793. if ! nginx -t; then
  794. err_msg "testing nginx configuration failed"
  795. return 42
  796. fi
  797. systemctl restart nginx
  798. }
  799. nginx_install_app() {
  800. # usage: nginx_install_app [<template option> ...] <myapp.conf>
  801. #
  802. # <template option>: see install_template
  803. local template_opts=()
  804. local pos_args=("$0")
  805. for i in "$@"; do
  806. case $i in
  807. -*) template_opts+=("$i");;
  808. *) pos_args+=("$i");;
  809. esac
  810. done
  811. nginx_include_apps_enabled "${NGINX_DEFAULT_SERVER}"
  812. install_template "${template_opts[@]}" \
  813. "${NGINX_APPS_AVAILABLE}/${pos_args[1]}" \
  814. root root 644
  815. nginx_enable_app "${pos_args[1]}"
  816. info_msg "installed nginx app: ${pos_args[1]}"
  817. }
  818. nginx_include_apps_enabled() {
  819. # Add the *NGINX_APPS_ENABLED* infrastructure to a nginx server block. Such
  820. # infrastructure is already known from fedora and centos, including apps (location
  821. # directives) from the /etc/nginx/default.d folder into the *default* nginx
  822. # server.
  823. # usage: nginx_include_apps_enabled <config-file>
  824. #
  825. # config-file: Config file with server directive in.
  826. [[ -z $1 ]] && die_caller 42 "missing argument <config-file>"
  827. local server_conf="$1"
  828. # include /etc/nginx/default.d/*.conf;
  829. local include_directive="include ${NGINX_APPS_ENABLED}/*.conf;"
  830. local include_directive_re="^\s*include ${NGINX_APPS_ENABLED}/\*\.conf;"
  831. info_msg "checking existence: '${include_directive}' in file ${server_conf}"
  832. if grep "${include_directive_re}" "${server_conf}"; then
  833. info_msg "OK, already exists."
  834. return
  835. fi
  836. info_msg "add missing directive: '${include_directive}'"
  837. cp "${server_conf}" "${server_conf}.bak"
  838. (
  839. local line
  840. local stage=0
  841. while IFS= read -r line
  842. do
  843. echo "$line"
  844. if [[ $stage = 0 ]]; then
  845. if [[ $line =~ ^[[:space:]]*server*[[:space:]]*\{ ]]; then
  846. stage=1
  847. fi
  848. fi
  849. if [[ $stage = 1 ]]; then
  850. echo " # Load configuration files for the default server block."
  851. echo " $include_directive"
  852. echo ""
  853. stage=2
  854. fi
  855. done < "${server_conf}.bak"
  856. ) > "${server_conf}"
  857. }
  858. nginx_remove_app() {
  859. # usage: nginx_remove_app <myapp.conf>
  860. info_msg "remove nginx app: $1"
  861. nginx_disable_app "$1"
  862. rm -f "${NGINX_APPS_AVAILABLE}/$1"
  863. }
  864. nginx_enable_app() {
  865. # usage: nginx_enable_app <myapp.conf>
  866. local CONF="$1"
  867. info_msg "enable nginx app: ${CONF}"
  868. mkdir -p "${NGINX_APPS_ENABLED}"
  869. rm -f "${NGINX_APPS_ENABLED}/${CONF}"
  870. ln -s "${NGINX_APPS_AVAILABLE}/${CONF}" "${NGINX_APPS_ENABLED}/${CONF}"
  871. nginx_reload
  872. }
  873. nginx_disable_app() {
  874. # usage: nginx_disable_app <myapp.conf>
  875. local CONF="$1"
  876. info_msg "disable nginx app: ${CONF}"
  877. rm -f "${NGINX_APPS_ENABLED}/${CONF}"
  878. nginx_reload
  879. }
  880. # Apache
  881. # ------
  882. apache_distro_setup() {
  883. # shellcheck disable=SC2034
  884. case $DIST_ID-$DIST_VERS in
  885. ubuntu-*|debian-*)
  886. # debian uses the /etc/apache2 path, while other distros use
  887. # the apache default at /etc/httpd
  888. APACHE_SITES_AVAILABLE="/etc/apache2/sites-available"
  889. APACHE_SITES_ENABLED="/etc/apache2/sites-enabled"
  890. APACHE_MODULES="/usr/lib/apache2/modules"
  891. APACHE_PACKAGES="apache2"
  892. ;;
  893. arch-*)
  894. APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
  895. APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
  896. APACHE_MODULES="modules"
  897. APACHE_PACKAGES="apache"
  898. ;;
  899. fedora-*|centos-7)
  900. APACHE_SITES_AVAILABLE="/etc/httpd/sites-available"
  901. APACHE_SITES_ENABLED="/etc/httpd/sites-enabled"
  902. APACHE_MODULES="modules"
  903. APACHE_PACKAGES="httpd mod_ssl"
  904. ;;
  905. *)
  906. err_msg "$DIST_ID-$DIST_VERS: apache not yet implemented"
  907. ;;
  908. esac
  909. }
  910. install_apache(){
  911. info_msg "installing apache ..."
  912. pkg_install "$APACHE_PACKAGES"
  913. case $DIST_ID-$DIST_VERS in
  914. arch-*|fedora-*|centos-7)
  915. if ! grep "IncludeOptional sites-enabled" "/etc/httpd/conf/httpd.conf"; then
  916. echo "IncludeOptional sites-enabled/*.conf" >> "/etc/httpd/conf/httpd.conf"
  917. fi
  918. systemctl enable httpd
  919. systemctl start httpd
  920. ;;
  921. esac
  922. }
  923. apache_is_installed() {
  924. case $DIST_ID-$DIST_VERS in
  925. ubuntu-*|debian-*) (command -v apachectl) &>/dev/null;;
  926. arch-*) (command -v httpd) &>/dev/null;;
  927. fedora-*|centos-7) (command -v httpd) &>/dev/null;;
  928. esac
  929. }
  930. apache_reload() {
  931. info_msg "reload apache .."
  932. echo
  933. case $DIST_ID-$DIST_VERS in
  934. ubuntu-*|debian-*)
  935. sudo -H apachectl configtest
  936. sudo -H systemctl force-reload apache2
  937. ;;
  938. arch-*|fedora-*|centos-7)
  939. sudo -H httpd -t
  940. sudo -H systemctl force-reload httpd
  941. ;;
  942. esac
  943. }
  944. apache_install_site() {
  945. # usage: apache_install_site [<template option> ...] <mysite.conf>
  946. #
  947. # <template option>: see install_template
  948. local template_opts=()
  949. local pos_args=("$0")
  950. for i in "$@"; do
  951. case $i in
  952. -*) template_opts+=("$i");;
  953. *) pos_args+=("$i");;
  954. esac
  955. done
  956. install_template "${template_opts[@]}" \
  957. "${APACHE_SITES_AVAILABLE}/${pos_args[1]}" \
  958. root root 644
  959. apache_enable_site "${pos_args[1]}"
  960. info_msg "installed apache site: ${pos_args[1]}"
  961. }
  962. apache_remove_site() {
  963. # usage: apache_remove_site <mysite.conf>
  964. info_msg "remove apache site: $1"
  965. apache_disable_site "$1"
  966. rm -f "${APACHE_SITES_AVAILABLE}/$1"
  967. }
  968. apache_enable_site() {
  969. # usage: apache_enable_site <mysite.conf>
  970. local CONF="$1"
  971. info_msg "enable apache site: ${CONF}"
  972. case $DIST_ID-$DIST_VERS in
  973. ubuntu-*|debian-*)
  974. sudo -H a2ensite -q "${CONF}"
  975. ;;
  976. arch-*)
  977. mkdir -p "${APACHE_SITES_ENABLED}"
  978. rm -f "${APACHE_SITES_ENABLED}/${CONF}"
  979. ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
  980. ;;
  981. fedora-*|centos-7)
  982. mkdir -p "${APACHE_SITES_ENABLED}"
  983. rm -f "${APACHE_SITES_ENABLED}/${CONF}"
  984. ln -s "${APACHE_SITES_AVAILABLE}/${CONF}" "${APACHE_SITES_ENABLED}/${CONF}"
  985. ;;
  986. esac
  987. apache_reload
  988. }
  989. apache_disable_site() {
  990. # usage: apache_disable_site <mysite.conf>
  991. local CONF="$1"
  992. info_msg "disable apache site: ${CONF}"
  993. case $DIST_ID-$DIST_VERS in
  994. ubuntu-*|debian-*)
  995. sudo -H a2dissite -q "${CONF}"
  996. ;;
  997. arch-*)
  998. rm -f "${APACHE_SITES_ENABLED}/${CONF}"
  999. ;;
  1000. fedora-*|centos-7)
  1001. rm -f "${APACHE_SITES_ENABLED}/${CONF}"
  1002. ;;
  1003. esac
  1004. apache_reload
  1005. }
  1006. # uWSGI
  1007. # -----
  1008. uWSGI_SETUP="${uWSGI_SETUP:=/etc/uwsgi}"
  1009. # How distros manage uWSGI apps is very different. From uWSGI POV read:
  1010. # - https://uwsgi-docs.readthedocs.io/en/latest/Management.html
  1011. uWSGI_distro_setup() {
  1012. case $DIST_ID-$DIST_VERS in
  1013. ubuntu-*|debian-*)
  1014. # init.d --> /usr/share/doc/uwsgi/README.Debian.gz
  1015. # For uWSGI debian uses the LSB init process, this might be changed
  1016. # one day, see https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=833067
  1017. uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
  1018. uWSGI_APPS_ENABLED="${uWSGI_SETUP}/apps-enabled"
  1019. uWSGI_PACKAGES="uwsgi"
  1020. ;;
  1021. arch-*)
  1022. # systemd --> /usr/lib/systemd/system/uwsgi@.service
  1023. # For uWSGI archlinux uses systemd template units, see
  1024. # - http://0pointer.de/blog/projects/instances.html
  1025. # - https://uwsgi-docs.readthedocs.io/en/latest/Systemd.html#one-service-per-app-in-systemd
  1026. uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-archlinux"
  1027. uWSGI_APPS_ENABLED="${uWSGI_SETUP}"
  1028. uWSGI_PACKAGES="uwsgi"
  1029. ;;
  1030. fedora-*|centos-7)
  1031. # systemd --> /usr/lib/systemd/system/uwsgi.service
  1032. # Fedora runs uWSGI in emperor-tyrant mode: in Tyrant mode the
  1033. # Emperor will run the vassal using the UID/GID of the vassal
  1034. # configuration file [1] (user and group of the app .ini file).
  1035. # There are some quirks about additional POSIX groups in uWSGI
  1036. # 2.0.x, read at least: https://github.com/unbit/uwsgi/issues/2099
  1037. uWSGI_APPS_AVAILABLE="${uWSGI_SETUP}/apps-available"
  1038. uWSGI_APPS_ENABLED="${uWSGI_SETUP}.d"
  1039. uWSGI_PACKAGES="uwsgi"
  1040. ;;
  1041. *)
  1042. err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
  1043. ;;
  1044. esac
  1045. }
  1046. install_uwsgi(){
  1047. info_msg "installing uwsgi ..."
  1048. pkg_install "$uWSGI_PACKAGES"
  1049. case $DIST_ID-$DIST_VERS in
  1050. fedora-*|centos-7)
  1051. # enable & start should be called once at uWSGI installation time
  1052. systemctl enable uwsgi
  1053. systemctl restart uwsgi
  1054. ;;
  1055. esac
  1056. }
  1057. uWSGI_restart() {
  1058. # usage: uWSGI_restart() <myapp.ini>
  1059. local CONF="$1"
  1060. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1061. info_msg "restart uWSGI service"
  1062. case $DIST_ID-$DIST_VERS in
  1063. ubuntu-*|debian-*)
  1064. # the 'service' method seems broken in that way, that it (re-)starts
  1065. # the whole uwsgi process.
  1066. service uwsgi restart "${CONF%.*}"
  1067. ;;
  1068. arch-*)
  1069. # restart systemd template instance
  1070. if uWSGI_app_available "${CONF}"; then
  1071. systemctl restart "uwsgi@${CONF%.*}"
  1072. else
  1073. info_msg "[uWSGI:systemd-template] ${CONF} not installed (no need to restart)"
  1074. fi
  1075. ;;
  1076. fedora-*|centos-7)
  1077. # in emperor mode, just touch the file to restart
  1078. if uWSGI_app_enabled "${CONF}"; then
  1079. touch "${uWSGI_APPS_ENABLED}/${CONF}"
  1080. # it seems, there is a polling time in between touch and restart
  1081. # of the service.
  1082. sleep 3
  1083. else
  1084. info_msg "[uWSGI:emperor] ${CONF} not installed (no need to restart)"
  1085. fi
  1086. ;;
  1087. *)
  1088. err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
  1089. return 42
  1090. ;;
  1091. esac
  1092. }
  1093. uWSGI_app_available() {
  1094. # usage: uWSGI_app_available <myapp.ini>
  1095. local CONF="$1"
  1096. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1097. [[ -f "${uWSGI_APPS_AVAILABLE}/${CONF}" ]]
  1098. }
  1099. uWSGI_install_app() {
  1100. # usage: uWSGI_install_app [<template option> ...] <myapp.ini> [{owner} [{group} [{chmod}]]]
  1101. #
  1102. # <template option>: see install_template
  1103. local pos_args=("$0")
  1104. for i in "$@"; do
  1105. case $i in
  1106. -*) template_opts+=("$i");;
  1107. *) pos_args+=("$i");;
  1108. esac
  1109. done
  1110. mkdir -p "${uWSGI_APPS_AVAILABLE}"
  1111. install_template "${template_opts[@]}" \
  1112. "${uWSGI_APPS_AVAILABLE}/${pos_args[1]}" \
  1113. "${pos_args[2]:-root}" "${pos_args[3]:-root}" "${pos_args[4]:-644}"
  1114. uWSGI_enable_app "${pos_args[1]}"
  1115. uWSGI_restart "${pos_args[1]}"
  1116. info_msg "uWSGI app: ${pos_args[1]} is installed"
  1117. }
  1118. uWSGI_remove_app() {
  1119. # usage: uWSGI_remove_app <myapp.ini>
  1120. local CONF="$1"
  1121. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1122. info_msg "remove uWSGI app: ${CONF}"
  1123. uWSGI_disable_app "${CONF}"
  1124. uWSGI_restart "${CONF}"
  1125. rm -f "${uWSGI_APPS_AVAILABLE}/${CONF}"
  1126. }
  1127. uWSGI_app_enabled() {
  1128. # usage: uWSGI_app_enabled <myapp.ini>
  1129. local exit_val=0
  1130. local CONF="$1"
  1131. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1132. case $DIST_ID-$DIST_VERS in
  1133. ubuntu-*|debian-*)
  1134. [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
  1135. exit_val=$?
  1136. ;;
  1137. arch-*)
  1138. systemctl -q is-enabled "uwsgi@${CONF%.*}"
  1139. exit_val=$?
  1140. ;;
  1141. fedora-*|centos-7)
  1142. [[ -f "${uWSGI_APPS_ENABLED}/${CONF}" ]]
  1143. exit_val=$?
  1144. ;;
  1145. *)
  1146. # FIXME
  1147. err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
  1148. exit_val=1
  1149. ;;
  1150. esac
  1151. return $exit_val
  1152. }
  1153. # shellcheck disable=SC2164
  1154. uWSGI_enable_app() {
  1155. # usage: uWSGI_enable_app <myapp.ini>
  1156. local CONF="$1"
  1157. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1158. case $DIST_ID-$DIST_VERS in
  1159. ubuntu-*|debian-*)
  1160. mkdir -p "${uWSGI_APPS_ENABLED}"
  1161. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1162. ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
  1163. info_msg "enabled uWSGI app: ${CONF} (restart required)"
  1164. ;;
  1165. arch-*)
  1166. mkdir -p "${uWSGI_APPS_ENABLED}"
  1167. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1168. ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
  1169. systemctl enable "uwsgi@${CONF%.*}"
  1170. info_msg "enabled uWSGI app: ${CONF} (restart required)"
  1171. ;;
  1172. fedora-*|centos-7)
  1173. mkdir -p "${uWSGI_APPS_ENABLED}"
  1174. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1175. ln -s "${uWSGI_APPS_AVAILABLE}/${CONF}" "${uWSGI_APPS_ENABLED}/${CONF}"
  1176. info_msg "enabled uWSGI app: ${CONF}"
  1177. ;;
  1178. *)
  1179. # FIXME
  1180. err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
  1181. ;;
  1182. esac
  1183. }
  1184. uWSGI_disable_app() {
  1185. # usage: uWSGI_disable_app <myapp.ini>
  1186. local CONF="$1"
  1187. [[ -z $CONF ]] && die_caller 42 "missing argument <myapp.ini>"
  1188. case $DIST_ID-$DIST_VERS in
  1189. ubuntu-*|debian-*)
  1190. service uwsgi stop "${CONF%.*}"
  1191. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1192. info_msg "disabled uWSGI app: ${CONF} (restart uWSGI required)"
  1193. ;;
  1194. arch-*)
  1195. systemctl stop "uwsgi@${CONF%.*}"
  1196. systemctl disable "uwsgi@${CONF%.*}"
  1197. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1198. ;;
  1199. fedora-*|centos-7)
  1200. # in emperor mode, just remove the app.ini file
  1201. rm -f "${uWSGI_APPS_ENABLED}/${CONF}"
  1202. ;;
  1203. *)
  1204. # FIXME
  1205. err_msg "$DIST_ID-$DIST_VERS: uWSGI not yet implemented"
  1206. ;;
  1207. esac
  1208. }
  1209. # distro's package manager
  1210. # ------------------------
  1211. _apt_pkg_info_is_updated=0
  1212. pkg_install() {
  1213. # usage: TITLE='install foobar' pkg_install foopkg barpkg
  1214. rst_title "${TITLE:-installation of packages}" section
  1215. echo -e "\npackage(s)::\n"
  1216. # shellcheck disable=SC2068
  1217. echo " " $@ | $FMT
  1218. if ! ask_yn "Should packages be installed?" Yn 30; then
  1219. return 42
  1220. fi
  1221. case $DIST_ID in
  1222. ubuntu|debian)
  1223. if [[ $_apt_pkg_info_is_updated == 0 ]]; then
  1224. export _apt_pkg_info_is_updated=1
  1225. apt update
  1226. fi
  1227. # shellcheck disable=SC2068
  1228. apt-get install -m -y $@
  1229. ;;
  1230. arch)
  1231. # shellcheck disable=SC2068
  1232. pacman --noprogressbar -Sy --noconfirm --needed $@
  1233. ;;
  1234. fedora)
  1235. # shellcheck disable=SC2068
  1236. dnf install -y $@
  1237. ;;
  1238. centos)
  1239. # shellcheck disable=SC2068
  1240. yum install -y $@
  1241. ;;
  1242. esac
  1243. }
  1244. pkg_remove() {
  1245. # usage: TITLE='remove foobar' pkg_remove foopkg barpkg
  1246. rst_title "${TITLE:-remove packages}" section
  1247. echo -e "\npackage(s)::\n"
  1248. # shellcheck disable=SC2068
  1249. echo " " $@ | $FMT
  1250. if ! ask_yn "Should packages be removed (purge)?" Yn 30; then
  1251. return 42
  1252. fi
  1253. case $DIST_ID in
  1254. ubuntu|debian)
  1255. # shellcheck disable=SC2068
  1256. apt-get purge --autoremove --ignore-missing -y $@
  1257. ;;
  1258. arch)
  1259. # shellcheck disable=SC2068
  1260. pacman --noprogressbar -R --noconfirm $@
  1261. ;;
  1262. fedora)
  1263. # shellcheck disable=SC2068
  1264. dnf remove -y $@
  1265. ;;
  1266. centos)
  1267. # shellcheck disable=SC2068
  1268. yum remove -y $@
  1269. ;;
  1270. esac
  1271. }
  1272. pkg_is_installed() {
  1273. # usage: pkg_is_install foopkg || pkg_install foopkg
  1274. case $DIST_ID in
  1275. ubuntu|debian)
  1276. dpkg -l "$1" &> /dev/null
  1277. return $?
  1278. ;;
  1279. arch)
  1280. pacman -Qsq "$1" &> /dev/null
  1281. return $?
  1282. ;;
  1283. fedora)
  1284. dnf list -q --installed "$1" &> /dev/null
  1285. return $?
  1286. ;;
  1287. centos)
  1288. yum list -q --installed "$1" &> /dev/null
  1289. return $?
  1290. ;;
  1291. esac
  1292. }
  1293. # git tooling
  1294. # -----------
  1295. # shellcheck disable=SC2164
  1296. git_clone() {
  1297. # usage:
  1298. #
  1299. # git_clone <url> <name> [<branch> [<user>]]
  1300. # git_clone <url> <path> [<branch> [<user>]]
  1301. #
  1302. # First form uses $CACHE/<name> as destination folder, second form clones
  1303. # into <path>. If repository is already cloned, pull from <branch> and
  1304. # update working tree (if needed, the caller has to stash local changes).
  1305. #
  1306. # git clone https://github.com/searxng/searxng searx-src origin/master searxlogin
  1307. #
  1308. local url="$1"
  1309. local dest="$2"
  1310. local branch="$3"
  1311. local user="$4"
  1312. local bash_cmd="bash"
  1313. local remote="origin"
  1314. if [[ ! "${dest:0:1}" = "/" ]]; then
  1315. dest="$CACHE/$dest"
  1316. fi
  1317. [[ -z $branch ]] && branch=master
  1318. [[ -z $user ]] && [[ -n "${SUDO_USER}" ]] && user="${SUDO_USER}"
  1319. [[ -n $user ]] && bash_cmd="sudo -H -u $user -i"
  1320. if [[ -d "${dest}" ]] ; then
  1321. info_msg "already cloned: $dest"
  1322. tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
  1323. cd "${dest}"
  1324. git checkout -m -B "$branch" --track "$remote/$branch"
  1325. git pull --all
  1326. EOF
  1327. else
  1328. info_msg "clone into: $dest"
  1329. tee_stderr 0.1 <<EOF | $bash_cmd 2>&1 | prefix_stdout " ${_Yellow}|$user|${_creset} "
  1330. mkdir -p "$(dirname "$dest")"
  1331. cd "$(dirname "$dest")"
  1332. git clone --branch "$branch" --origin "$remote" "$url" "$(basename "$dest")"
  1333. EOF
  1334. fi
  1335. }
  1336. # containers
  1337. # ----------
  1338. in_container() {
  1339. # Test if shell runs in a container.
  1340. #
  1341. # usage: in_container && echo "process running inside a LXC container"
  1342. # in_container || echo "process is not running inside a LXC container"
  1343. #
  1344. # sudo_or_exit
  1345. # hint: Reads init process environment, therefore root access is required!
  1346. # to be safe, take a look at the environment of process 1 (/sbin/init)
  1347. # grep -qa 'container=lxc' /proc/1/environ
  1348. # see lxc_init_container_env
  1349. [[ -f /.lxcenv ]]
  1350. }
  1351. LXC_ENV_FOLDER=
  1352. if in_container; then
  1353. # shellcheck disable=SC2034
  1354. LXC_ENV_FOLDER="lxc-env/$(hostname)/"
  1355. PY_ENV="${LXC_ENV_FOLDER}${PY_ENV}"
  1356. PY_ENV_BIN="${LXC_ENV_FOLDER}${PY_ENV_BIN}"
  1357. PYDIST="${LXC_ENV_FOLDER}${PYDIST}"
  1358. PYBUILD="${LXC_ENV_FOLDER}${PYBUILD}"
  1359. DOCS_DIST="${LXC_ENV_FOLDER}${DOCS_DIST}"
  1360. DOCS_BUILD="${LXC_ENV_FOLDER}${DOCS_BUILD}"
  1361. fi
  1362. lxc_init_container_env() {
  1363. # usage: lxc_init_container_env <name>
  1364. # Create a /.lxcenv file in the root folder. Call this once after the
  1365. # container is initial started and before installing any boilerplate stuff.
  1366. info_msg "create /.lxcenv in container $1"
  1367. cat <<EOF | lxc exec "${1}" -- bash | prefix_stdout "[${_BBlue}${1}${_creset}] "
  1368. touch "/.lxcenv"
  1369. ls -l "/.lxcenv"
  1370. EOF
  1371. }
  1372. # apt packages
  1373. LXC_BASE_PACKAGES_debian="bash git build-essential python3 python3-venv python-is-python3"
  1374. # pacman packages
  1375. LXC_BASE_PACKAGES_arch="bash git base-devel python"
  1376. # dnf packages
  1377. LXC_BASE_PACKAGES_fedora="bash git @development-tools python"
  1378. # yum packages
  1379. LXC_BASE_PACKAGES_centos="bash git python3"
  1380. lxc_distro_setup() {
  1381. case $DIST_ID in
  1382. ubuntu|debian) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_debian}" ;;
  1383. arch) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_arch}" ;;
  1384. fedora) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_fedora}" ;;
  1385. centos) LXC_BASE_PACKAGES="${LXC_BASE_PACKAGES_centos}" ;;
  1386. *) err_msg "$DIST_ID-$DIST_VERS: pkg_install LXC_BASE_PACKAGES not yet implemented" ;;
  1387. esac
  1388. }
  1389. lxc_install_base_packages() {
  1390. info_msg "install LXC_BASE_PACKAGES in container $1"
  1391. case $DIST_ID in
  1392. centos) yum groupinstall "Development Tools" -y ;;
  1393. esac
  1394. pkg_install "${LXC_BASE_PACKAGES}"
  1395. }
  1396. lxc_image_copy() {
  1397. # usage: lxc_image_copy <remote image> <local image>
  1398. #
  1399. # lxc_image_copy "images:ubuntu/20.04" "ubu2004"
  1400. if lxc_image_exists "local:${LXC_SUITE[i+1]}"; then
  1401. info_msg "image ${LXC_SUITE[i]} already copied --> ${LXC_SUITE[i+1]}"
  1402. else
  1403. info_msg "copy image locally ${LXC_SUITE[i]} --> ${LXC_SUITE[i+1]}"
  1404. lxc image copy "${LXC_SUITE[i]}" local: \
  1405. --alias "${LXC_SUITE[i+1]}" | prefix_stdout
  1406. fi
  1407. }
  1408. lxc_init_container() {
  1409. # usage: lxc_init_container <image name> <container name>
  1410. local image_name="$1"
  1411. local container_name="$2"
  1412. if lxc info "${container_name}" &>/dev/null; then
  1413. info_msg "container '${container_name}' already exists"
  1414. else
  1415. info_msg "create container instance: ${container_name}"
  1416. lxc init "local:${image_name}" "${container_name}"
  1417. fi
  1418. }
  1419. lxc_exists(){
  1420. # usage: lxc_exists <name> || echo "container <name> does not exists"
  1421. lxc info "$1" &>/dev/null
  1422. }
  1423. lxc_image_exists(){
  1424. # usage: lxc_image_exists <alias> || echo "image <alias> does locally not exists"
  1425. lxc image info "local:$1" &>/dev/null
  1426. }
  1427. lxc_delete_container() {
  1428. # usage: lxc_delete_container <container-name>
  1429. if lxc info "$1" &>/dev/null; then
  1430. info_msg "stop & delete instance ${_BBlue}${1}${_creset}"
  1431. lxc stop "$1" &>/dev/null
  1432. lxc delete "$1" | prefix_stdout
  1433. else
  1434. warn_msg "instance '$1' does not exist / can't delete :o"
  1435. fi
  1436. }
  1437. lxc_delete_local_image() {
  1438. # usage: lxc_delete_local_image <container-name>
  1439. info_msg "delete image 'local:$i'"
  1440. lxc image delete "local:$i"
  1441. }
  1442. # IP
  1443. # --
  1444. global_IPs(){
  1445. # usage: global_IPS
  1446. #
  1447. # print list of host's SCOPE global addresses and adapters e.g::
  1448. #
  1449. # $ global_IPs
  1450. # enp4s0|192.168.1.127
  1451. # lxdbr0|10.246.86.1
  1452. # lxdbr0|fd42:8c58:2cd:b73f::1
  1453. ip -o addr show | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\1|\2/p'
  1454. }
  1455. primary_ip() {
  1456. case $DIST_ID in
  1457. arch)
  1458. ip -o addr show \
  1459. | sed -nr 's/[0-9]*:\s*([a-z0-9]*).*inet[6]?\s*([a-z0-9.:]*).*scope global.*/\2/p' \
  1460. | head -n 1
  1461. ;;
  1462. *) hostname -I | cut -d' ' -f1 ;;
  1463. esac
  1464. }
  1465. # URL
  1466. # ---
  1467. url_replace_hostname(){
  1468. # usage: url_replace_hostname <url> <new hostname>
  1469. # to replace hostname by primary IP::
  1470. #
  1471. # url_replace_hostname http://searx-ubu1604/example $(primary_ip)
  1472. # http://10.246.86.250/example
  1473. # shellcheck disable=SC2001
  1474. echo "$1" | sed "s|\(http[s]*://\)[^/]*\(.*\)|\1$2\2|"
  1475. }