manage 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767
  1. #!/usr/bin/env bash
  2. # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
  3. # SPDX-License-Identifier: AGPL-3.0-or-later
  4. # shellcheck disable=SC2034
  5. main_cmd="$(basename "$0")"
  6. # shellcheck source=utils/lib.sh
  7. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
  8. # shellcheck source=utils/lib.sh
  9. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_nvm.sh"
  10. # shellcheck source=utils/lib_static.sh
  11. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
  12. # shellcheck source=utils/lib_go.sh
  13. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_go.sh"
  14. # config
  15. PYOBJECTS="searx"
  16. PY_SETUP_EXTRAS='[test]'
  17. GECKODRIVER_VERSION="v0.30.0"
  18. export NODE_MINIMUM_VERSION="16.13.0"
  19. # SPHINXOPTS=
  20. BLACK_OPTIONS=("--target-version" "py37" "--line-length" "120" "--skip-string-normalization")
  21. BLACK_TARGETS=("--exclude" "searx/static,searx/languages.py" "searx" "searxng_extra" "tests")
  22. pylint.FILES() {
  23. # List files tagged by comment:
  24. #
  25. # # lint: pylint
  26. #
  27. # These py files are linted by test.pylint()
  28. grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
  29. }
  30. YAMLLINT_FILES=()
  31. while IFS= read -r line; do
  32. YAMLLINT_FILES+=("$line")
  33. done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
  34. PYLINT_SEARXNG_DISABLE_OPTION="\
  35. I,C,R,\
  36. W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
  37. E1136"
  38. PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories"
  39. PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
  40. help() {
  41. nvm.help
  42. cat <<EOF
  43. buildenv:
  44. rebuild ./utils/brand.env
  45. weblate.:
  46. push.translations: push translation changes from SearXNG to Weblate's counterpart
  47. to.translations: Update 'translations' branch with last additions from Weblate.
  48. data.:
  49. all : update searx/languages.py and ./data/*
  50. languages : update searx/data/engines_languages.json & searx/languages.py
  51. useragents: update searx/data/useragents.json with the most recent versions of Firefox.
  52. docs.:
  53. html : build HTML documentation
  54. live : autobuild HTML documentation while editing
  55. gh-pages : deploy on gh-pages branch
  56. prebuild : build reST include files (./${DOCS_BUILD}/includes)
  57. clean : clean documentation build
  58. docker.:
  59. build : build docker image
  60. push : build and push docker image
  61. gecko.driver:
  62. download & install geckodriver if not already installed (required for
  63. robot_tests)
  64. EOF
  65. nvm.help
  66. cat <<EOF
  67. node.:
  68. env : download & install npm dependencies locally
  69. clean : drop locally npm installations
  70. py.:
  71. build : Build python packages at ./${PYDIST}
  72. clean : delete virtualenv and intermediate py files
  73. pyenv.:
  74. install : developer install of SearXNG into virtualenv
  75. uninstall : uninstall developer installation
  76. cmd ... : run command ... in virtualenv
  77. OK : test if virtualenv is OK
  78. pypi.upload:
  79. Upload python packages to PyPi (to test use pypi.upload.test)
  80. format.:
  81. python : format Python code source using black
  82. test.:
  83. yamllint : lint YAML files (YAMLLINT_FILES)
  84. pylint : lint PYLINT_FILES, searx/engines, searx & tests
  85. black : check black code format
  86. unit : run unit tests
  87. coverage : run unit tests with coverage
  88. robot : run robot test
  89. clean : clean intermediate test stuff
  90. themes.:
  91. all : build all themes
  92. oscar : build oscar theme
  93. simple : build simple theme
  94. pygments.:
  95. less : build LESS files for pygments
  96. EOF
  97. go.help
  98. static_help
  99. }
  100. if [ "$VERBOSE" = "1" ]; then
  101. SPHINX_VERBOSE="-v"
  102. PYLINT_VERBOSE="-v"
  103. fi
  104. # needed by sphinx-docs
  105. export DOCS_BUILD
  106. buildenv() {
  107. # settings file from repository's working tree are used by default
  108. SEARXNG_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"
  109. if [ -f /etc/searx/settings.yml ]; then
  110. err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
  111. fi
  112. if [ -r '/etc/searxng/settings.yml' ]; then
  113. if ask_yn "should settings read from: /etc/searxng/settings.yml"; then
  114. SEARXNG_SETTINGS_PATH='/etc/searxng/settings.yml'
  115. fi
  116. fi
  117. export SEARXNG_SETTINGS_PATH
  118. (
  119. set -e
  120. SEARXNG_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
  121. | prefix_stdout "${_Blue}BUILDENV${_creset} "
  122. )
  123. return "${PIPESTATUS[0]}"
  124. }
  125. TRANSLATIONS_WORKTREE="$CACHE/translations"
  126. weblate.translations.worktree() {
  127. # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
  128. # 'translations' from Weblate's counterpart (weblate) of the SearXNG
  129. # (origin).
  130. #
  131. # remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
  132. ( set -e
  133. if ! git remote get-url weblate 2> /dev/null; then
  134. git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
  135. fi
  136. if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
  137. pushd .
  138. cd "${TRANSLATIONS_WORKTREE}"
  139. git reset --hard HEAD
  140. git pull origin translations
  141. popd
  142. else
  143. mkdir -p "${TRANSLATIONS_WORKTREE}"
  144. git worktree add "${TRANSLATIONS_WORKTREE}" translations
  145. fi
  146. )
  147. }
  148. weblate.to.translations() {
  149. # Update 'translations' branch of SearXNG (origin) with last additions from
  150. # Weblate.
  151. # 1. Check if Weblate is locked, if not die with error message
  152. # 2. On Weblate's counterpart (weblate), pull master and translations branch
  153. # from SearXNG (origin).
  154. # 3. Commit changes made in a Weblate object on Weblate's counterpart
  155. # (weblate).
  156. # 4. In translations worktree, merge changes of branch 'translations' from
  157. # remote 'weblate' and push it on branch 'translations' of 'origin'
  158. ( set -e
  159. pyenv.activate
  160. if [ "$(wlc lock-status)" != "locked: True" ]; then
  161. die 1 "weblate must be locked, currently: $(wlc lock-status)"
  162. fi
  163. # weblate: commit pending changes
  164. wlc pull
  165. wlc commit
  166. # get the translations in a worktree
  167. weblate.translations.worktree
  168. pushd "${TRANSLATIONS_WORKTREE}"
  169. git remote update weblate
  170. git merge weblate/translations
  171. git push
  172. popd
  173. )
  174. dump_return $?
  175. }
  176. weblate.translations.commit() {
  177. # Update 'translations' branch of SearXNG (origin) with last additions from
  178. # Weblate. Copy the changes to the master branch, compile translations and
  179. # create a commit in the local branch (master)
  180. local existing_commit_hash commit_body commit_message exitcode
  181. ( set -e
  182. pyenv.activate
  183. # lock change on weblate
  184. wlc lock
  185. # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
  186. weblate.translations.worktree
  187. existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
  188. # pull weblate commits
  189. weblate.to.translations
  190. # copy the changes to the master branch
  191. cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
  192. # compile translations
  193. build_msg BABEL 'compile translation catalogs into binary MO files'
  194. pybabel compile --statistics \
  195. -d "searx/translations"
  196. # git add/commit (no push)
  197. commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
  198. commit_message=$(echo -e "[translations] update from Weblate\n\n${commit_body}")
  199. git add searx/translations
  200. git commit -m "${commit_message}"
  201. )
  202. exitcode=$?
  203. ( # make sure to always unlock weblate
  204. set -e
  205. pyenv.cmd wlc unlock
  206. )
  207. dump_return $exitcode
  208. }
  209. weblate.push.translations() {
  210. # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
  211. # (weblate).
  212. # In branch master of SearXNG (origin) check for meaningful changes in
  213. # folder 'searx/translations', commit changes on branch 'translations' and
  214. # at least, pull updated branches on Weblate's counterpart (weblate).
  215. # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
  216. # 'translations' from remote 'weblate'.
  217. # 2. Stop if there is no meaningful change in the 'master' branch (origin),
  218. # compared to the 'translations' branch (weblate), otherwise ...
  219. # 3. Update 'translations' branch of SearXNG (origin) with last additions
  220. # from Weblate.
  221. # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
  222. local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
  223. exitcode
  224. messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
  225. ( set -e
  226. pyenv.activate
  227. # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
  228. weblate.translations.worktree
  229. # update messages.pot in the master branch
  230. build_msg BABEL 'extract messages from source files and generate POT file'
  231. pybabel extract -F babel.cfg \
  232. -o "${messages_pot}" \
  233. "searx/"
  234. # stop if there is no meaningful change in the master branch
  235. diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
  236. git diff -- "searx/translations/messages.pot")
  237. if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
  238. build_msg BABEL 'no changes detected, exiting'
  239. return 42
  240. fi
  241. return 0
  242. )
  243. exitcode=$?
  244. if [ "$exitcode" -eq 42 ]; then
  245. return 0
  246. fi
  247. if [ "$exitcode" -gt 0 ]; then
  248. return $exitcode
  249. fi
  250. (
  251. set -e
  252. pyenv.activate
  253. # lock change on weblate
  254. # weblate may add commit(s) since the call to "weblate.translations.worktree".
  255. # this is not a problem because after this line, "weblate.to.translations"
  256. # calls again "weblate.translations.worktree" which calls "git pull"
  257. wlc lock
  258. # save messages.pot in the translations branch for later
  259. pushd "${TRANSLATIONS_WORKTREE}"
  260. git stash push
  261. popd
  262. # merge weblate commits into the translations branch
  263. weblate.to.translations
  264. # restore messages.pot in the translations branch
  265. pushd "${TRANSLATIONS_WORKTREE}"
  266. git stash pop
  267. popd
  268. # update messages.po files in the master branch
  269. build_msg BABEL 'update existing message catalogs from POT file'
  270. pybabel update -N \
  271. -i "${messages_pot}" \
  272. -d "${TRANSLATIONS_WORKTREE}/searx/translations"
  273. # git add/commit/push
  274. last_commit_hash=$(git log -n1 --pretty=format:'%h')
  275. last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
  276. pushd "${TRANSLATIONS_WORKTREE}"
  277. git add searx/translations
  278. git commit \
  279. -m "[translations] update messages.pot and messages.po files" \
  280. -m "From ${last_commit_detail}"
  281. git push
  282. popd
  283. # notify weblate to pull updated master & translations branch
  284. wlc pull
  285. )
  286. exitcode=$?
  287. ( # make sure to always unlock weblate
  288. set -e
  289. pyenv.activate
  290. wlc unlock
  291. )
  292. dump_return $exitcode
  293. }
  294. data.all() {
  295. ( set -e
  296. pyenv.activate
  297. data.languages
  298. data.useragents
  299. data.osm_keys_tags
  300. build_msg DATA "update searx/data/ahmia_blacklist.txt"
  301. python searxng_extra/update/update_ahmia_blacklist.py
  302. build_msg DATA "update searx/data/wikidata_units.json"
  303. python searxng_extra/update/update_wikidata_units.py
  304. build_msg DATA "update searx/data/currencies.json"
  305. python searxng_extra/update/update_currencies.py
  306. )
  307. }
  308. data.languages() {
  309. ( set -e
  310. pyenv.activate
  311. build_msg ENGINES "fetch languages .."
  312. python searxng_extra/update/update_languages.py
  313. build_msg ENGINES "update update searx/languages.py"
  314. build_msg DATA "update searx/data/engines_languages.json"
  315. )
  316. dump_return $?
  317. }
  318. data.useragents() {
  319. build_msg DATA "update searx/data/useragents.json"
  320. pyenv.cmd python searxng_extra/update/update_firefox_version.py
  321. dump_return $?
  322. }
  323. data.osm_keys_tags() {
  324. build_msg DATA "update searx/data/osm_keys_tags.json"
  325. pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
  326. dump_return $?
  327. }
  328. docs.prebuild() {
  329. build_msg DOCS "build ${DOCS_BUILD}/includes"
  330. (
  331. set -e
  332. [ "$VERBOSE" = "1" ] && set -x
  333. mkdir -p "${DOCS_BUILD}/includes"
  334. ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
  335. ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
  336. ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
  337. )
  338. dump_return $?
  339. }
  340. docker.push() {
  341. docker.build push
  342. }
  343. docker.buildx() {
  344. docker.build buildx
  345. }
  346. # shellcheck disable=SC2119
  347. docker.build() {
  348. pyenv.install
  349. local SEARXNG_GIT_VERSION
  350. local VERSION_GITCOMMIT
  351. local GITHUB_USER
  352. local SEARXNG_IMAGE_NAME
  353. local BUILD
  354. build_msg DOCKER build
  355. # run installation in a subprocess and activate pyenv
  356. # See https://www.shellcheck.net/wiki/SC1001 and others ..
  357. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
  358. ( set -e
  359. pyenv.activate
  360. # Check if it is a git repository
  361. if [ ! -d .git ]; then
  362. die 1 "This is not Git repository"
  363. fi
  364. if [ ! -x "$(which git)" ]; then
  365. die 1 "git is not installed"
  366. fi
  367. if ! git remote get-url origin 2> /dev/null; then
  368. die 1 "there is no remote origin"
  369. fi
  370. # This is a git repository
  371. git update-index -q --refresh
  372. python -m searx.version freeze
  373. eval "$(python -m searx.version)"
  374. # Get the last git commit id
  375. VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3)
  376. build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
  377. # define the docker image name
  378. GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
  379. SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
  380. BUILD="build"
  381. if [ "$1" = "buildx" ]; then
  382. # buildx includes the push option
  383. CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
  384. BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
  385. shift
  386. fi
  387. build_msg DOCKER "Build command: ${BUILD}"
  388. # build Docker image
  389. build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
  390. # shellcheck disable=SC2086
  391. docker $BUILD \
  392. --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
  393. --build-arg GIT_URL="${GIT_URL}" \
  394. --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
  395. --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
  396. --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
  397. --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
  398. --build-arg LABEL_VCS_URL="${GIT_URL}" \
  399. --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
  400. --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
  401. -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${VERSION_STRING}" .
  402. if [ "$1" = "push" ]; then
  403. docker push "${SEARXNG_IMAGE_NAME}:latest"
  404. docker push "${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
  405. fi
  406. )
  407. dump_return $?
  408. }
  409. # shellcheck disable=SC2119
  410. gecko.driver() {
  411. pyenv.install
  412. build_msg INSTALL "gecko.driver"
  413. # run installation in a subprocess and activate pyenv
  414. ( set -e
  415. pyenv.activate
  416. INSTALLED_VERSION=$(geckodriver -V 2> /dev/null | head -1 | awk '{ print "v" $2}') || INSTALLED_VERSION=""
  417. set +e
  418. if [ "${INSTALLED_VERSION}" = "${GECKODRIVER_VERSION}" ]; then
  419. build_msg INSTALL "geckodriver already installed"
  420. return
  421. fi
  422. PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
  423. case "$PLATFORM" in
  424. "linux 32bit" | "linux2 32bit") ARCH="linux32";;
  425. "linux 64bit" | "linux2 64bit") ARCH="linux64";;
  426. "windows 32 bit") ARCH="win32";;
  427. "windows 64 bit") ARCH="win64";;
  428. "mac 64bit") ARCH="macos";;
  429. esac
  430. GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
  431. build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
  432. FILE="$(mktemp)"
  433. wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
  434. rm -- "$FILE"
  435. chmod 755 -- "${PY_ENV_BIN}/geckodriver"
  436. )
  437. dump_return $?
  438. }
  439. node.env() {
  440. if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
  441. info_msg "install Node.js by NVM"
  442. nvm.nodejs
  443. fi
  444. ( set -e
  445. build_msg INSTALL "searx/static/themes/oscar/package.json"
  446. npm --prefix searx/static/themes/oscar install
  447. build_msg INSTALL "searx/static/themes/simple/package.json"
  448. npm --prefix searx/static/themes/simple install
  449. )
  450. dump_return $?
  451. }
  452. node.clean() {
  453. if ! required_commands npm 2>/dev/null; then
  454. build_msg CLEAN "npm is not installed / ignore npm dependencies"
  455. return 0
  456. fi
  457. build_msg CLEAN "themes -- locally installed npm dependencies"
  458. ( set -e
  459. npm --prefix searx/static/themes/oscar run clean
  460. npm --prefix searx/static/themes/simple run clean
  461. )
  462. dump_return $?
  463. }
  464. pygments.less() {
  465. build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
  466. if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
  467. build_msg PYGMENTS "building LESS files for pygments failed"
  468. return 1
  469. fi
  470. return 0
  471. }
  472. py.build() {
  473. build_msg BUILD "python package ${PYDIST}"
  474. pyenv.cmd python setup.py \
  475. sdist -d "${PYDIST}" \
  476. bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
  477. }
  478. py.clean() {
  479. build_msg CLEAN pyenv
  480. ( set -e
  481. pyenv.drop
  482. [ "$VERBOSE" = "1" ] && set -x
  483. rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
  484. find . -name '*.pyc' -exec rm -f {} +
  485. find . -name '*.pyo' -exec rm -f {} +
  486. find . -name __pycache__ -exec rm -rf {} +
  487. )
  488. }
  489. pyenv.check() {
  490. cat <<EOF
  491. import yaml
  492. print('import yaml --> OK')
  493. EOF
  494. }
  495. pyenv.install() {
  496. if ! pyenv.OK; then
  497. py.clean > /dev/null
  498. fi
  499. if pyenv.install.OK > /dev/null; then
  500. return 0
  501. fi
  502. ( set -e
  503. pyenv
  504. build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
  505. "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
  506. buildenv
  507. )
  508. local exit_val=$?
  509. if [ ! $exit_val -eq 0 ]; then
  510. die 42 "error while pip install (${PY_ENV_BIN})"
  511. fi
  512. }
  513. pyenv.uninstall() {
  514. build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
  515. pyenv.cmd python setup.py develop --uninstall 2>&1 \
  516. | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
  517. }
  518. pypi.upload() {
  519. py.clean
  520. py.build
  521. # https://github.com/pypa/twine
  522. pyenv.cmd twine upload "${PYDIST}"/*
  523. }
  524. pypi.upload.test() {
  525. py.clean
  526. py.build
  527. pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
  528. }
  529. format.python() {
  530. build_msg TEST "[format.python] black \$BLACK_TARGETS"
  531. pyenv.cmd black "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
  532. dump_return $?
  533. }
  534. test.yamllint() {
  535. build_msg TEST "[yamllint] \$YAMLLINT_FILES"
  536. pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
  537. }
  538. test.pylint() {
  539. # shellcheck disable=SC2086
  540. ( set -e
  541. build_msg TEST "[pylint] \$PYLINT_FILES"
  542. pyenv.activate
  543. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  544. --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
  545. "${PYLINT_FILES[@]}"
  546. build_msg TEST "[pylint] searx/engines"
  547. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  548. --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
  549. --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
  550. searx/engines
  551. build_msg TEST "[pylint] searx tests"
  552. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  553. --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
  554. --ignore=searx/engines \
  555. searx tests
  556. )
  557. dump_return $?
  558. }
  559. test.black() {
  560. build_msg TEST "[black] \$BLACK_TARGETS"
  561. pyenv.cmd black --check --diff "${BLACK_OPTIONS[@]}" "${BLACK_TARGETS[@]}"
  562. dump_return $?
  563. }
  564. test.unit() {
  565. build_msg TEST 'tests/unit'
  566. pyenv.cmd python -m nose2 -s tests/unit
  567. dump_return $?
  568. }
  569. test.coverage() {
  570. build_msg TEST 'unit test coverage'
  571. ( set -e
  572. pyenv.activate
  573. python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
  574. coverage report
  575. coverage html
  576. )
  577. dump_return $?
  578. }
  579. test.robot() {
  580. build_msg TEST 'robot'
  581. gecko.driver
  582. PYTHONPATH=. pyenv.cmd python -m tests.robot
  583. dump_return $?
  584. }
  585. test.clean() {
  586. build_msg CLEAN "test stuff"
  587. rm -rf geckodriver.log .coverage coverage/
  588. dump_return $?
  589. }
  590. themes.all() {
  591. ( set -e
  592. pygments.less
  593. node.env
  594. themes.oscar
  595. themes.simple
  596. )
  597. dump_return $?
  598. }
  599. themes.oscar() {
  600. build_msg GRUNT "theme: oscar"
  601. npm --prefix searx/static/themes/oscar run build
  602. dump_return $?
  603. }
  604. themes.simple() {
  605. local static="searx/static/themes/simple"
  606. ( set -e
  607. build_msg GRUNT "theme: simple"
  608. npm --prefix searx/static/themes/simple run build
  609. )
  610. dump_return $?
  611. }
  612. themes.simple.test() {
  613. build_msg TEST "theme: simple"
  614. if ! nvm.min_node "${NODE_MINIMUM_VERSION}"; then
  615. info_msg "install Node.js by NVM"
  616. nvm.nodejs
  617. fi
  618. npm --prefix searx/static/themes/simple install
  619. npm --prefix searx/static/themes/simple run test
  620. dump_return $?
  621. }
  622. PYLINT_FILES=()
  623. while IFS= read -r line; do
  624. PYLINT_FILES+=("$line")
  625. done <<< "$(pylint.FILES)"
  626. # shellcheck disable=SC2119
  627. main() {
  628. local _type
  629. local cmd="$1"; shift
  630. if [ "$cmd" == "" ]; then
  631. help
  632. err_msg "missing command"
  633. return 42
  634. fi
  635. case "$cmd" in
  636. --getenv) var="$1"; echo "${!var}";;
  637. --help) help;;
  638. --*)
  639. help
  640. err_msg "unknown option $cmd"
  641. return 42
  642. ;;
  643. *)
  644. _type="$(type -t "$cmd")"
  645. if [ "$_type" != 'function' ]; then
  646. err_msg "unknown command: $cmd / use --help"
  647. return 42
  648. else
  649. "$cmd" "$@"
  650. fi
  651. ;;
  652. esac
  653. }
  654. main "$@"