manage 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772
  1. #!/usr/bin/env bash
  2. # -*- coding: utf-8; mode: sh indent-tabs-mode: nil -*-
  3. # SPDX-License-Identifier: AGPL-3.0-or-later
  4. # shellcheck disable=SC2031
  5. # shellcheck source=utils/lib.sh
  6. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib.sh"
  7. # shellcheck source=utils/lib_static.sh
  8. source "$(dirname "${BASH_SOURCE[0]}")/utils/lib_static.sh"
  9. # config
  10. PYOBJECTS="searx"
  11. PY_SETUP_EXTRAS='[test]'
  12. GECKODRIVER_VERSION="v0.28.0"
  13. # SPHINXOPTS=
  14. pylint.FILES() {
  15. # List files tagged by comment:
  16. #
  17. # # lint: pylint
  18. #
  19. # These py files are linted by test.pylint(), all other files are linted by
  20. # test.pep8()
  21. grep -l -r --include \*.py '^#[[:blank:]]*lint:[[:blank:]]*pylint' searx searxng_extra tests
  22. }
  23. YAMLLINT_FILES=()
  24. while IFS= read -r line; do
  25. YAMLLINT_FILES+=("$line")
  26. done <<< "$(git ls-files './tests/*.yml' './searx/*.yml' './utils/templates/etc/searxng/*.yml')"
  27. PYLINT_SEARXNG_DISABLE_OPTION="\
  28. I,C,R,\
  29. W0105,W0212,W0511,W0603,W0613,W0621,W0702,W0703,W1401,\
  30. E1136"
  31. PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES="supported_languages,language_aliases,logger,categories"
  32. PYLINT_OPTIONS="-m pylint -j 0 --rcfile .pylintrc"
  33. help() {
  34. cat <<EOF
  35. buildenv:
  36. rebuild ./utils/brand.env
  37. weblate.:
  38. push.translations: push translation changes from SearXNG to Weblate's counterpart
  39. to.translations: Update 'translations' branch with last additions from Weblate.
  40. data.:
  41. all : update searx/languages.py and ./data/*
  42. languages : update searx/data/engines_languages.json & searx/languages.py
  43. useragents: update searx/data/useragents.json with the most recent versions of Firefox.
  44. docs.:
  45. html : build HTML documentation
  46. live : autobuild HTML documentation while editing
  47. gh-pages : deploy on gh-pages branch
  48. prebuild : build reST include files (./${DOCS_BUILD}/includes)
  49. clean : clean documentation build
  50. docker.:
  51. build : build docker image
  52. push : build and push docker image
  53. gecko.driver:
  54. download & install geckodriver if not already installed (required for
  55. robot_tests)
  56. node.:
  57. env : download & install npm dependencies locally
  58. clean : drop npm installations
  59. py.:
  60. build : Build python packages at ./${PYDIST}
  61. clean : delete virtualenv and intermediate py files
  62. pyenv.:
  63. install : developer install of SearXNG into virtualenv
  64. uninstall : uninstall developer installation
  65. cmd ... : run command ... in virtualenv
  66. OK : test if virtualenv is OK
  67. pypi.upload:
  68. Upload python packages to PyPi (to test use pypi.upload.test)
  69. test.:
  70. yamllint : lint YAML files (YAMLLINT_FILES)
  71. pylint : lint PYLINT_FILES, searx/engines, searx & tests
  72. pep8 : pycodestyle (pep8) for all files except PYLINT_FILES
  73. unit : run unit tests
  74. coverage : run unit tests with coverage
  75. robot : run robot test
  76. clean : clean intermediate test stuff
  77. themes.:
  78. all : build all themes
  79. oscar : build oscar theme
  80. simple : build simple theme
  81. pygments.:
  82. less : build LESS files for pygments
  83. EOF
  84. static_help
  85. }
  86. if [ "$VERBOSE" = "1" ]; then
  87. SPHINX_VERBOSE="-v"
  88. PYLINT_VERBOSE="-v"
  89. fi
  90. # needed by sphinx-docs
  91. export DOCS_BUILD
  92. buildenv() {
  93. # settings file from repository's working tree are used by default
  94. SEARXNG_SETTINGS_PATH="${REPO_ROOT}/searx/settings.yml"
  95. if [ -f /etc/searx/settings.yml ]; then
  96. err_msg "settings.yml in /etc/searx/ is deprecated, move file to folder /etc/searxng/"
  97. fi
  98. if [ -r '/etc/searxng/settings.yml' ]; then
  99. if ask_yn "should settings read from: /etc/searxng/settings.yml"; then
  100. SEARXNG_SETTINGS_PATH='/etc/searxng/settings.yml'
  101. fi
  102. fi
  103. export SEARXNG_SETTINGS_PATH
  104. (
  105. set -e
  106. SEARXNG_DEBUG=1 pyenv.cmd python utils/build_env.py 2>&1 \
  107. | prefix_stdout "${_Blue}BUILDENV${_creset} "
  108. )
  109. return "${PIPESTATUS[0]}"
  110. }
  111. TRANSLATIONS_WORKTREE="$CACHE/translations"
  112. weblate.translations.worktree() {
  113. # Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
  114. # 'translations' from Weblate's counterpart (weblate) of the SearXNG
  115. # (origin).
  116. #
  117. # remote weblate https://weblate.bubu1.eu/git/searxng/searxng/
  118. ( set -e
  119. if ! git remote get-url weblate 2> /dev/null; then
  120. git remote add weblate https://weblate.bubu1.eu/git/searxng/searxng/
  121. fi
  122. if [ -d "${TRANSLATIONS_WORKTREE}" ]; then
  123. pushd .
  124. cd "${TRANSLATIONS_WORKTREE}"
  125. git reset --hard HEAD
  126. git pull origin translations
  127. popd
  128. else
  129. mkdir -p "${TRANSLATIONS_WORKTREE}"
  130. git worktree add "${TRANSLATIONS_WORKTREE}" translations
  131. fi
  132. )
  133. }
  134. weblate.to.translations() {
  135. # Update 'translations' branch of SearXNG (origin) with last additions from
  136. # Weblate.
  137. # 1. Check if Weblate is locked, if not die with error message
  138. # 2. On Weblate's counterpart (weblate), pull master and translations branch
  139. # from SearXNG (origin).
  140. # 3. Commit changes made in a Weblate object on Weblate's counterpart
  141. # (weblate).
  142. # 4. In translations worktree, merge changes of branch 'translations' from
  143. # remote 'weblate' and push it on branch 'translations' of 'origin'
  144. ( set -e
  145. pyenv.activate
  146. if [ "$(wlc lock-status)" != "locked: True" ]; then
  147. die 1 "weblate must be locked, currently: $(wlc lock-status)"
  148. fi
  149. # weblate: commit pending changes
  150. wlc pull
  151. wlc commit
  152. # get the translations in a worktree
  153. weblate.translations.worktree
  154. pushd "${TRANSLATIONS_WORKTREE}"
  155. git remote update weblate
  156. git merge weblate/translations
  157. git push
  158. popd
  159. )
  160. dump_return $?
  161. }
  162. weblate.translations.commit() {
  163. # Update 'translations' branch of SearXNG (origin) with last additions from
  164. # Weblate. Copy the changes to the master branch, compile translations and
  165. # create a commit in the local branch (master)
  166. local existing_commit_hash commit_body commit_message exitcode
  167. ( set -e
  168. pyenv.activate
  169. # lock change on weblate
  170. wlc lock
  171. # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
  172. weblate.translations.worktree
  173. existing_commit_hash=$(cd "${TRANSLATIONS_WORKTREE}"; git log -n1 --pretty=format:'%h')
  174. # pull weblate commits
  175. weblate.to.translations
  176. # copy the changes to the master branch
  177. cp -rv --preserve=mode,timestamps "${TRANSLATIONS_WORKTREE}/searx/translations" "searx"
  178. # compile translations
  179. build_msg BABEL 'compile translation catalogs into binary MO files'
  180. pybabel compile --statistics \
  181. -d "searx/translations"
  182. # git add/commit (no push)
  183. commit_body=$(cd "${TRANSLATIONS_WORKTREE}"; git log --pretty=format:'%h - %as - %aN <%ae>' "${existing_commit_hash}..HEAD")
  184. commit_message=$(echo -e "[translations] update\n${commit_body}")
  185. git add searx/translations
  186. git commit -m "${commit_message}"
  187. )
  188. exitcode=$?
  189. ( # make sure to always unlock weblate
  190. set -e
  191. pyenv.cmd wlc unlock
  192. )
  193. dump_return $exitcode
  194. }
  195. weblate.push.translations() {
  196. # Push *translation changes* from SearXNG (origin) to Weblate's counterpart
  197. # (weblate).
  198. # In branch master of SearXNG (origin) check for meaningful changes in
  199. # folder 'searx/translations', commit changes on branch 'translations' and
  200. # at least, pull updated branches on Weblate's counterpart (weblate).
  201. # 1. Create git worktree ${TRANSLATIONS_WORKTREE} and checkout branch
  202. # 'translations' from remote 'weblate'.
  203. # 2. Stop if there is no meaningful change in the 'master' branch (origin),
  204. # compared to the 'translations' branch (weblate), otherwise ...
  205. # 3. Update 'translations' branch of SearXNG (origin) with last additions
  206. # from Weblate.
  207. # 5. Notify Weblate to pull updated 'master' & 'translations' branch.
  208. local messages_pot diff_messages_pot last_commit_hash last_commit_detail \
  209. exitcode
  210. messages_pot="${TRANSLATIONS_WORKTREE}/searx/translations/messages.pot"
  211. ( set -e
  212. pyenv.activate
  213. # get translations branch in git worktree (TRANSLATIONS_WORKTREE)
  214. weblate.translations.worktree
  215. # update messages.pot in the master branch
  216. build_msg BABEL 'extract messages from source files and generate POT file'
  217. pybabel extract -F babel.cfg \
  218. -o "${messages_pot}" \
  219. "searx/"
  220. # stop if there is no meaningful change in the master branch
  221. diff_messages_pot=$(cd "${TRANSLATIONS_WORKTREE}";\
  222. git diff -- "searx/translations/messages.pot")
  223. if ! echo "$diff_messages_pot" | grep -qE "[\+\-](msgid|msgstr)"; then
  224. build_msg BABEL 'no changes detected, exiting'
  225. return 42
  226. fi
  227. return 0
  228. )
  229. exitcode=$?
  230. if [ "$exitcode" -eq 42 ]; then
  231. return 0
  232. fi
  233. if [ "$exitcode" -gt 0 ]; then
  234. return $exitcode
  235. fi
  236. (
  237. set -e
  238. pyenv.activate
  239. # lock change on weblate
  240. # weblate may add commit(s) since the call to "weblate.translations.worktree".
  241. # this is not a problem because after this line, "weblate.to.translations"
  242. # calls again "weblate.translations.worktree" which calls "git pull"
  243. wlc lock
  244. # save messages.pot in the translations branch for later
  245. pushd "${TRANSLATIONS_WORKTREE}"
  246. git stash push
  247. popd
  248. # merge weblate commits into the translations branch
  249. weblate.to.translations
  250. # restore messages.pot in the translations branch
  251. pushd "${TRANSLATIONS_WORKTREE}"
  252. git stash pop
  253. popd
  254. # update messages.po files in the master branch
  255. build_msg BABEL 'update existing message catalogs from POT file'
  256. pybabel update -N \
  257. -i "${messages_pot}" \
  258. -d "${TRANSLATIONS_WORKTREE}/searx/translations"
  259. # git add/commit/push
  260. last_commit_hash=$(git log -n1 --pretty=format:'%h')
  261. last_commit_detail=$(git log -n1 --pretty=format:'%h - %as - %aN <%ae>' "${last_commit_hash}")
  262. pushd "${TRANSLATIONS_WORKTREE}"
  263. git add searx/translations
  264. git commit \
  265. -m "[translations] update messages.pot and messages.po files" \
  266. -m "From ${last_commit_detail}"
  267. git push
  268. popd
  269. # notify weblate to pull updated master & translations branch
  270. wlc pull
  271. )
  272. exitcode=$?
  273. ( # make sure to always unlock weblate
  274. set -e
  275. pyenv.activate
  276. wlc unlock
  277. )
  278. dump_return $exitcode
  279. }
  280. data.all() {
  281. ( set -e
  282. pyenv.activate
  283. data.languages
  284. data.useragents
  285. data.osm_keys_tags
  286. build_msg DATA "update searx/data/ahmia_blacklist.txt"
  287. python searxng_extra/update/update_ahmia_blacklist.py
  288. build_msg DATA "update searx/data/wikidata_units.json"
  289. python searxng_extra/update/update_wikidata_units.py
  290. build_msg DATA "update searx/data/currencies.json"
  291. python searxng_extra/update/update_currencies.py
  292. )
  293. }
  294. data.languages() {
  295. ( set -e
  296. pyenv.activate
  297. build_msg ENGINES "fetch languages .."
  298. python searxng_extra/update/update_languages.py
  299. build_msg ENGINES "update update searx/languages.py"
  300. build_msg DATA "update searx/data/engines_languages.json"
  301. )
  302. dump_return $?
  303. }
  304. data.useragents() {
  305. build_msg DATA "update searx/data/useragents.json"
  306. pyenv.cmd python searxng_extra/update/update_firefox_version.py
  307. dump_return $?
  308. }
  309. data.osm_keys_tags() {
  310. build_msg DATA "update searx/data/osm_keys_tags.json"
  311. pyenv.cmd python searxng_extra/update/update_osm_keys_tags.py
  312. dump_return $?
  313. }
  314. docs.prebuild() {
  315. build_msg DOCS "build ${DOCS_BUILD}/includes"
  316. (
  317. set -e
  318. [ "$VERBOSE" = "1" ] && set -x
  319. mkdir -p "${DOCS_BUILD}/includes"
  320. ./utils/searx.sh doc | cat > "${DOCS_BUILD}/includes/searx.rst"
  321. ./utils/filtron.sh doc | cat > "${DOCS_BUILD}/includes/filtron.rst"
  322. ./utils/morty.sh doc | cat > "${DOCS_BUILD}/includes/morty.rst"
  323. )
  324. dump_return $?
  325. }
  326. docker.push() {
  327. docker.build push
  328. }
  329. docker.buildx() {
  330. docker.build buildx
  331. }
  332. # shellcheck disable=SC2119
  333. docker.build() {
  334. pyenv.install
  335. local SEARXNG_GIT_VERSION
  336. local VERSION_GITCOMMIT
  337. local GITHUB_USER
  338. local SEARXNG_IMAGE_NAME
  339. local BUILD
  340. build_msg DOCKER build
  341. # run installation in a subprocess and activate pyenv
  342. # See https://www.shellcheck.net/wiki/SC1001 and others ..
  343. # shellcheck disable=SC2031,SC2230,SC2002,SC2236,SC2143,SC1001
  344. ( set -e
  345. pyenv.activate
  346. # Check if it is a git repository
  347. if [ ! -d .git ]; then
  348. die 1 "This is not Git repository"
  349. fi
  350. if [ ! -x "$(which git)" ]; then
  351. die 1 "git is not installed"
  352. fi
  353. if ! git remote get-url origin 2> /dev/null; then
  354. die 1 "there is no remote origin"
  355. fi
  356. # This is a git repository
  357. git update-index -q --refresh
  358. python -m searx.version freeze
  359. eval "$(python -m searx.version)"
  360. # Get the last git commit id
  361. VERSION_GITCOMMIT=$(echo "$VERSION_STRING" | cut -d- -f3)
  362. build_msg DOCKER "Last commit : $VERSION_GITCOMMIT"
  363. # define the docker image name
  364. GITHUB_USER=$(echo "${GIT_URL}" | sed 's/.*github\.com\/\([^\/]*\).*/\1/')
  365. SEARXNG_IMAGE_NAME="${SEARXNG_IMAGE_NAME:-${GITHUB_USER:-searxng}/searxng}"
  366. BUILD="build"
  367. if [ "$1" = "buildx" ]; then
  368. # buildx includes the push option
  369. CACHE_TAG="${SEARXNG_IMAGE_NAME}:latest-build-cache"
  370. BUILD="buildx build --platform linux/amd64,linux/arm64,linux/arm/v7 --push --cache-from=type=registry,ref=$CACHE_TAG --cache-to=type=registry,ref=$CACHE_TAG,mode=max"
  371. shift
  372. fi
  373. build_msg DOCKER "Build command: ${BUILD}"
  374. # build Docker image
  375. build_msg DOCKER "Building image ${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
  376. # shellcheck disable=SC2086
  377. docker $BUILD \
  378. --build-arg BASE_IMAGE="${DEPENDENCIES_IMAGE_NAME}" \
  379. --build-arg GIT_URL="${GIT_URL}" \
  380. --build-arg SEARXNG_GIT_VERSION="${VERSION_STRING}" \
  381. --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \
  382. --build-arg LABEL_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \
  383. --build-arg LABEL_VCS_REF="$(git rev-parse HEAD)" \
  384. --build-arg LABEL_VCS_URL="${GIT_URL}" \
  385. --build-arg TIMESTAMP_SETTINGS="$(git log -1 --format="%cd" --date=unix -- searx/settings.yml)" \
  386. --build-arg TIMESTAMP_UWSGI="$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini)" \
  387. -t "${SEARXNG_IMAGE_NAME}:latest" -t "${SEARXNG_IMAGE_NAME}:${VERSION_STRING}" .
  388. if [ "$1" = "push" ]; then
  389. docker push "${SEARXNG_IMAGE_NAME}:latest"
  390. docker push "${SEARXNG_IMAGE_NAME}:${SEARXNG_GIT_VERSION}"
  391. fi
  392. )
  393. dump_return $?
  394. }
  395. # shellcheck disable=SC2119
  396. gecko.driver() {
  397. pyenv.install
  398. build_msg INSTALL "gecko.driver"
  399. # run installation in a subprocess and activate pyenv
  400. ( set -e
  401. pyenv.activate
  402. # TODO : check the current geckodriver version
  403. geckodriver -V > /dev/null 2>&1 || NOTFOUND=1
  404. set +e
  405. if [ -z "$NOTFOUND" ]; then
  406. build_msg INSTALL "geckodriver already installed"
  407. return
  408. fi
  409. PLATFORM="$(python3 -c 'import platform; print(platform.system().lower(), platform.architecture()[0])')"
  410. case "$PLATFORM" in
  411. "linux 32bit" | "linux2 32bit") ARCH="linux32";;
  412. "linux 64bit" | "linux2 64bit") ARCH="linux64";;
  413. "windows 32 bit") ARCH="win32";;
  414. "windows 64 bit") ARCH="win64";;
  415. "mac 64bit") ARCH="macos";;
  416. esac
  417. GECKODRIVER_URL="https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-$ARCH.tar.gz";
  418. build_msg GECKO "Installing ${PY_ENV_BIN}/geckodriver from $GECKODRIVER_URL"
  419. FILE="$(mktemp)"
  420. wget -qO "$FILE" -- "$GECKODRIVER_URL" && tar xz -C "${PY_ENV_BIN}" -f "$FILE" geckodriver
  421. rm -- "$FILE"
  422. chmod 755 -- "${PY_ENV_BIN}/geckodriver"
  423. )
  424. dump_return $?
  425. }
  426. node.env() {
  427. if ! required_commands npm; then
  428. info_msg "to install build tools use::"
  429. info_msg " sudo -H ./utils/searx.sh install buildhost"
  430. die 1 "install needed build tools first"
  431. fi
  432. ( set -e
  433. build_msg INSTALL "searx/static/themes/oscar/package.json"
  434. npm --prefix searx/static/themes/oscar install
  435. build_msg INSTALL "searx/static/themes/simple/package.json"
  436. npm --prefix searx/static/themes/simple install
  437. )
  438. dump_return $?
  439. }
  440. node.clean() {
  441. if ! required_commands npm 2>/dev/null; then
  442. build_msg CLEAN "npm is not installed / ignore npm dependencies"
  443. return 0
  444. fi
  445. build_msg CLEAN "locally installed npm dependencies"
  446. ( set -e
  447. npm --prefix searx/static/themes/oscar run clean
  448. npm --prefix searx/static/themes/simple run clean
  449. )
  450. dump_return $?
  451. }
  452. pygments.less() {
  453. build_msg PYGMENTS "searxng_extra/update/update_pygments.py"
  454. if ! pyenv.cmd python searxng_extra/update/update_pygments.py; then
  455. build_msg PYGMENTS "building LESS files for pygments failed"
  456. return 1
  457. fi
  458. return 0
  459. }
  460. py.build() {
  461. build_msg BUILD "python package ${PYDIST}"
  462. pyenv.cmd python setup.py \
  463. sdist -d "${PYDIST}" \
  464. bdist_wheel --bdist-dir "${PYBUILD}" -d "${PYDIST}"
  465. }
  466. py.clean() {
  467. build_msg CLEAN pyenv
  468. ( set -e
  469. pyenv.drop
  470. [ "$VERBOSE" = "1" ] && set -x
  471. rm -rf "${PYDIST}" "${PYBUILD}" "${PY_ENV}" ./.tox ./*.egg-info
  472. find . -name '*.pyc' -exec rm -f {} +
  473. find . -name '*.pyo' -exec rm -f {} +
  474. find . -name __pycache__ -exec rm -rf {} +
  475. )
  476. }
  477. pyenv.check() {
  478. cat <<EOF
  479. import yaml
  480. print('import yaml --> OK')
  481. EOF
  482. }
  483. pyenv.install() {
  484. if ! pyenv.OK; then
  485. py.clean > /dev/null
  486. fi
  487. if pyenv.install.OK > /dev/null; then
  488. return 0
  489. fi
  490. ( set -e
  491. pyenv
  492. build_msg PYENV "[install] pip install -e 'searx${PY_SETUP_EXTRAS}'"
  493. "${PY_ENV_BIN}/python" -m pip install -e ".${PY_SETUP_EXTRAS}"
  494. buildenv
  495. )
  496. local exit_val=$?
  497. if [ ! $exit_val -eq 0 ]; then
  498. die 42 "error while pip install (${PY_ENV_BIN})"
  499. fi
  500. }
  501. pyenv.uninstall() {
  502. build_msg PYENV "[pyenv.uninstall] uninstall packages: ${PYOBJECTS}"
  503. pyenv.cmd python setup.py develop --uninstall 2>&1 \
  504. | prefix_stdout "${_Blue}PYENV ${_creset}[pyenv.uninstall] "
  505. }
  506. pypi.upload() {
  507. py.clean
  508. py.build
  509. # https://github.com/pypa/twine
  510. pyenv.cmd twine upload "${PYDIST}"/*
  511. }
  512. pypi.upload.test() {
  513. py.clean
  514. py.build
  515. pyenv.cmd twine upload -r testpypi "${PYDIST}"/*
  516. }
  517. test.yamllint() {
  518. build_msg TEST "[yamllint] \$YAMLLINT_FILES"
  519. pyenv.cmd yamllint --format parsable "${YAMLLINT_FILES[@]}"
  520. }
  521. test.pylint() {
  522. # shellcheck disable=SC2086
  523. ( set -e
  524. build_msg TEST "[pylint] \$PYLINT_FILES"
  525. pyenv.activate
  526. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  527. --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
  528. "${PYLINT_FILES[@]}"
  529. build_msg TEST "[pylint] searx/engines"
  530. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  531. --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
  532. --additional-builtins="${PYLINT_ADDITIONAL_BUILTINS_FOR_ENGINES}" \
  533. searx/engines
  534. build_msg TEST "[pylint] searx tests"
  535. python ${PYLINT_OPTIONS} ${PYLINT_VERBOSE} \
  536. --disable="${PYLINT_SEARXNG_DISABLE_OPTION}" \
  537. --ignore=searx/engines \
  538. searx tests
  539. )
  540. dump_return $?
  541. }
  542. test.pep8() {
  543. build_msg TEST 'pycodestyle (formerly pep8)'
  544. local _exclude=""
  545. printf -v _exclude '%s, ' "${PYLINT_FILES[@]}"
  546. pyenv.cmd pycodestyle \
  547. --exclude="searx/static, searx/languages.py, $_exclude " \
  548. --max-line-length=120 \
  549. --ignore "E117,E252,E402,E722,E741,W503,W504,W605" \
  550. searx tests
  551. dump_return $?
  552. }
  553. test.unit() {
  554. build_msg TEST 'tests/unit'
  555. pyenv.cmd python -m nose2 -s tests/unit
  556. dump_return $?
  557. }
  558. test.coverage() {
  559. build_msg TEST 'unit test coverage'
  560. ( set -e
  561. pyenv.activate
  562. python -m nose2 -C --log-capture --with-coverage --coverage searx -s tests/unit
  563. coverage report
  564. coverage html
  565. )
  566. dump_return $?
  567. }
  568. test.robot() {
  569. build_msg TEST 'robot'
  570. gecko.driver
  571. PYTHONPATH=. pyenv.cmd python -m tests.robot
  572. dump_return $?
  573. }
  574. test.clean() {
  575. build_msg CLEAN "test stuff"
  576. rm -rf geckodriver.log .coverage coverage/
  577. dump_return $?
  578. }
  579. themes.all() {
  580. ( set -e
  581. pygments.less
  582. node.env
  583. themes.oscar
  584. themes.simple
  585. )
  586. dump_return $?
  587. }
  588. themes.oscar() {
  589. build_msg GRUNT "theme: oscar"
  590. npm --prefix searx/static/themes/oscar run build
  591. dump_return $?
  592. }
  593. themes.simple() {
  594. local src="searx/static/themes/simple/src"
  595. local static="searx/static/themes/simple"
  596. ( set -e
  597. convert_if_newer "$src/svg/searxng-wordmark.svg" "$static/img/favicon.png" \
  598. -transparent white -resize 64x64
  599. build_msg GRUNT "theme: simple"
  600. npm --prefix searx/static/themes/simple run build
  601. )
  602. dump_return $?
  603. }
  604. convert_if_newer() {
  605. # usage: convert_if_newer <origfile> <outfile> [<options>, ...]
  606. #
  607. # convert_if_newer "path/to/origin.svg" "path/to/converted.png" -resize 100x100
  608. #
  609. # Run's ImageMagik' convert comand to generate <outfile> from <origfile>, if
  610. # <origfile> is newer than <outfile>. The command line is to convert is::
  611. #
  612. # convert <origfile> [<options>, ...] <outfile>
  613. local src_file="$1" && shift
  614. local dst_file="$1" && shift
  615. if [[ "${src_file}" -nt "${dst_file}" ]]; then
  616. if ! required_commands convert; then
  617. info_msg "to install build tools use::"
  618. info_msg " sudo -H ./utils/searx.sh install buildhost"
  619. die 1 "install needed build tools first"
  620. fi
  621. build_msg CONVERT "${src_file}" "$@" "${dst_file}"
  622. convert "${src_file}" "$@" "${dst_file}"
  623. else
  624. build_msg CONVERT "${dst_file} (up-to-date)"
  625. fi
  626. }
  627. PYLINT_FILES=()
  628. while IFS= read -r line; do
  629. PYLINT_FILES+=("$line")
  630. done <<< "$(pylint.FILES)"
  631. # shellcheck disable=SC2119
  632. main() {
  633. local _type
  634. local cmd="$1"; shift
  635. if [ "$cmd" == "" ]; then
  636. help
  637. err_msg "missing command"
  638. return 42
  639. fi
  640. case "$cmd" in
  641. --getenv) var="$1"; echo "${!var}";;
  642. --help) help;;
  643. --*)
  644. help
  645. err_msg "unknown option $cmd"
  646. return 42
  647. ;;
  648. *)
  649. _type="$(type -t "$cmd")"
  650. if [ "$_type" != 'function' ]; then
  651. err_msg "unknown command: $cmd / use --help"
  652. return 42
  653. else
  654. "$cmd" "$@"
  655. fi
  656. ;;
  657. esac
  658. }
  659. main "$@"