From fbe40001d35ea5cf74c92f35d76c642a480a5c9f Mon Sep 17 00:00:00 2001 From: Dalf Date: Sat, 29 Jun 2019 11:59:13 +0200 Subject: [PATCH 1/3] Update Docker image See #1561 , use uwsgi and Alpine Linux Volume: /var/log/uwsgi contains error log for 2 days (file uwsgi.log) /etc/searx contains the settings.yml and uwsgi.ini files. The docker image creates them if they don't exist. The two files can be modified after the first run. See below. Environement variables: MORTY_URL : external URL of Morty MORTY_KEY : base64 encoded key BASE_URL : external URL of Searx BIND_ADDRESS : internal HTTP port to listen to Labels : org.label-schema.schema.* Parameters: -h : display this help -d : will update the settings and quit immediately (settings.yml and uwsgi.ini) -f : always update the settings (previous version saved with suffix .old). without this parameter, the new settings are copied with suffix .new When the Docker image contains newer settings: - without -f parameter: the new versions are copied to /etc/searx/settings.yml.new and /etc/searx/uwsgi.ini.new. - with -f parameter: the old versions are renamed with .old suffix. The new version replaces /etc/searx/settings.yml and /etc/searx/uwsgi.ini Build using "./manage.sh docker_build", add "push" as parameter also push the Docker image. The script requires a git repository to work (it makes sure that the last git tag matches searx/version.py) "git describe" is used to create a meaningful version. Example : 0.15.0-90-49c5bcb4-dirty (dirty means that the docker image was made with uncommited changes). Use "docker inspect -f {{.Config.Labels.version}} searx" to get the version of an existing image. .dockerignore based on .gitignore .travis.yml: include docker stage --- .dockerignore | 41 ++++++++++ .gitignore | 1 + .travis.yml | 33 ++++++-- Dockerfile | 104 +++++++++++++++---------- dockerfiles/docker-entrypoint.sh | 128 +++++++++++++++++++++++++++++++ dockerfiles/uwsgi.ini | 33 ++++++++ manage.sh | 69 +++++++++++++++++ 7 files changed, 362 insertions(+), 47 deletions(-) create mode 100644 .dockerignore create mode 100755 dockerfiles/docker-entrypoint.sh create mode 100644 dockerfiles/uwsgi.ini diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..f4d03376c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,41 @@ +*~ +*/*~ +*/*/*~ +*/*/*/*~ +*/*/*/*/*~ + +# Git +.git +.gitignore + +# CI +.codeclimate.yml +.travis.yml +.taskcluster.yml + +# Byte-compiled / optimized / DLL files +__pycache__/ +*/__pycache__/ +*/*/__pycache__/ +*/*/*/__pycache__/ +*.py[cod] +*/*.py[cod] +*/*/*.py[cod] +*/*/*/*.py[cod] + +# to sync with .gitignore +.coverage +coverage/ +.installed.cfg +engines.cfg +env +searx-ve +robot_log.html +robot_output.xml +robot_report.html +test_basic/ +setup.cfg + +node_modules/ + +.tx/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3da4d0386..db20da83e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +# to sync with .dockerignore .coverage coverage/ .installed.cfg diff --git a/.travis.yml b/.travis.yml index 08bcfaadd..04654ac23 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ +language: python sudo: false cache: - pip - npm - directories: - $HOME/.cache/pip + addons: firefox: "latest" -language: python -python: - - "2.7" - - "3.5" - - "3.6" + before_install: - "export DISPLAY=:99.0" - "sh -e /etc/init.d/xvfb start" @@ -27,6 +25,31 @@ script: after_success: - ./manage.sh py_test_coverage - codecov + +stages: + - test + - name: docker + if: branch = master AND type != pull_request AND env(DOCKER_USERNAME) IS present + +jobs: + include: + - python: "2.7" + - python: "3.5" + - python: "3.6" + - stage: docker + python: "3.6" + git: + depth: false + services: + - docker + addons: [] + before_install: true + install: true + script: + - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + - ./manage.sh docker_build push + after_success: true + notifications: irc: channels: diff --git a/Dockerfile b/Dockerfile index 95e21813f..03c4b76a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,77 @@ -FROM alpine:3.8 -LABEL maintainer="searx " -LABEL description="A privacy-respecting, hackable metasearch engine." +FROM alpine:3.10 + +ARG VERSION_GITCOMMIT=unknow +ARG SEARX_GIT_VERSION=unknow + +ARG SEARX_GID=1000 +ARG SEARX_UID=1000 + +ARG TIMESTAMP_SETTINGS=0 +ARG TIMESTAMP_UWSGI=0 +ARG LABEL_VCS_REF= +ARG LABEL_VCS_URL= + +ENV BASE_URL= \ + MORTY_KEY= \ + MORTY_URL= +EXPOSE 8080 +VOLUME /etc/searx +VOLUME /var/log/uwsgi -ENV BASE_URL=False IMAGE_PROXY=False HTTP_PROXY_URL= HTTPS_PROXY_URL= -EXPOSE 8888 WORKDIR /usr/local/searx -CMD ["/sbin/tini","--","/usr/local/searx/run.sh"] -RUN adduser -D -h /usr/local/searx -s /bin/sh searx searx \ - && echo '#!/bin/sh' >> run.sh \ - && echo 'sed -i "s|base_url : False|base_url : $BASE_URL|g" searx/settings.yml' >> run.sh \ - && echo 'sed -i "s/image_proxy : False/image_proxy : $IMAGE_PROXY/g" searx/settings.yml' >> run.sh \ - && echo 'sed -i "s/ultrasecretkey/`openssl rand -hex 16`/g" searx/settings.yml' >> run.sh \ - && echo 'if [ -n "$HTTP_PROXY_URL" ] || [ -n "$HTTPS_PROXY_URL" ]; then' >> run.sh \ - && echo ' sed -i "s~^# proxies :~ proxies:\\n http: ${HTTP_PROXY_URL}\\n https: ${HTTPS_PROXY_URL}\\n~" searx/settings.yml' >> run.sh \ - && echo 'fi' >> run.sh \ - && echo 'python searx/webapp.py' >> run.sh \ - && chmod +x run.sh +RUN addgroup -g ${SEARX_GID} searx && \ + adduser -u ${SEARX_UID} -D -h /usr/local/searx -s /bin/sh -G searx searx COPY requirements.txt ./requirements.txt -RUN echo "@commuedge http://nl.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories \ - && apk -U add \ +RUN apk -U upgrade \ + && apk add -t build-dependencies \ build-base \ - python \ - python-dev \ - py-pip \ + py3-setuptools \ + python3-dev \ + libffi-dev \ + libxslt-dev \ + libxml2-dev \ + openssl-dev \ + tar \ + git \ + && apk add \ + ca-certificates \ + su-exec \ + python3 \ libxml2 \ - libxml2-dev \ libxslt \ - libxslt-dev \ - libffi-dev \ openssl \ - openssl-dev \ - ca-certificates \ - tini@commuedge \ - && pip install --upgrade pip \ - && pip install --no-cache -r requirements.txt \ - && apk del \ - build-base \ - python-dev \ - libffi-dev \ - openssl-dev \ - libxslt-dev \ - libxml2-dev \ - openssl-dev \ - ca-certificates \ + tini \ + uwsgi \ + uwsgi-python3 \ + && pip3 install --upgrade pip \ + && pip3 install --no-cache -r requirements.txt \ + && apk del build-dependencies \ && rm -f /var/cache/apk/* -COPY . . +COPY --chown=searx:searx . . -RUN chown -R searx:searx * +RUN su searx -c "/usr/bin/python3 -m compileall -q searx"; \ + touch -c --date=@${TIMESTAMP_SETTINGS} searx/settings.yml; \ + touch -c --date=@${TIMESTAMP_UWSGI} dockerfiles/uwsgi.ini; \ + if [ ! -z $VERSION_GITCOMMIT ]; then\ + echo "VERSION_STRING = VERSION_STRING + \"-$VERSION_GITCOMMIT\"" >> /usr/local/searx/searx/version.py; \ + fi -USER searx +ENTRYPOINT ["/sbin/tini","--","/usr/local/searx/dockerfiles/docker-entrypoint.sh"] -RUN sed -i "s/127.0.0.1/0.0.0.0/g" searx/settings.yml +# Keep this argument at the end since it change each time +ARG LABEL_DATE= +LABEL maintainer="searx " \ + description="A privacy-respecting, hackable metasearch engine." \ + version="${SEARX_GIT_VERSION}" \ + org.label-schema.schema-version="1.0" \ + org.label-schema.name="searx" \ + org.label-schema.schema-version="${SEARX_GIT_VERSION}" \ + org.label-schema.url="${LABEL_VCS_URL}" \ + org.label-schema.vcs-ref=${LABEL_VCS_REF} \ + org.label-schema.vcs-url=${LABEL_VCS_URL} \ + org.label-schema.build-date="${LABEL_DATE}" \ + org.label-schema.usage="https://github.com/searx/searx-docker" diff --git a/dockerfiles/docker-entrypoint.sh b/dockerfiles/docker-entrypoint.sh new file mode 100755 index 000000000..60e26fd94 --- /dev/null +++ b/dockerfiles/docker-entrypoint.sh @@ -0,0 +1,128 @@ +#!/bin/sh + +export SEARX_VERSION=$(su searx -c 'python3 -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)"') +printf 'searx version %s\n\n' "${SEARX_VERSION}" + +export UWSGI_SETTINGS_PATH=/etc/searx/uwsgi.ini +export SEARX_SETTINGS_PATH=/etc/searx/settings.yml + +if [ -z "${BIND_ADDRESS}" ]; then + export BIND_ADDRESS=":8080" +fi + +# Parse command line +FORCE_CONF_UPDATE=0 +DRY_RUN=0 +while getopts "fdh" option +do + case $option in + f) + FORCE_CONF_UPDATE=1 + ;; + d) + DRY_RUN=1 + ;; + h) + printf "Command line:\n\n" + printf " -h Display this help\n" + printf " -d Dry run to update the configuration files.\n" + printf " -f Always update on the configuration files (existing files are renamed with the .old suffix)\n" + printf " Without this option, new configuration files are copied with the .new suffix\n" + printf "\nEnvironment variables:\n\n" + printf " BASE_URL settings.yml : server.base_url\n" + printf " MORTY_URL settings.yml : result_proxy.url\n" + printf " MORTY_KEY settings.yml : result_proxy.key\n" + printf " BIND_ADDRESS where uwsgi will accept HTTP request (format : host:port)\n" + exit 0 + esac +done + +# helpers to update the configuration files +patch_uwsgi_settings() { + CONF="$1" + + # Nothing +} + +patch_searx_settings() { + CONF="$1" + + # Make sure that there is trailing slash at the end of BASE_URL + # see http://www.gnu.org/savannah-checkouts/gnu/bash/manual/bash.html#Shell-Parameter-Expansion + export BASE_URL="${BASE_URL%/}/" + + # update settings.yml + sed -i -e "s|base_url : False|base_url : ${BASE_URL}|g" \ + -e "s/ultrasecretkey/$(openssl rand -hex 32)/g" \ + "${CONF}" + + # Morty configuration + if [ ! -z "${MORTY_KEY}" -a ! -z "${MORTY_URL}" ]; then + sed -i -e "s/image_proxy : False/image_proxy : True/g" \ + "${CONF}" + cat >> "${CONF}" <<-EOF + +# Morty configuration +result_proxy: + url : ${MORTY_URL} + key : !!binary "${MORTY_KEY}" +EOF + fi +} + +update_conf() { + FORCE_CONF_UPDATE="$1" + CONF="$2" + NEW_CONF="${2}.new" + OLD_CONF="${2}.old" + REF_CONF="$3" + PATCH_REF_CONF="$4" + + if [ -f "${CONF}" ]; then + if [ "${REF_CONF}" -nt "${CONF}" ]; then + # There is a new version + if [ $FORCE_CONF_UPDATE ]; then + # Replace the current configuration + printf '⚠️ Automaticaly update %s to the new version\n' "${CONF}" + if [ ! -f "${OLD_CONF}" ]; then + printf 'The previous configuration is saved to %s\n' "${OLD_CONF}" + mv "${CONF}" "${OLD_CONF}" + fi + cp "${REF_CONF}" "${CONF}" + $PATCH_REF_CONF "${CONF}" + else + # Keep the current configuration + printf '⚠️ Check new version %s to make sure searx is working properly\n' "${NEW_CONF}" + cp "${REF_CONF}" "${NEW_CONF}" + $PATCH_REF_CONF "${NEW_CONF}" + fi + else + printf 'Use existing %s\n' "${CONF}" + fi + else + printf 'Create %s\n' "${CONF}" + cp "${REF_CONF}" "${CONF}" + $PATCH_REF_CONF "${CONF}" + fi +} + +# make sure there are uwsgi settings +update_conf "${FORCE_CONF_UPDATE}" "${UWSGI_SETTINGS_PATH}" "/usr/local/searx/dockerfiles/uwsgi.ini" "patch_uwsgi_settings" + +# make sure there are searx settings +update_conf "${FORCE_CONF_UPDATE}" "${SEARX_SETTINGS_PATH}" "/usr/local/searx/searx/settings.yml" "patch_searx_settings" + +# dry run (to update configuration files, then inspect them) +if [ $DRY_RUN -eq 1 ]; then + printf 'Dry run\n' + exit +fi + +# +touch /var/run/uwsgi-logrotate +chown -R searx:searx /var/log/uwsgi /var/run/uwsgi-logrotate +unset MORTY_KEY + +# Start uwsgi +printf 'Listen on %s\n' "${BIND_ADDRESS}" +exec su-exec searx:searx uwsgi --master --http-socket "${BIND_ADDRESS}" "${UWSGI_SETTINGS_PATH}" diff --git a/dockerfiles/uwsgi.ini b/dockerfiles/uwsgi.ini new file mode 100644 index 000000000..fa2fd6302 --- /dev/null +++ b/dockerfiles/uwsgi.ini @@ -0,0 +1,33 @@ +[uwsgi] +# Who will run the code +uid = searx +gid = searx + +# Number of workers (usually CPU count) +workers = 4 + +# The right granted on the created socket +chmod-socket = 666 + +# Plugin to use and interpretor config +single-interpreter = true +master = true +plugin = python3 +lazy-apps = true +enable-threads = true + +# Module to import +module = searx.webapp + +# Virtualenv and python path +pythonpath = /usr/local/searx/ +chdir = /usr/local/searx/searx/ + +# Disable logging for privacy +disable-logging=True + +# But keep errors for 2 days +touch-logrotate = /run/uwsgi-logrotate +unique-cron = 15 0 -1 -1 -1 { touch /run/uwsgi-logrotate } +log-backupname = /var/log/uwsgi/uwsgi.log.1 +logto = /var/log/uwsgi/uwsgi.log diff --git a/manage.sh b/manage.sh index c564827ac..7279eaf39 100755 --- a/manage.sh +++ b/manage.sh @@ -158,6 +158,74 @@ grunt_build() { grunt --gruntfile "$SEARX_DIR/static/themes/simple/gruntfile.js" } +docker_build() { + # Check if it is a git repository + if [ ! -d .git ]; then + echo "This is not Git repository" + exit 1 + fi + + if [ ! -x "$(which git)" ]; then + echo "git is not installed" + exit 1 + fi + + if [ ! git remote get-url origin 2> /dev/null ]; then + echo "there is no remote origin" + exit 1 + fi + + # This is a git repository + + # "git describe" to get the Docker version (for example : v0.15.0-89-g0585788e) + # awk to remove the "v" and the "g" + SEARX_GIT_VERSION=$(git describe --match "v[0-9]*\.[0-9]*\.[0-9]*" HEAD 2>/dev/null | awk -F'-' '{OFS="-"; $1=substr($1, 2); $3=substr($3, 2); print}') + + # add the suffix "-dirty" if the repository has uncommited change + git update-index -q --refresh + if [ ! -z "$(git diff-index --name-only HEAD --)" ]; then + SEARX_GIT_VERSION="${SEARX_GIT_VERSION}-dirty" + fi + + # Get the last git commit id, will be added to the Searx version (see Dockerfile) + VERSION_GITCOMMIT=$(echo $SEARX_GIT_VERSION | cut -d- -f2-4) + echo "Last commit : $VERSION_GITCOMMIT" + + # Check consistency between the git tag and the searx/version.py file + # /!\ HACK : parse Python file with bash /!\ + # otherwise it is not possible build the docker image without all Python dependencies ( version.py loads __init__.py ) + # SEARX_PYTHON_VERSION=$(python -c "import six; import searx.version; six.print_(searx.version.VERSION_STRING)") + SEARX_PYTHON_VERSION=$(cat searx/version.py | grep "\(VERSION_MAJOR\|VERSION_MINOR\|VERSION_BUILD\) =" | cut -d\= -f2 | sed -e 's/^[[:space:]]*//' | paste -sd "." -) + if [ $(echo "$SEARX_GIT_VERSION" | cut -d- -f1) != "$SEARX_PYTHON_VERSION" ]; then + echo "Inconsistency between the last git tag and the searx/version.py file" + echo "git tag: $SEARX_GIT_VERSION" + echo "searx/version.py: $SEARX_PYTHON_VERSION" + exit 1 + fi + + # define the docker image name + # /!\ HACK to get the user name /!\ + GITHUB_USER=$(git remote get-url origin | sed 's/.*github\.com\/\([^\/]*\).*/\1/') + SEARX_IMAGE_NAME="${GITHUB_USER:-searx}/searx" + + # build Docker image + echo "Building image ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION}" + sudo docker build \ + --build-arg SEARX_GIT_VERSION="${SEARX_GIT_VERSION}" \ + --build-arg VERSION_GITCOMMIT="${VERSION_GITCOMMIT}" \ + --build-arg LABEL_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \ + --build-arg LABEL_VCS_REF=$(git rev-parse HEAD) \ + --build-arg LABEL_VCS_URL=$(git remote get-url origin) \ + --build-arg TIMESTAMP_SETTINGS=$(git log -1 --format="%cd" --date=unix -- searx/settings.yml) \ + --build-arg TIMESTAMP_UWSGI=$(git log -1 --format="%cd" --date=unix -- dockerfiles/uwsgi.ini) \ + -t ${SEARX_IMAGE_NAME}:latest -t ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} . + + if [ "$1" = "push" ]; then + sudo docker push ${SEARX_IMAGE_NAME}:latest + sudo docker push ${SEARX_IMAGE_NAME}:${SEARX_GIT_VERSION} + fi +} + # # Help # @@ -182,6 +250,7 @@ Commands locales - Compile locales styles - Build less files grunt_build - Build files for themes + docker_build - Build Docker image Tests ----- From 609ac5795ab16ab8a80a89feb54a9e23d4dec725 Mon Sep 17 00:00:00 2001 From: Dalf Date: Fri, 12 Jul 2019 19:35:36 +0200 Subject: [PATCH 2/3] .travis.yml: robot tests: remove the xvfb dependency --- .travis.yml | 3 --- searx/testing.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 04654ac23..e37b65864 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,6 @@ cache: addons: firefox: "latest" -before_install: - - "export DISPLAY=:99.0" - - "sh -e /etc/init.d/xvfb start" install: - ./manage.sh install_geckodriver ~/drivers - export PATH=~/drivers:$PATH diff --git a/searx/testing.py b/searx/testing.py index 647c236fd..08a53e3f4 100644 --- a/searx/testing.py +++ b/searx/testing.py @@ -71,7 +71,7 @@ class SearxRobotLayer(): def run_robot_tests(tests): print('Running {0} tests'.format(len(tests))) for test in tests: - with Browser() as browser: + with Browser('firefox', headless=True) as browser: test(browser) From 45702b77ca4759043f3b87f24983039f21ba2c32 Mon Sep 17 00:00:00 2001 From: Dalf Date: Sat, 13 Jul 2019 07:57:10 +0200 Subject: [PATCH 3/3] embedded iframe (youtube, dailymotion, vimeo): use https --- searx/engines/dailymotion.py | 2 +- searx/engines/vimeo.py | 2 +- searx/engines/youtube_api.py | 2 +- searx/engines/youtube_noapi.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/searx/engines/dailymotion.py b/searx/engines/dailymotion.py index 06a9c41f3..069aceaa3 100644 --- a/searx/engines/dailymotion.py +++ b/searx/engines/dailymotion.py @@ -26,7 +26,7 @@ language_support = True # see http://www.dailymotion.com/doc/api/obj-video.html search_url = 'https://api.dailymotion.com/videos?fields=created_time,title,description,duration,url,thumbnail_360_url,id&sort=relevance&limit=5&page={pageno}&{query}' # noqa embedded_url = '' + 'data-src="https://www.dailymotion.com/embed/video/{videoid}" allowfullscreen>' supported_languages_url = 'https://api.dailymotion.com/languages' diff --git a/searx/engines/vimeo.py b/searx/engines/vimeo.py index 1408be8df..a92271019 100644 --- a/searx/engines/vimeo.py +++ b/searx/engines/vimeo.py @@ -24,7 +24,7 @@ paging = True base_url = 'https://vimeo.com/' search_url = base_url + '/search/page:{pageno}?{query}' -embedded_url = '' diff --git a/searx/engines/youtube_api.py b/searx/engines/youtube_api.py index 6de18aa2c..bc4c0d58e 100644 --- a/searx/engines/youtube_api.py +++ b/searx/engines/youtube_api.py @@ -23,7 +23,7 @@ base_url = 'https://www.googleapis.com/youtube/v3/search' search_url = base_url + '?part=snippet&{query}&maxResults=20&key={api_key}' embedded_url = '' base_youtube_url = 'https://www.youtube.com/watch?v=' diff --git a/searx/engines/youtube_noapi.py b/searx/engines/youtube_noapi.py index 3bf25932b..53a10bf35 100644 --- a/searx/engines/youtube_noapi.py +++ b/searx/engines/youtube_noapi.py @@ -30,7 +30,7 @@ time_range_dict = {'day': 'Ag', 'year': 'BQ'} embedded_url = '' base_youtube_url = 'https://www.youtube.com/watch?v='