From 6c429e6dd1836768a4c73f46663f8a70a1979bb0 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Sat, 5 Dec 2020 17:01:21 -0500 Subject: [PATCH] Allow setting site alts using environment vars (#155) * Add ability to configure site alts w/ env vars Site alternatives (i.e. twitter.com -> nitter.net) can now be configured using environment variables: WHOOGLE_ALT_TW='nitter.net' # twitter alt WHOOGLE_ALT_YT='invidio.us' # youtube alt WHOOGLE_ALT_IG='bibliogram.art/u' # instagram alt Updated testing to confirm results have been modified. * Add site alt vars to docker settings and readme --- Dockerfile | 7 +++++++ README.md | 11 ++++++++++- app.json | 15 +++++++++++++++ app/utils/filter_utils.py | 7 ++++--- docker-compose.yml | 6 ++++++ test/conftest.py | 11 +++++++++++ test/test_results.py | 14 ++++++++++++++ test/test_routes.py | 13 ++----------- 8 files changed, 69 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index bf038ba..96e5f81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,6 +36,13 @@ ENV HTTPS_ONLY=$use_https ARG whoogle_port=5000 ENV EXPOSE_PORT=$whoogle_port +ARG twitter_alt='' +ENV WHOOGLE_ALT_TW=$twitter_alt +ARG youtube_alt='' +ENV WHOOGLE_ALT_YT=$youtube_alt +ARG instagram_alt='' +ENV WHOOGLE_ALT_YT=$instagram_alt + COPY . . EXPOSE $EXPOSE_PORT diff --git a/README.md b/README.md index 217302f..fff3ada 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,12 @@ Description=Whoogle #Environment=WHOOGLE_PROXY_PASS= #Environment=WHOOGLE_PROXY_TYPE= +# Site alternative configurations, uncomment to enable +# Note: If not set, the feature will still be available +# with default values. +#Environment=WHOOGLE_ALT_TW=nitter.net +#Environment=WHOOGLE_ALT_YT=invidious.snopyta.org +#Environment=WHOOGLE_ALT_IG=bibliogram.art/u Type=simple User=root WorkingDirectory= @@ -229,7 +235,10 @@ There are a few optional environment variables available for customizing a Whoog | WHOOGLE_PROXY_TYPE | The type of the proxy server. Can be "socks5", "socks4", or "http". | | WHOOGLE_PROXY_LOC | The location of the proxy server (host or ip). | | EXPOSE_PORT | The port where Whoogle will be exposed. | -| HTTPS_ONLY | Enforce HTTPS. (See [here](https://github.com/benbusby/whoogle-search#https-enforcement)) | +| HTTPS_ONLY | Enforce HTTPS. (See [here](https://github.com/benbusby/whoogle-search#https-enforcement)) | +| WHOOGLE_ALT_TW | The twitter.com alternative to use when site alternatives are enabled in the config. | +| WHOOGLE_ALT_YT | The youtube.com alternative to use when site alternatives are enabled in the config. | +| WHOOGLE_ALT_IG | The instagram.com alternative to use when site alternatives are enabled in the config. | ## Usage Same as most search engines, with the exception of filtering by time range. diff --git a/app.json b/app.json index 7482c98..18846d9 100644 --- a/app.json +++ b/app.json @@ -44,6 +44,21 @@ "description": "The location of the proxy server (host or ip). Leave empty to disable.", "value": "", "required": false + }, + "WHOOGLE_ALT_TW": { + "description": "The site to use as a replacement for twitter.com when site alternatives are enabled in the config." + "value": "", + "required": false + }, + "WHOOGLE_ALT_YT": { + "description": "The site to use as a replacement for youtube.com when site alternatives are enabled in the config." + "value": "", + "required": false + }, + "WHOOGLE_ALT_IG": { + "description": "The site to use as a replacement for instagram.com when site alternatives are enabled in the config." + "value": "", + "required": false } } } diff --git a/app/utils/filter_utils.py b/app/utils/filter_utils.py index ed6804c..9b2310f 100644 --- a/app/utils/filter_utils.py +++ b/app/utils/filter_utils.py @@ -1,4 +1,5 @@ from bs4 import BeautifulSoup +import os import urllib.parse as urlparse from urllib.parse import parse_qs @@ -17,9 +18,9 @@ BLACKLIST = [ ] SITE_ALTS = { - 'twitter.com': 'nitter.net', - 'youtube.com': 'invidious.snopyta.org', - 'instagram.com': 'bibliogram.art/u' + 'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'nitter.net'), + 'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'invidious.snopyta.org'), + 'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'bibliogram.art/u') } diff --git a/docker-compose.yml b/docker-compose.yml index 0d58aac..75ad171 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,6 +13,12 @@ services: #- WHOOGLE_PROXY_PASS= #- WHOOGLE_PROXY_TYPE= + # Site alternative configurations, uncomment to enable + # Note: If not set, the feature will still be available + # with default values. + #- WHOOGLE_ALT_TW=nitter.net + #- WHOOGLE_ALT_YT=invidious.snopyta.org + #- WHOOGLE_ALT_IG=bibliogram.art/u ports: - 5000:5000 restart: unless-stopped diff --git a/test/conftest.py b/test/conftest.py index 7a15f00..0b29a0f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,17 @@ from app import app +from app.models.config import Config from app.utils.session_utils import generate_user_keys import pytest +import random + +demo_config = { + 'near': random.choice(['Seattle', 'New York', 'San Francisco']), + 'dark_mode': str(random.getrandbits(1)), + 'nojs': str(random.getrandbits(1)), + 'lang_interface': random.choice(Config.LANGUAGES)['value'], + 'lang_search': random.choice(Config.LANGUAGES)['value'], + 'ctry': random.choice(Config.COUNTRIES)['value'] +} @pytest.fixture diff --git a/test/test_results.py b/test/test_results.py index a7aa771..d9f8a6a 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -3,6 +3,8 @@ from app.filter import Filter from app.utils.session_utils import generate_user_keys from datetime import datetime from dateutil.parser import * +import json +import os def get_search_results(data): @@ -43,6 +45,18 @@ def test_post_results(client): assert len(get_search_results(rv.data)) <= 15 +def test_site_alts(client): + rv = client.post('/search', data=dict(q='twitter official account')) + assert b'twitter.com/Twitter' in rv.data + + client.post('/config', data=dict(alts=True)) + assert json.loads(client.get('/config').data)['alts'] + + rv = client.post('/search', data=dict(q='twitter official account')) + assert b'twitter.com/Twitter' not in rv.data + assert b'nitter.net/Twitter' in rv.data + + def test_recent_results(client): times = { 'past year': 365, diff --git a/test/test_routes.py b/test/test_routes.py index ff9e0a2..e3ba084 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -1,15 +1,6 @@ -from app.models.config import Config import json -import random -demo_config = { - 'near': random.choice(['Seattle', 'New York', 'San Francisco']), - 'dark_mode': str(random.getrandbits(1)), - 'nojs': str(random.getrandbits(1)), - 'lang_interface': random.choice(Config.LANGUAGES)['value'], - 'lang_search': random.choice(Config.LANGUAGES)['value'], - 'ctry': random.choice(Config.COUNTRIES)['value'] -} +from test.conftest import demo_config def test_main(client): @@ -58,4 +49,4 @@ def test_config(client): def test_opensearch(client): rv = client.get('/opensearch.xml') assert rv._status_code == 200 - assert 'Whoogle' in str(rv.data) + assert 'Whoogle' in str(rv.data)