Use farside.link for frontend alternatives in results (#560)

* Integrate Farside into Whoogle

When instances are ratelimited (when a captcha is returned instead of
the user's search results) the user can now hop to a new instance via
Farside, a new backend service that redirects users to working instances
of a particular frontend. In this case, it presents a user with a
Farside link to a new Whoogle (or Searx) instance instead, so that the
user can resume their search.

For the generated Farside->Whoogle link, the generated link includes the
user's current Whoogle configuration settings as URL params, to ensure a
more seamless transition between instances. This doesn't translate to
the Farside->Searx link, but potentially could with some changes.

* Expand conversion of config<->url params

Config settings can now be translated to and from URL params using a
predetermined set of "safe" keys (i.e. config settings that easily
translate to URL params).

* Allow jumping instances via Farside when ratelimited

When instances are ratelimited (when a captcha is returned instead of
the user's search results) the user can now hop to a new instance via
Farside, a new backend service that redirects users to working instances
of a particular frontend. In this case, it presents a user with a
Farside link to a new Whoogle (or Searx) instance instead, so that the
user can resume their search.

For the generated Farside->Whoogle link, the generated link includes the
user's current Whoogle configuration settings as URL params, to ensure a
more seamless transition between instances. This doesn't translate to
the Farside->Searx link, but potentially could with some changes.

Closes #554

Closes #559

Dockerfile

@@ -47,18 +47,18 @@ ENV HTTPS_ONLY=$use_https
 ARG whoogle_port=5000
 ENV EXPOSE_PORT=$whoogle_port
 
-ARG twitter_alt='nitter.net'
+ARG twitter_alt='farside.link/nitter'
 ENV WHOOGLE_ALT_TW=$twitter_alt
-ARG youtube_alt='invidious.snopyta.org'
+ARG youtube_alt='farside.link/invidious'
 ENV WHOOGLE_ALT_YT=$youtube_alt
-ARG instagram_alt='bibliogram.art/u'
+ARG instagram_alt='farside.link/bibliogram'
 ENV WHOOGLE_ALT_IG=$instagram_alt
-ARG reddit_alt='libredd.it'
+ARG reddit_alt='farside.link/libreddit'
 ENV WHOOGLE_ALT_RD=$reddit_alt
+ARG medium_alt='farside.link/scribe'
+ENV WHOOGLE_ALT_MD=$medium_alt
 ARG translate_alt='lingva.ml'
 ENV WHOOGLE_ALT_TL=$translate_alt
-ARG medium_alt='scribe.rip'
-ENV WHOOGLE_ALT_MD=$medium_alt
 
 WORKDIR /whoogle
 

README.md

@@ -197,12 +197,12 @@ Description=Whoogle
 # Site alternative configurations, uncomment to enable
 # Note: If not set, the feature will still be available
 # with default values.
-#Environment=WHOOGLE_ALT_TW=nitter.net
+#Environment=WHOOGLE_ALT_TW=farside.link/nitter
-#Environment=WHOOGLE_ALT_YT=invidious.snopyta.org
+#Environment=WHOOGLE_ALT_YT=farside.link/invidious
-#Environment=WHOOGLE_ALT_IG=bibliogram.art/u
+#Environment=WHOOGLE_ALT_IG=farside.link/bibliogram/u
-#Environment=WHOOGLE_ALT_RD=libredd.it
+#Environment=WHOOGLE_ALT_RD=farside.link/libreddit
+#Environment=WHOOGLE_ALT_MD=farside.link/scribe
 #Environment=WHOOGLE_ALT_TL=lingva.ml
-#Environment=WHOOGLE_ALT_MD=scribe.rip
 # Load values from dotenv only
 #Environment=WHOOGLE_DOTENV=1
 Type=simple

app.json

@@ -47,22 +47,27 @@
     },
     "WHOOGLE_ALT_TW": {
       "description": "The site to use as a replacement for twitter.com when site alternatives are enabled in the config.",
-      "value": "nitter.net",
+      "value": "farside.link/nitter",
       "required": false
     },
     "WHOOGLE_ALT_YT": {
       "description": "The site to use as a replacement for youtube.com when site alternatives are enabled in the config.",
-      "value": "invidious.snopyta.org",
+      "value": "farside.link/invidious",
       "required": false
     },
     "WHOOGLE_ALT_IG": {
       "description": "The site to use as a replacement for instagram.com when site alternatives are enabled in the config.",
-      "value": "bibliogram.art/u",
+      "value": "farside.link/bibliogram/u",
       "required": false
     },
     "WHOOGLE_ALT_RD": {
       "description": "The site to use as a replacement for reddit.com when site alternatives are enabled in the config.",
-      "value": "libredd.it",
+      "value": "farside.link/libreddit",
+      "required": false
+    },
+    "WHOOGLE_ALT_MD": {
+      "description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
+      "value": "farside.link/scribe",
       "required": false
     },
     "WHOOGLE_ALT_TL": {
@@ -70,11 +75,6 @@
       "value": "lingva.ml",
       "required": false
     },
-    "WHOOGLE_ALT_MD": {
-      "description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
-      "value": "scribe.rip",
-      "required": false
-    },
     "WHOOGLE_MINIMAL": {
         "description": "Remove everything except basic result cards from all search queries (set to 1 or leave blank)",
         "value": "",

app/models/config.py

@@ -17,7 +17,7 @@ class Config:
         self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
         self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
         self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
-        self.ctry = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
+        self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
         self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
         self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
         self.dark = read_config_bool('WHOOGLE_CONFIG_DARK')  # deprecated
@@ -33,9 +33,13 @@ class Config:
         self.safe_keys = [
             'lang_search',
             'lang_interface',
-            'ctry',
+            'country',
-            'dark',
+            'theme',
-            'theme'
+            'alts',
+            'new_tab',
+            'view_image',
+            'block',
+            'safe'
         ]
 
         # Skip setting custom config if there isn't one
@@ -105,5 +109,26 @@ class Config:
         for param_key in params.keys():
             if not self.is_safe_key(param_key):
                 continue
-            self[param_key] = params.get(param_key)
+            param_val = params.get(param_key)
+
+            if param_val == 'off':
+                param_val = False
+            elif param_val.isdigit():
+                param_val = int(param_val)
+
+            self[param_key] = param_val
         return self
+
+    def to_params(self) -> str:
+        """Generates a set of safe params for using in Whoogle URLs
+
+        Returns:
+            str -- a set of URL parameters
+        """
+        param_str = ''
+        for safe_key in self.safe_keys:
+            if not self[safe_key]:
+                continue
+            param_str = param_str + f'&{safe_key}={self[safe_key]}'
+
+        return param_str

app/request.py

@@ -120,7 +120,7 @@ def gen_query(query, args, config) -> str:
     if 'chips' in args:
         param_dict['chips'] = '&chips=' + args.get('chips')
 
-    param_dict['gl'] = ('&gl=' + config.ctry) if config.ctry else ''
+    param_dict['gl'] = ('&gl=' + config.country) if config.country else ''
     param_dict['hl'] = '&hl=' + (
         config.lang_interface.replace('lang_', '')
         if config.lang_interface else ''

app/routes.py

@@ -310,7 +310,16 @@ def search():
     translate_to = localization_lang.replace('lang_', '')
 
     # Return 503 if temporarily blocked by captcha
-    resp_code = 503 if has_captcha(str(response)) else 200
+    if has_captcha(str(response)):
+        return render_template(
+            'error.html',
+            blocked=True,
+            error_message=translation['ratelimit'],
+            translation=translation,
+            farside='https://farside.link',
+            config=g.user_config,
+            query=urlparse.unquote(query),
+            params=g.user_config.to_params()), 503
     response = bold_search_terms(response, query)
 
     # Feature to display IP address
@@ -351,7 +360,7 @@ def search():
             search_type=search_util.search_type,
             mobile=g.user_request.mobile)
                        if 'isch' not in
-                          search_util.search_type else '')), resp_code
+                          search_util.search_type else '')), 200
 
 
 @app.route(f'/{Endpoint.config}', methods=['GET', 'POST', 'PUT'])

app/static/css/dark-theme.css

@@ -138,10 +138,14 @@ select {
     color: var(--whoogle-dark-contrast-text) !important;
 }
 
-#gh-link {
+.link {
     color: var(--whoogle-dark-contrast-text);
 }
 
+.link-color {
+    color: var(--whoogle-dark-result-url) !important;
+}
+
 .autocomplete-items {
     border: 1px solid var(--whoogle-dark-element-bg);
 }

app/static/css/error.css

@@ -0,0 +1,9 @@
+html {
+    font-size: 1.3rem;
+}
+
+@media (max-width: 1000px) {
+    html {
+        font-size: 3rem;
+    }
+}

app/static/css/light-theme.css

@@ -125,10 +125,14 @@ input {
     color: var(--whoogle-contrast-text);
 }
 
-#gh-link {
+.link {
     color: var(--whoogle-element-bg);
 }
 
+.link-color {
+    color: var(--whoogle-result-url) !important;
+}
+
 .autocomplete-items {
     border: 1px solid var(--whoogle-element-bg);
 }

app/static/css/main.css

@@ -177,3 +177,10 @@ details summary {
     padding: 10px;
     font-weight: bold;
 }
+
+/* Mobile styles */
+@media (max-width: 1000px) {
+    select {
+        width: 100%;
+    }
+}

app/static/settings/translations.json

@@ -33,7 +33,9 @@
         "translate": "translate",
         "light": "light",
         "dark": "dark",
-        "system": "system"
+        "system": "system",
+        "ratelimit": "Instance has been ratelimited",
+        "continue-search": "Continue your search with "
     },
     "lang_nl": {
         "search": "Zoeken",
@@ -69,7 +71,9 @@
         "translate": "vertalen",
         "light": "helder",
         "dark": "donker",
-        "system": "systeeminstellingen"
+        "system": "systeeminstellingen",
+        "ratelimit": "Instantie is beperkt in snelheid",
+        "continue-search": "Ga verder met zoeken met "
     },
     "lang_de": {
         "search": "Suchen",
@@ -105,7 +109,9 @@
         "translate": "Übersetzen",
         "light": "hell",
         "dark": "dunkel",
-        "system": "Systemeinstellung"
+        "system": "Systemeinstellung",
+        "ratelimit": "Instanz wurde ratenbegrenzt",
+        "continue-search": "Setzen Sie Ihre Suche fort mit "
     },
     "lang_es": {
         "search": "Buscar",
@@ -141,7 +147,9 @@
         "translate": "traducir",
         "light": "brillante",
         "dark": "oscuro",
-        "system": "configuración del sistema"
+        "system": "configuración del sistema",
+        "ratelimit": "La instancia ha sido ratelimited",
+        "continue-search": "Continúe su búsqueda con "
     },
     "lang_it": {
         "search": "Cerca",
@@ -177,7 +185,9 @@
         "translate": "tradurre",
         "light": "luminoso",
         "dark": "notte",
-        "system": "impostazioni di sistema"
+        "system": "impostazioni di sistema",
+        "ratelimit": "L'istanza è stata limitata alla velocità",
+        "continue-search": "Continua la tua ricerca con "
     },
     "lang_pt": {
         "search": "Pesquisar",
@@ -213,7 +223,9 @@
         "translate": "traduzir",
         "light": "brilhante",
         "dark": "escuro",
-        "system": "configuração de sistema"
+        "system": "configuração de sistema",
+        "ratelimit": "A instância foi limitada pela taxa",
+        "continue-search": "Continue sua pesquisa com "
     },
     "lang_ru": {
         "search": "Поиск",
@@ -249,7 +261,9 @@
         "translate": "перевести",
         "light": "светлое",
         "dark": "темное",
-        "system": "системное"
+        "system": "системное",
+        "ratelimit": "Число экземпляров ограничено",
+        "continue-search": "Продолжайте поиск с "
     },
     "lang_zh-CN": {
         "search": "搜索",
@@ -285,7 +299,9 @@
         "translate": "翻译",
         "light": "明亮的",
         "dark": "黑暗的",
-        "system": "系统设置"
+        "system": "系统设置",
+        "ratelimit": "实例已被限速",
+        "continue-search": "继续搜索 "
     },
     "lang_si": {
         "search": "සොයන්න",
@@ -321,7 +337,9 @@
         "translate": "පරිවර්තනය කරන්න",
         "light": "දීප්තිමත්",
         "dark": "අඳුරු",
-        "system": "පද්ධතිය"
+        "system": "පද්ධතිය",
+        "ratelimit": "උදාහරණය අනුපාත කර ඇත",
+        "continue-search": "සමඟ ඔබේ සෙවීම දිගටම කරගෙන යන්න"
     },
     "lang_fr": {
         "search": "Chercher",
@@ -357,7 +375,9 @@
         "translate": "Traduire",
         "light": "clair",
         "dark": "sombre",
-        "system": "système"
+        "system": "système",
+        "ratelimit": "Le débit de l'instance a été limité",
+        "continue-search": "Continuez votre recherche avec "
     },
     "lang_fa": {
         "search": "جستجو",
@@ -393,7 +413,9 @@
         "translate": "ترجمه",
         "light": "روشن",
         "dark": "تیره",
-        "system": "سیستم"
+        "system": "سیستم",
+        "ratelimit": "نمونه با نرخ محدود شده است",
+        "continue-search": "جستجوی خود را با "
     },
     "lang_cs": {
         "search": "Hledat",
@@ -429,7 +451,9 @@
         "translate": "Přeložit",
         "light": "Světlý",
         "dark": "Tmavý",
-        "system": "Systémový"
+        "system": "Systémový",
+        "ratelimit": "Instance byla omezena sazbou",
+        "continue-search": "Pokračujte ve vyhledávání pomocí "
     },
     "lang_zh-TW": {
         "search": "搜尋",
@@ -465,7 +489,9 @@
         "translate": "翻譯",
         "light": "明亮的",
         "dark": "黑暗的",
-        "system": "依系統"
+        "system": "依系統",
+        "ratelimit": "實例已被限速",
+        "continue-search": "繼續搜索 "
     },
     "lang_bg": {
         "search": "Търсене",
@@ -501,7 +527,9 @@
         "translate": "превод",
         "light": "светла",
         "dark": "тъмна",
-        "system": "системна"
+        "system": "системна",
+        "ratelimit": "Екземплярът е с ограничена скорост",
+        "continue-search": "Продължете търсенето си с "
     },
     "lang_hi": {
         "search": "खोज",
@@ -537,7 +565,9 @@
         "translate": "अनुवाद करना",
         "light": "रोशनी",
         "dark": "अंधेरा",
-        "system": "प्रणाली"
+        "system": "प्रणाली",
+        "ratelimit": "इंस्टेंस को सीमित कर दिया गया है",
+        "continue-search": "के साथ अपनी खोज जारी रखें "
     },
     "lang_ja": {
         "search": "検索",
@@ -573,6 +603,8 @@
         "translate": "翻訳",
         "light": "ライト",
         "dark": "ダーク",
-        "system": "自動"
+        "system": "自動",
+        "ratelimit": "インスタンスはレート制限されています",
+        "continue-search": "で検索を続ける "
     }
 }

app/templates/error.html

@@ -1,6 +1,40 @@
+{% if config.theme %}
+    {% if config.theme == 'system' %}
+        <style>
+            @import "{{ cb_url('light-theme.css') }}" screen;
+            @import "{{ cb_url('dark-theme.css') }}" screen and (prefers-color-scheme: dark);
+        </style>
+    {% else %}
+        <link rel="stylesheet" href="{{ cb_url(config.theme + '-theme.css') }}"/>
+    {% endif %}
+{% else %}
+    <link rel="stylesheet" href="{{ cb_url(('dark' if config.dark else 'light') + '-theme.css') }}"/>
+{% endif %}
+<link rel="stylesheet" href="{{ cb_url('main.css') }}">
+<link rel="stylesheet" href="{{ cb_url('error.css') }}">
+<style>{{ config.style }}</style>
+<div>
     <h1>Error</h1>
+    <p>
+        {{ error_message|safe }}
+    </p>
     <hr>
     <p>
-    Error: "{{ error_message|safe }}"
+    {% if blocked is defined %}
+        <h4>{{ translation['continue-search'] }} <a class="link" href="https://github.com/benbusby/farside">Farside</a>!</h4>
+        Whoogle:
+        <br>
+        <a class="link-color" href="{{farside}}/whoogle/search?q={{query}}{{params}}">
+            {{farside}}/whoogle/search?q={{query}}{{params}}
+        </a>
+        <br><br>
+        Searx:
+        <br>
+        <a class="link-color" href="{{farside}}/searx/search?q={{query}}">
+            {{farside}}/searx/search?q={{query}}
+        </a>
+        <hr>
+    {% endif %}
     </p>
-<a href="/">Return Home</a>
+    <a class="link" href="home">Return Home</a>
+</div>

app/templates/footer.html

@@ -1,7 +1,7 @@
 <footer>
     <p class="footer">
         Whoogle Search v{{ version_number }} ||
-        <a id="gh-link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
+        <a class="link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
         {% if newest_version %}
              || <span class="update_available">Update Available 🟢</span>
         {% endif %}

app/templates/index.html

@@ -86,15 +86,15 @@
             <div class="config-fields">
                 <form id="config-form" action="config" method="post">
                     <div class="config-options">
-                        <div class="config-div config-div-ctry">
+                        <div class="config-div config-div-country">
-                            <label for="config-ctry">{{ translation['config-country'] }}: </label>
+                            <label for="config-country">{{ translation['config-country'] }}: </label>
-                            <select name="ctry" id="config-ctry">
+                            <select name="country" id="config-country">
-                                {% for ctry in countries %}
+                                {% for country in countries %}
-                                    <option value="{{ ctry.value }}"
+                                    <option value="{{ country.value }}"
-                                            {% if ctry.value in config.ctry %}
+                                            {% if country.value in config.country %}
                                             selected
                                             {% endif %}>
-                                        {{ ctry.name }}
+                                        {{ country.name }}
                                     </option>
                                 {% endfor %}
                             </select>

app/utils/results.py

@@ -24,14 +24,14 @@ BLACKLIST = [
 ]
 
 SITE_ALTS = {
-    'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'nitter.net'),
+    'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'farside.link/nitter'),
-    'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'invidious.snopyta.org'),
+    'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'farside.link/invidious'),
-    'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'bibliogram.art/u'),
+    'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'farside.link/bibliogram/u'),
-    'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'libredd.it'),
+    'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'farside.link/libreddit'),
     **dict.fromkeys([
         'medium.com',
         'levelup.gitconnected.com'
-    ], os.getenv('WHOOGLE_ALT_MD', 'scribe.rip'))
+    ], os.getenv('WHOOGLE_ALT_MD', 'farside.link/scribe'))
 }
 
 

docker-compose.yml

@@ -32,12 +32,12 @@ services:
       # Site alternative configurations, uncomment to enable
       # Note: If not set, the feature will still be available
       # with default values.
-      #- WHOOGLE_ALT_TW=nitter.net
+      #- WHOOGLE_ALT_TW=farside.link/nitter
-      #- WHOOGLE_ALT_YT=invidious.snopyta.org
+      #- WHOOGLE_ALT_YT=farside.link/invidious
-      #- WHOOGLE_ALT_IG=bibliogram.art/u
+      #- WHOOGLE_ALT_IG=farside.link/bibliogram/u
-      #- WHOOGLE_ALT_RD=libredd.it
+      #- WHOOGLE_ALT_RD=farside.link/libreddit
+      #- WHOOGLE_ALT_MD=farside.link/scribe
       #- WHOOGLE_ALT_TL=lingva.ml
-      #- WHOOGLE_ALT_MD=scribe.rip
     #env_file: # Alternatively, load variables from whoogle.env
       #- whoogle.env
     ports:

test/conftest.py

@@ -9,7 +9,7 @@ demo_config = {
     'nojs': str(random.getrandbits(1)),
     'lang_interface': random.choice(app.config['LANGUAGES'])['value'],
     'lang_search': random.choice(app.config['LANGUAGES'])['value'],
-    'ctry': random.choice(app.config['COUNTRIES'])['value']
+    'country': random.choice(app.config['COUNTRIES'])['value']
 }
 
 

test/test_results.py

@@ -86,20 +86,6 @@ def test_block_results(client):
         assert 'pinterest.com' not in urlparse(link['href']).netloc
 
 
-# TODO: Unit test the site alt method instead -- the results returned
-# are too unreliable for this test in particular.
-# def test_site_alts(client):
-    # rv = client.post('/search', data=dict(q='twitter official account'))
-    # assert b'twitter.com/Twitter' in rv.data
-
-    # client.post('/config', data=dict(alts=True))
-    # assert json.loads(client.get('/config').data)['alts']
-
-    # rv = client.post('/search', data=dict(q='twitter official account'))
-    # assert b'twitter.com/Twitter' not in rv.data
-    # assert b'nitter.net/Twitter' in rv.data
-
-
 def test_recent_results(client):
     times = {
         'past year': 365,

whoogle.template.env

@@ -7,12 +7,12 @@
 #  - docker-compose: Uncomment the env_file option
 #  - docker: Add "--env-file ./whoogle.env" to your build command
 
-#WHOOGLE_ALT_TW=nitter.net
+#WHOOGLE_ALT_TW=farside.link/nitter
-#WHOOGLE_ALT_YT=invidious.snopyta.org
+#WHOOGLE_ALT_YT=farside.link/invidious
-#WHOOGLE_ALT_IG=bibliogram.art/u
+#WHOOGLE_ALT_IG=farside.link/bibliogram/u
-#WHOOGLE_ALT_RD=libredd.it
+#WHOOGLE_ALT_RD=farside.link/libreddit
+#WHOOGLE_ALT_MD=farside.link/scribe
 #WHOOGLE_ALT_TL=lingva.ml
-#WHOOGLE_ALT_MD=scribe.rip
 #WHOOGLE_USER=""
 #WHOOGLE_PASS=""
 #WHOOGLE_PROXY_USER=""