nakidai
/
whoogle-search
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use farside.link for frontend alternatives in results (#560) 

* Integrate Farside into Whoogle

When instances are ratelimited (when a captcha is returned instead of
the user's search results) the user can now hop to a new instance via
Farside, a new backend service that redirects users to working instances
of a particular frontend. In this case, it presents a user with a
Farside link to a new Whoogle (or Searx) instance instead, so that the
user can resume their search.

For the generated Farside->Whoogle link, the generated link includes the
user's current Whoogle configuration settings as URL params, to ensure a
more seamless transition between instances. This doesn't translate to
the Farside->Searx link, but potentially could with some changes.

* Expand conversion of config<->url params

Config settings can now be translated to and from URL params using a
predetermined set of "safe" keys (i.e. config settings that easily
translate to URL params).

* Allow jumping instances via Farside when ratelimited

When instances are ratelimited (when a captcha is returned instead of
the user's search results) the user can now hop to a new instance via
Farside, a new backend service that redirects users to working instances
of a particular frontend. In this case, it presents a user with a
Farside link to a new Whoogle (or Searx) instance instead, so that the
user can resume their search.

For the generated Farside->Whoogle link, the generated link includes the
user's current Whoogle configuration settings as URL params, to ensure a
more seamless transition between instances. This doesn't translate to
the Farside->Searx link, but potentially could with some changes.

Closes #554

Closes #559
main
Ben Busby 2021-12-08 17:27:33 -07:00 committed by GitHub
parent 7bea6349a0
commit 634d179568
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 205 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
Dockerfile
View File

@ -47,18 +47,18 @@ ENV HTTPS_ONLY=$use_https
ARG whoogle_port=5000
ENV EXPOSE_PORT=$whoogle_port
ARG twitter_alt='nitter.net'
ARG twitter_alt='farside.link/nitter'
ENV WHOOGLE_ALT_TW=$twitter_alt
ARG youtube_alt='invidious.snopyta.org'
ARG youtube_alt='farside.link/invidious'
ENV WHOOGLE_ALT_YT=$youtube_alt
ARG instagram_alt='bibliogram.art/u'
ARG instagram_alt='farside.link/bibliogram'
ENV WHOOGLE_ALT_IG=$instagram_alt
ARG reddit_alt='libredd.it'
ARG reddit_alt='farside.link/libreddit'
ENV WHOOGLE_ALT_RD=$reddit_alt
ARG medium_alt='farside.link/scribe'
ENV WHOOGLE_ALT_MD=$medium_alt
ARG translate_alt='lingva.ml'
ENV WHOOGLE_ALT_TL=$translate_alt
ARG medium_alt='scribe.rip'
ENV WHOOGLE_ALT_MD=$medium_alt
WORKDIR /whoogle

10
README.md
View File

@ -197,12 +197,12 @@ Description=Whoogle
# Site alternative configurations, uncomment to enable
# Note: If not set, the feature will still be available
# with default values.
#Environment=WHOOGLE_ALT_TW=nitter.net
#Environment=WHOOGLE_ALT_YT=invidious.snopyta.org
#Environment=WHOOGLE_ALT_IG=bibliogram.art/u
#Environment=WHOOGLE_ALT_RD=libredd.it
#Environment=WHOOGLE_ALT_TW=farside.link/nitter
#Environment=WHOOGLE_ALT_YT=farside.link/invidious
#Environment=WHOOGLE_ALT_IG=farside.link/bibliogram/u
#Environment=WHOOGLE_ALT_RD=farside.link/libreddit
#Environment=WHOOGLE_ALT_MD=farside.link/scribe
#Environment=WHOOGLE_ALT_TL=lingva.ml
#Environment=WHOOGLE_ALT_MD=scribe.rip
# Load values from dotenv only
#Environment=WHOOGLE_DOTENV=1
Type=simple

18
app.json
View File

@ -47,22 +47,27 @@
},
"WHOOGLE_ALT_TW": {
"description": "The site to use as a replacement for twitter.com when site alternatives are enabled in the config.",
"value": "nitter.net",
"value": "farside.link/nitter",
"required": false
},
"WHOOGLE_ALT_YT": {
"description": "The site to use as a replacement for youtube.com when site alternatives are enabled in the config.",
"value": "invidious.snopyta.org",
"value": "farside.link/invidious",
"required": false
},
"WHOOGLE_ALT_IG": {
"description": "The site to use as a replacement for instagram.com when site alternatives are enabled in the config.",
"value": "bibliogram.art/u",
"value": "farside.link/bibliogram/u",
"required": false
},
"WHOOGLE_ALT_RD": {
"description": "The site to use as a replacement for reddit.com when site alternatives are enabled in the config.",
"value": "libredd.it",
"value": "farside.link/libreddit",
"required": false
},
"WHOOGLE_ALT_MD": {
"description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
"value": "farside.link/scribe",
"required": false
},
"WHOOGLE_ALT_TL": {
@ -70,11 +75,6 @@
"value": "lingva.ml",
"required": false
},
"WHOOGLE_ALT_MD": {
"description": "The site to use as a replacement for medium.com when site alternatives are enabled in the config.",
"value": "scribe.rip",
"required": false
},
"WHOOGLE_MINIMAL": {
"description": "Remove everything except basic result cards from all search queries (set to 1 or leave blank)",
"value": "",

35
app/models/config.py
View File

@ -17,7 +17,7 @@ class Config:
self.block = os.getenv('WHOOGLE_CONFIG_BLOCK', '')
self.block_title = os.getenv('WHOOGLE_CONFIG_BLOCK_TITLE', '')
self.block_url = os.getenv('WHOOGLE_CONFIG_BLOCK_URL', '')
self.ctry = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
self.country = os.getenv('WHOOGLE_CONFIG_COUNTRY', 'US')
self.theme = os.getenv('WHOOGLE_CONFIG_THEME', 'system')
self.safe = read_config_bool('WHOOGLE_CONFIG_SAFE')
self.dark = read_config_bool('WHOOGLE_CONFIG_DARK') # deprecated
@ -33,9 +33,13 @@ class Config:
self.safe_keys = [
'lang_search',
'lang_interface',
'ctry',
'dark',
'theme'
'country',
'theme',
'alts',
'new_tab',
'view_image',
'block',
'safe'
]
# Skip setting custom config if there isn't one
@ -105,5 +109,26 @@ class Config:
for param_key in params.keys():
if not self.is_safe_key(param_key):
continue
self[param_key] = params.get(param_key)
param_val = params.get(param_key)
if param_val == 'off':
param_val = False
elif param_val.isdigit():
param_val = int(param_val)
self[param_key] = param_val
return self
def to_params(self) -> str:
"""Generates a set of safe params for using in Whoogle URLs
Returns:
str -- a set of URL parameters
"""
param_str = ''
for safe_key in self.safe_keys:
if not self[safe_key]:
continue
param_str = param_str + f'&{safe_key}={self[safe_key]}'
return param_str

2
app/request.py
View File

@ -120,7 +120,7 @@ def gen_query(query, args, config) -> str:
if 'chips' in args:
param_dict['chips'] = '&chips=' + args.get('chips')
param_dict['gl'] = ('&gl=' + config.ctry) if config.ctry else ''
param_dict['gl'] = ('&gl=' + config.country) if config.country else ''
param_dict['hl'] = '&hl=' + (
config.lang_interface.replace('lang_', '')
if config.lang_interface else ''

13
app/routes.py
View File

@ -310,7 +310,16 @@ def search():
translate_to = localization_lang.replace('lang_', '')
# Return 503 if temporarily blocked by captcha
resp_code = 503 if has_captcha(str(response)) else 200
if has_captcha(str(response)):
return render_template(
'error.html',
blocked=True,
error_message=translation['ratelimit'],
translation=translation,
farside='https://farside.link',
config=g.user_config,
query=urlparse.unquote(query),
params=g.user_config.to_params()), 503
response = bold_search_terms(response, query)
# Feature to display IP address
@ -351,7 +360,7 @@ def search():
search_type=search_util.search_type,
mobile=g.user_request.mobile)
if 'isch' not in
search_util.search_type else '')), resp_code
search_util.search_type else '')), 200
@app.route(f'/{Endpoint.config}', methods=['GET', 'POST', 'PUT'])

6
app/static/css/dark-theme.css
View File

@ -138,10 +138,14 @@ select {
color: var(--whoogle-dark-contrast-text) !important;
}
#gh-link {
.link {
color: var(--whoogle-dark-contrast-text);
}
.link-color {
color: var(--whoogle-dark-result-url) !important;
}
.autocomplete-items {
border: 1px solid var(--whoogle-dark-element-bg);
}

9
app/static/css/error.css Normal file
View File

@ -0,0 +1,9 @@
html {
font-size: 1.3rem;
}
@media (max-width: 1000px) {
html {
font-size: 3rem;
}
}

6
app/static/css/light-theme.css
View File

@ -125,10 +125,14 @@ input {
color: var(--whoogle-contrast-text);
}
#gh-link {
.link {
color: var(--whoogle-element-bg);
}
.link-color {
color: var(--whoogle-result-url) !important;
}
.autocomplete-items {
border: 1px solid var(--whoogle-element-bg);
}

7
app/static/css/main.css
View File

@ -177,3 +177,10 @@ details summary {
padding: 10px;
font-weight: bold;
}
/* Mobile styles */
@media (max-width: 1000px) {
select {
width: 100%;
}
}

64
app/static/settings/translations.json
View File

@ -33,7 +33,9 @@
"translate": "translate",
"light": "light",
"dark": "dark",
"system": "system"
"system": "system",
"ratelimit": "Instance has been ratelimited",
"continue-search": "Continue your search with "
},
"lang_nl": {
"search": "Zoeken",
@ -69,7 +71,9 @@
"translate": "vertalen",
"light": "helder",
"dark": "donker",
"system": "systeeminstellingen"
"system": "systeeminstellingen",
"ratelimit": "Instantie is beperkt in snelheid",
"continue-search": "Ga verder met zoeken met "
},
"lang_de": {
"search": "Suchen",
@ -105,7 +109,9 @@
"translate": "Übersetzen",
"light": "hell",
"dark": "dunkel",
"system": "Systemeinstellung"
"system": "Systemeinstellung",
"ratelimit": "Instanz wurde ratenbegrenzt",
"continue-search": "Setzen Sie Ihre Suche fort mit "
},
"lang_es": {
"search": "Buscar",
@ -141,7 +147,9 @@
"translate": "traducir",
"light": "brillante",
"dark": "oscuro",
"system": "configuración del sistema"
"system": "configuración del sistema",
"ratelimit": "La instancia ha sido ratelimited",
"continue-search": "Continúe su búsqueda con "
},
"lang_it": {
"search": "Cerca",
@ -177,7 +185,9 @@
"translate": "tradurre",
"light": "luminoso",
"dark": "notte",
"system": "impostazioni di sistema"
"system": "impostazioni di sistema",
"ratelimit": "L'istanza è stata limitata alla velocità",
"continue-search": "Continua la tua ricerca con "
},
"lang_pt": {
"search": "Pesquisar",
@ -213,7 +223,9 @@
"translate": "traduzir",
"light": "brilhante",
"dark": "escuro",
"system": "configuração de sistema"
"system": "configuração de sistema",
"ratelimit": "A instância foi limitada pela taxa",
"continue-search": "Continue sua pesquisa com "
},
"lang_ru": {
"search": "Поиск",
@ -249,7 +261,9 @@
"translate": "перевести",
"light": "светлое",
"dark": "темное",
"system": "системное"
"system": "системное",
"ratelimit": "Число экземпляров ограничено",
"continue-search": "Продолжайте поиск с "
},
"lang_zh-CN": {
"search": "搜索",
@ -285,7 +299,9 @@
"translate": "翻译",
"light": "明亮的",
"dark": "黑暗的",
"system": "系统设置"
"system": "系统设置",
"ratelimit": "实例已被限速",
"continue-search": "继续搜索 "
},
"lang_si": {
"search": "සොයන්න",
@ -321,7 +337,9 @@
"translate": "පරිවර්තනය කරන්න",
"light": "දීප්තිමත්",
"dark": "අඳුරු",
"system": "පද්ධතිය"
"system": "පද්ධතිය",
"ratelimit": "උදාහරණය අනුපාත කර ඇත",
"continue-search": "සමඟ ඔබේ සෙවීම දිගටම කරගෙන යන්න"
},
"lang_fr": {
"search": "Chercher",
@ -357,7 +375,9 @@
"translate": "Traduire",
"light": "clair",
"dark": "sombre",
"system": "système"
"system": "système",
"ratelimit": "Le débit de l'instance a été limité",
"continue-search": "Continuez votre recherche avec "
},
"lang_fa": {
"search": "جستجو",
@ -393,7 +413,9 @@
"translate": "ترجمه",
"light": "روشن",
"dark": "تیره",
"system": "سیستم"
"system": "سیستم",
"ratelimit": "نمونه با نرخ محدود شده است",
"continue-search": "جستجوی خود را با "
},
"lang_cs": {
"search": "Hledat",
@ -429,7 +451,9 @@
"translate": "Přeložit",
"light": "Světlý",
"dark": "Tmavý",
"system": "Systémový"
"system": "Systémový",
"ratelimit": "Instance byla omezena sazbou",
"continue-search": "Pokračujte ve vyhledávání pomocí "
},
"lang_zh-TW": {
"search": "搜尋",
@ -465,7 +489,9 @@
"translate": "翻譯",
"light": "明亮的",
"dark": "黑暗的",
"system": "依系統"
"system": "依系統",
"ratelimit": "實例已被限速",
"continue-search": "繼續搜索 "
},
"lang_bg": {
"search": "Търсене",
@ -501,7 +527,9 @@
"translate": "превод",
"light": "светла",
"dark": "тъмна",
"system": "системна"
"system": "системна",
"ratelimit": "Екземплярът е с ограничена скорост",
"continue-search": "Продължете търсенето си с "
},
"lang_hi": {
"search": "खोज",
@ -537,7 +565,9 @@
"translate": "अनुवाद करना",
"light": "रोशनी",
"dark": "अंधेरा",
"system": "प्रणाली"
"system": "प्रणाली",
"ratelimit": "इंस्टेंस को सीमित कर दिया गया है",
"continue-search": "के साथ अपनी खोज जारी रखें "
},
"lang_ja": {
"search": "検索",
@ -573,6 +603,8 @@
"translate": "翻訳",
"light": "ライト",
"dark": "ダーク",
"system": "自動"
"system": "自動",
"ratelimit": "インスタンスはレート制限されています",
"continue-search": "で検索を続ける "
}
}

46
app/templates/error.html
View File

@ -1,6 +1,40 @@
<h1>Error</h1>
<hr>
<p>
Error: "{{ error_message|safe }}"
</p>
<a href="/">Return Home</a>
{% if config.theme %}
{% if config.theme == 'system' %}
<style>
@import "{{ cb_url('light-theme.css') }}" screen;
@import "{{ cb_url('dark-theme.css') }}" screen and (prefers-color-scheme: dark);
</style>
{% else %}
<link rel="stylesheet" href="{{ cb_url(config.theme + '-theme.css') }}"/>
{% endif %}
{% else %}
<link rel="stylesheet" href="{{ cb_url(('dark' if config.dark else 'light') + '-theme.css') }}"/>
{% endif %}
<link rel="stylesheet" href="{{ cb_url('main.css') }}">
<link rel="stylesheet" href="{{ cb_url('error.css') }}">
<style>{{ config.style }}</style>
<div>
<h1>Error</h1>
<p>
{{ error_message|safe }}
</p>
<hr>
<p>
{% if blocked is defined %}
<h4>{{ translation['continue-search'] }} <a class="link" href="https://github.com/benbusby/farside">Farside</a>!</h4>
Whoogle:
<br>
<a class="link-color" href="{{farside}}/whoogle/search?q={{query}}{{params}}">
{{farside}}/whoogle/search?q={{query}}{{params}}
</a>
<br><br>
Searx:
<br>
<a class="link-color" href="{{farside}}/searx/search?q={{query}}">
{{farside}}/searx/search?q={{query}}
</a>
<hr>
{% endif %}
</p>
<a class="link" href="home">Return Home</a>
</div>

2
app/templates/footer.html
View File

@ -1,7 +1,7 @@
<footer>
<p class="footer">
Whoogle Search v{{ version_number }} ||
<a id="gh-link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
<a class="link" href="https://github.com/benbusby/whoogle-search">{{ translation['github-link'] }}</a>
{% if newest_version %}
|| <span class="update_available">Update Available 🟢</span>
{% endif %}

14
app/templates/index.html
View File

@ -86,15 +86,15 @@
<div class="config-fields">
<form id="config-form" action="config" method="post">
<div class="config-options">
<div class="config-div config-div-ctry">
<label for="config-ctry">{{ translation['config-country'] }}: </label>
<select name="ctry" id="config-ctry">
{% for ctry in countries %}
<option value="{{ ctry.value }}"
{% if ctry.value in config.ctry %}
<div class="config-div config-div-country">
<label for="config-country">{{ translation['config-country'] }}: </label>
<select name="country" id="config-country">
{% for country in countries %}
<option value="{{ country.value }}"
{% if country.value in config.country %}
selected
{% endif %}>
{{ ctry.name }}
{{ country.name }}
</option>
{% endfor %}
</select>

10
app/utils/results.py
View File

@ -24,14 +24,14 @@ BLACKLIST = [
]
SITE_ALTS = {
'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'nitter.net'),
'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'invidious.snopyta.org'),
'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'bibliogram.art/u'),
'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'libredd.it'),
'twitter.com': os.getenv('WHOOGLE_ALT_TW', 'farside.link/nitter'),
'youtube.com': os.getenv('WHOOGLE_ALT_YT', 'farside.link/invidious'),
'instagram.com': os.getenv('WHOOGLE_ALT_IG', 'farside.link/bibliogram/u'),
'reddit.com': os.getenv('WHOOGLE_ALT_RD', 'farside.link/libreddit'),
**dict.fromkeys([
'medium.com',
'levelup.gitconnected.com'
], os.getenv('WHOOGLE_ALT_MD', 'scribe.rip'))
], os.getenv('WHOOGLE_ALT_MD', 'farside.link/scribe'))
}

10
docker-compose.yml
View File

@ -32,12 +32,12 @@ services:
# Site alternative configurations, uncomment to enable
# Note: If not set, the feature will still be available
# with default values.
#- WHOOGLE_ALT_TW=nitter.net
#- WHOOGLE_ALT_YT=invidious.snopyta.org
#- WHOOGLE_ALT_IG=bibliogram.art/u
#- WHOOGLE_ALT_RD=libredd.it
#- WHOOGLE_ALT_TW=farside.link/nitter
#- WHOOGLE_ALT_YT=farside.link/invidious
#- WHOOGLE_ALT_IG=farside.link/bibliogram/u
#- WHOOGLE_ALT_RD=farside.link/libreddit
#- WHOOGLE_ALT_MD=farside.link/scribe
#- WHOOGLE_ALT_TL=lingva.ml
#- WHOOGLE_ALT_MD=scribe.rip
#env_file: # Alternatively, load variables from whoogle.env
#- whoogle.env
ports:

2
test/conftest.py
View File

@ -9,7 +9,7 @@ demo_config = {
'nojs': str(random.getrandbits(1)),
'lang_interface': random.choice(app.config['LANGUAGES'])['value'],
'lang_search': random.choice(app.config['LANGUAGES'])['value'],
'ctry': random.choice(app.config['COUNTRIES'])['value']
'country': random.choice(app.config['COUNTRIES'])['value']
}

14
test/test_results.py
View File

@ -86,20 +86,6 @@ def test_block_results(client):
assert 'pinterest.com' not in urlparse(link['href']).netloc
# TODO: Unit test the site alt method instead -- the results returned
# are too unreliable for this test in particular.
# def test_site_alts(client):
# rv = client.post('/search', data=dict(q='twitter official account'))
# assert b'twitter.com/Twitter' in rv.data
# client.post('/config', data=dict(alts=True))
# assert json.loads(client.get('/config').data)['alts']
# rv = client.post('/search', data=dict(q='twitter official account'))
# assert b'twitter.com/Twitter' not in rv.data
# assert b'nitter.net/Twitter' in rv.data
def test_recent_results(client):
times = {
'past year': 365,

10
whoogle.template.env
View File

@ -7,12 +7,12 @@
# - docker-compose: Uncomment the env_file option
# - docker: Add "--env-file ./whoogle.env" to your build command
#WHOOGLE_ALT_TW=nitter.net
#WHOOGLE_ALT_YT=invidious.snopyta.org
#WHOOGLE_ALT_IG=bibliogram.art/u
#WHOOGLE_ALT_RD=libredd.it
#WHOOGLE_ALT_TW=farside.link/nitter
#WHOOGLE_ALT_YT=farside.link/invidious
#WHOOGLE_ALT_IG=farside.link/bibliogram/u
#WHOOGLE_ALT_RD=farside.link/libreddit
#WHOOGLE_ALT_MD=farside.link/scribe
#WHOOGLE_ALT_TL=lingva.ml
#WHOOGLE_ALT_MD=scribe.rip
#WHOOGLE_USER=""
#WHOOGLE_PASS=""
#WHOOGLE_PROXY_USER=""