2020-05-07 03:13:02 +03:00
|
|
|
import argparse
|
2020-05-23 23:27:23 +03:00
|
|
|
import base64
|
2020-04-28 05:21:36 +03:00
|
|
|
import io
|
2020-04-05 04:30:53 +03:00
|
|
|
import json
|
2020-06-02 21:54:47 +03:00
|
|
|
import pickle
|
2020-02-22 02:52:29 +03:00
|
|
|
import urllib.parse as urlparse
|
2020-06-02 21:54:47 +03:00
|
|
|
import uuid
|
2020-06-11 21:14:57 +03:00
|
|
|
from functools import wraps
|
|
|
|
|
2020-05-13 02:14:55 +03:00
|
|
|
import waitress
|
2020-12-18 00:06:47 +03:00
|
|
|
from flask import jsonify, make_response, request, redirect, render_template, \
|
|
|
|
send_file, session, url_for
|
2020-06-11 21:14:57 +03:00
|
|
|
from requests import exceptions
|
|
|
|
|
|
|
|
from app import app
|
|
|
|
from app.models.config import Config
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
from app.request import Request, TorError
|
2021-03-08 20:22:04 +03:00
|
|
|
from app.utils.bangs import resolve_bang
|
2021-05-07 18:45:53 +03:00
|
|
|
from app.utils.session import generate_user_key, valid_user_session
|
2021-03-08 20:22:04 +03:00
|
|
|
from app.utils.search import *
|
2020-01-21 23:26:49 +03:00
|
|
|
|
2020-10-10 22:55:14 +03:00
|
|
|
# Load DDG bang json files only on init
|
|
|
|
bang_json = json.load(open(app.config['BANG_FILE']))
|
|
|
|
|
2020-02-22 02:52:29 +03:00
|
|
|
|
2020-05-18 19:30:32 +03:00
|
|
|
def auth_required(f):
|
|
|
|
@wraps(f)
|
|
|
|
def decorated(*args, **kwargs):
|
|
|
|
auth = request.authorization
|
|
|
|
|
|
|
|
# Skip if username/password not set
|
|
|
|
whoogle_user = os.getenv('WHOOGLE_USER', '')
|
|
|
|
whoogle_pass = os.getenv('WHOOGLE_PASS', '')
|
2020-12-18 00:06:47 +03:00
|
|
|
if (not whoogle_user or not whoogle_pass) or (
|
|
|
|
auth
|
|
|
|
and whoogle_user == auth.username
|
|
|
|
and whoogle_pass == auth.password):
|
2020-05-18 19:30:32 +03:00
|
|
|
return f(*args, **kwargs)
|
|
|
|
else:
|
2020-12-18 00:06:47 +03:00
|
|
|
return make_response('Not logged in', 401, {
|
|
|
|
'WWW-Authenticate': 'Basic realm="Login Required"'})
|
|
|
|
|
2020-05-18 19:30:32 +03:00
|
|
|
return decorated
|
|
|
|
|
|
|
|
|
2020-04-24 05:59:43 +03:00
|
|
|
@app.before_request
|
|
|
|
def before_request_func():
|
2020-12-18 00:06:47 +03:00
|
|
|
g.request_params = (
|
|
|
|
request.args if request.method == 'GET' else request.form
|
|
|
|
)
|
2020-06-06 00:24:44 +03:00
|
|
|
g.cookies_disabled = False
|
|
|
|
|
|
|
|
# Generate session values for user if unavailable
|
2020-06-02 21:54:47 +03:00
|
|
|
if not valid_user_session(session):
|
2020-06-06 00:24:44 +03:00
|
|
|
session['config'] = json.load(open(app.config['DEFAULT_CONFIG'])) \
|
2021-03-28 21:24:57 +03:00
|
|
|
if os.path.exists(app.config['DEFAULT_CONFIG']) else {}
|
2020-06-02 21:54:47 +03:00
|
|
|
session['uuid'] = str(uuid.uuid4())
|
2021-04-01 07:23:30 +03:00
|
|
|
session['key'] = generate_user_key(True)
|
2020-06-06 00:24:44 +03:00
|
|
|
|
|
|
|
# Flag cookies as possibly disabled in order to prevent against
|
|
|
|
# unnecessary session directory expansion
|
|
|
|
g.cookies_disabled = True
|
2020-06-02 21:54:47 +03:00
|
|
|
|
2020-12-05 23:51:06 +03:00
|
|
|
# Handle https upgrade
|
2021-01-23 22:50:30 +03:00
|
|
|
if needs_https(request.url):
|
2020-12-18 00:06:47 +03:00
|
|
|
return redirect(
|
|
|
|
request.url.replace('http://', 'https://', 1),
|
|
|
|
code=308)
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
g.user_config = Config(**session['config'])
|
2020-05-10 22:27:02 +03:00
|
|
|
|
2020-05-13 02:15:53 +03:00
|
|
|
if not g.user_config.url:
|
2020-12-18 00:06:47 +03:00
|
|
|
g.user_config.url = request.url_root.replace(
|
|
|
|
'http://',
|
2021-01-23 22:50:30 +03:00
|
|
|
'https://') if os.getenv('HTTPS_ONLY', False) else request.url_root
|
2020-05-10 22:27:02 +03:00
|
|
|
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
g.user_request = Request(
|
|
|
|
request.headers.get('User-Agent'),
|
|
|
|
request.url_root,
|
|
|
|
config=g.user_config)
|
|
|
|
|
2020-05-13 02:15:53 +03:00
|
|
|
g.app_location = g.user_config.url
|
2020-01-22 08:51:02 +03:00
|
|
|
|
2020-01-21 23:26:49 +03:00
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
@app.after_request
|
2021-03-07 22:04:05 +03:00
|
|
|
def after_request_func(resp):
|
2020-12-18 00:06:47 +03:00
|
|
|
# Check if address consistently has cookies blocked,
|
|
|
|
# in which case start removing session files after creation.
|
|
|
|
#
|
|
|
|
# Note: This is primarily done to prevent overpopulation of session
|
|
|
|
# directories, since browsers that block cookies will still trigger
|
|
|
|
# Flask's session creation routine with every request.
|
2020-06-06 00:24:44 +03:00
|
|
|
if g.cookies_disabled and request.remote_addr not in app.no_cookie_ips:
|
|
|
|
app.no_cookie_ips.append(request.remote_addr)
|
|
|
|
elif g.cookies_disabled and request.remote_addr in app.no_cookie_ips:
|
|
|
|
session_list = list(session.keys())
|
|
|
|
for key in session_list:
|
|
|
|
session.pop(key)
|
|
|
|
|
2021-03-07 22:04:05 +03:00
|
|
|
resp.headers['Content-Security-Policy'] = app.config['CSP']
|
|
|
|
if os.environ.get('HTTPS_ONLY', False):
|
|
|
|
resp.headers['Content-Security-Policy'] += 'upgrade-insecure-requests'
|
|
|
|
|
|
|
|
return resp
|
2020-06-02 21:54:47 +03:00
|
|
|
|
|
|
|
|
2020-05-06 03:28:43 +03:00
|
|
|
@app.errorhandler(404)
|
|
|
|
def unknown_page(e):
|
2020-12-18 00:06:47 +03:00
|
|
|
app.logger.warn(e)
|
2020-05-10 22:27:02 +03:00
|
|
|
return redirect(g.app_location)
|
2020-05-06 03:28:43 +03:00
|
|
|
|
|
|
|
|
2021-05-18 18:48:15 +03:00
|
|
|
@app.route('/healthz', methods=['GET'])
|
|
|
|
def healthz():
|
|
|
|
return ''
|
|
|
|
|
|
|
|
|
2021-10-14 05:55:26 +03:00
|
|
|
@app.route('/home', methods=['GET'])
|
|
|
|
def home():
|
|
|
|
return redirect(url_for('.index'))
|
|
|
|
|
|
|
|
|
2020-01-21 23:26:49 +03:00
|
|
|
@app.route('/', methods=['GET'])
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-01-21 23:26:49 +03:00
|
|
|
def index():
|
2020-06-06 00:24:44 +03:00
|
|
|
# Reset keys
|
2021-04-01 07:23:30 +03:00
|
|
|
session['key'] = generate_user_key(g.cookies_disabled)
|
2021-03-07 22:04:05 +03:00
|
|
|
|
|
|
|
# Redirect if an error was raised
|
|
|
|
if 'error_message' in session and session['error_message']:
|
|
|
|
error_message = session['error_message']
|
|
|
|
session['error_message'] = ''
|
|
|
|
return render_template('error.html', error_message=error_message)
|
2020-06-06 00:24:44 +03:00
|
|
|
|
2020-05-13 02:15:53 +03:00
|
|
|
return render_template('index.html',
|
2020-12-18 00:39:35 +03:00
|
|
|
languages=app.config['LANGUAGES'],
|
|
|
|
countries=app.config['COUNTRIES'],
|
2021-06-28 17:26:51 +03:00
|
|
|
themes=app.config['THEMES'],
|
2021-05-25 00:03:02 +03:00
|
|
|
translation=app.config['TRANSLATIONS'][
|
|
|
|
g.user_config.get_localization_lang()
|
|
|
|
],
|
2021-04-05 17:37:39 +03:00
|
|
|
logo=render_template(
|
|
|
|
'logo.html',
|
2021-04-09 18:00:02 +03:00
|
|
|
dark=g.user_config.dark),
|
2021-04-27 17:36:03 +03:00
|
|
|
config_disabled=app.config['CONFIG_DISABLE'],
|
2020-06-02 21:54:47 +03:00
|
|
|
config=g.user_config,
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
tor_available=int(os.environ.get('TOR_AVAILABLE')),
|
2020-06-02 21:54:47 +03:00
|
|
|
version_number=app.config['VERSION_NUMBER'])
|
2020-01-21 23:26:49 +03:00
|
|
|
|
|
|
|
|
2020-04-25 03:45:57 +03:00
|
|
|
@app.route('/opensearch.xml', methods=['GET'])
|
|
|
|
def opensearch():
|
2020-05-10 22:27:02 +03:00
|
|
|
opensearch_url = g.app_location
|
|
|
|
if opensearch_url.endswith('/'):
|
|
|
|
opensearch_url = opensearch_url[:-1]
|
2020-04-25 03:45:57 +03:00
|
|
|
|
2021-01-23 22:50:30 +03:00
|
|
|
# Enforce https for opensearch template
|
|
|
|
if needs_https(opensearch_url):
|
|
|
|
opensearch_url = opensearch_url.replace('http://', 'https://', 1)
|
|
|
|
|
2020-12-18 00:06:47 +03:00
|
|
|
get_only = g.user_config.get_only or 'Chrome' in request.headers.get(
|
|
|
|
'User-Agent')
|
2020-11-18 18:31:19 +03:00
|
|
|
|
2020-08-15 22:02:17 +03:00
|
|
|
return render_template(
|
|
|
|
'opensearch.xml',
|
|
|
|
main_url=opensearch_url,
|
2020-11-18 18:31:19 +03:00
|
|
|
request_type='' if get_only else 'method="post"'
|
2020-08-15 22:02:17 +03:00
|
|
|
), 200, {'Content-Disposition': 'attachment; filename="opensearch.xml"'}
|
2020-04-25 03:45:57 +03:00
|
|
|
|
|
|
|
|
2021-05-21 17:35:46 +03:00
|
|
|
@app.route('/search.html', methods=['GET'])
|
|
|
|
def search_html():
|
|
|
|
search_url = g.app_location
|
|
|
|
if search_url.endswith('/'):
|
|
|
|
search_url = search_url[:-1]
|
|
|
|
return render_template('search.html', url=search_url)
|
|
|
|
|
|
|
|
|
2020-05-24 23:03:11 +03:00
|
|
|
@app.route('/autocomplete', methods=['GET', 'POST'])
|
|
|
|
def autocomplete():
|
2020-06-06 00:24:44 +03:00
|
|
|
q = g.request_params.get('q')
|
2020-10-29 06:02:41 +03:00
|
|
|
if not q:
|
|
|
|
# FF will occasionally (incorrectly) send the q field without a
|
|
|
|
# mimetype in the format "b'q=<query>'" through the request.data field
|
|
|
|
q = str(request.data).replace('q=', '')
|
2020-05-24 23:03:11 +03:00
|
|
|
|
2020-10-10 22:55:14 +03:00
|
|
|
# Search bangs if the query begins with "!", but not "! " (feeling lucky)
|
|
|
|
if q.startswith('!') and len(q) > 1 and not q.startswith('! '):
|
2020-12-18 00:06:47 +03:00
|
|
|
return jsonify([q, [bang_json[_]['suggestion'] for _ in bang_json if
|
|
|
|
_.startswith(q)]])
|
2020-10-10 22:55:14 +03:00
|
|
|
|
2020-05-24 23:03:11 +03:00
|
|
|
if not q and not request.data:
|
|
|
|
return jsonify({'?': []})
|
|
|
|
elif request.data:
|
2020-12-18 00:06:47 +03:00
|
|
|
q = urlparse.unquote_plus(
|
|
|
|
request.data.decode('utf-8').replace('q=', ''))
|
2020-05-24 23:03:11 +03:00
|
|
|
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
# Return a list of suggestions for the query
|
2020-12-18 00:06:47 +03:00
|
|
|
#
|
|
|
|
# Note: If Tor is enabled, this returns nothing, as the request is
|
|
|
|
# almost always rejected
|
|
|
|
return jsonify([
|
|
|
|
q,
|
|
|
|
g.user_request.autocomplete(q) if not g.user_config.tor else []
|
|
|
|
])
|
2020-05-24 23:03:11 +03:00
|
|
|
|
|
|
|
|
2020-04-29 03:19:34 +03:00
|
|
|
@app.route('/search', methods=['GET', 'POST'])
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-01-21 23:26:49 +03:00
|
|
|
def search():
|
2020-11-11 08:40:49 +03:00
|
|
|
# Update user config if specified in search args
|
|
|
|
g.user_config = g.user_config.from_params(g.request_params)
|
|
|
|
|
2021-03-08 20:22:04 +03:00
|
|
|
search_util = Search(request, g.user_config, session,
|
|
|
|
cookies_disabled=g.cookies_disabled)
|
2020-06-02 21:54:47 +03:00
|
|
|
query = search_util.new_search_query()
|
2020-05-18 19:28:23 +03:00
|
|
|
|
2021-03-08 20:22:04 +03:00
|
|
|
bang = resolve_bang(query=query, bangs_dict=bang_json)
|
|
|
|
if bang != '':
|
|
|
|
return redirect(bang)
|
2020-06-26 01:26:02 +03:00
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
# Redirect to home if invalid/blank search
|
|
|
|
if not query:
|
|
|
|
return redirect('/')
|
2020-05-18 19:28:23 +03:00
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
# Generate response and number of external elements from the page
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
try:
|
2021-04-01 07:23:30 +03:00
|
|
|
response = search_util.generate_response()
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
except TorError as e:
|
2020-12-18 00:06:47 +03:00
|
|
|
session['error_message'] = e.message + (
|
|
|
|
"\\n\\nTor config is now disabled!" if e.disable else "")
|
|
|
|
session['config']['tor'] = False if e.disable else session['config'][
|
|
|
|
'tor']
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
return redirect(url_for('.index'))
|
|
|
|
|
2021-04-01 07:23:30 +03:00
|
|
|
if search_util.feeling_lucky:
|
2020-06-02 21:54:47 +03:00
|
|
|
return redirect(response, code=303)
|
2020-05-18 19:28:23 +03:00
|
|
|
|
2021-06-15 17:14:42 +03:00
|
|
|
# If the user is attempting to translate a string, determine the correct
|
|
|
|
# string for formatting the lingva.ml url
|
|
|
|
localization_lang = g.user_config.get_localization_lang()
|
|
|
|
translation = app.config['TRANSLATIONS'][localization_lang]
|
|
|
|
translate_to = localization_lang.replace('lang_', '')
|
|
|
|
|
2021-03-21 04:51:24 +03:00
|
|
|
# Return 503 if temporarily blocked by captcha
|
|
|
|
resp_code = 503 if has_captcha(str(response)) else 200
|
|
|
|
|
2020-05-24 23:03:11 +03:00
|
|
|
return render_template(
|
|
|
|
'display.html',
|
2020-06-02 21:54:47 +03:00
|
|
|
query=urlparse.unquote(query),
|
|
|
|
search_type=search_util.search_type,
|
2021-03-21 04:21:41 +03:00
|
|
|
config=g.user_config,
|
2021-06-15 17:14:42 +03:00
|
|
|
lingva_url=app.config['TRANSLATE_URL'],
|
|
|
|
translation=translation,
|
|
|
|
translate_to=translate_to,
|
|
|
|
translate_str=query.replace(
|
|
|
|
'translate', ''
|
|
|
|
).replace(
|
|
|
|
translation['translate'], ''
|
|
|
|
),
|
|
|
|
is_translation=any(
|
|
|
|
_ in query.lower() for _ in [translation['translate'], 'translate']
|
|
|
|
) and not search_util.search_type, # Standard search queries only
|
2020-06-02 21:54:47 +03:00
|
|
|
response=response,
|
2020-06-11 22:25:23 +03:00
|
|
|
version_number=app.config['VERSION_NUMBER'],
|
2020-12-18 00:06:47 +03:00
|
|
|
search_header=(render_template(
|
2020-05-24 23:03:11 +03:00
|
|
|
'header.html',
|
2021-03-21 04:21:41 +03:00
|
|
|
config=g.user_config,
|
2021-04-09 18:00:02 +03:00
|
|
|
logo=render_template('logo.html', dark=g.user_config.dark),
|
2021-06-04 18:09:30 +03:00
|
|
|
query=urlparse.unquote(query),
|
2020-06-02 21:54:47 +03:00
|
|
|
search_type=search_util.search_type,
|
2020-12-18 00:06:47 +03:00
|
|
|
mobile=g.user_request.mobile)
|
2021-03-21 04:51:24 +03:00
|
|
|
if 'isch' not in search_util.search_type else '')), resp_code
|
2020-01-21 23:26:49 +03:00
|
|
|
|
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
@app.route('/config', methods=['GET', 'POST', 'PUT'])
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-04-06 02:59:50 +03:00
|
|
|
def config():
|
2021-04-27 17:36:03 +03:00
|
|
|
config_disabled = app.config['CONFIG_DISABLE']
|
2020-04-16 02:41:53 +03:00
|
|
|
if request.method == 'GET':
|
2020-05-13 02:15:53 +03:00
|
|
|
return json.dumps(g.user_config.__dict__)
|
2021-04-27 17:36:03 +03:00
|
|
|
elif request.method == 'PUT' and not config_disabled:
|
2020-06-02 21:54:47 +03:00
|
|
|
if 'name' in request.args:
|
2020-12-18 00:06:47 +03:00
|
|
|
config_pkl = os.path.join(
|
|
|
|
app.config['CONFIG_PATH'],
|
|
|
|
request.args.get('name'))
|
|
|
|
session['config'] = (pickle.load(open(config_pkl, 'rb'))
|
|
|
|
if os.path.exists(config_pkl)
|
|
|
|
else session['config'])
|
2020-06-02 21:54:47 +03:00
|
|
|
return json.dumps(session['config'])
|
|
|
|
else:
|
|
|
|
return json.dumps({})
|
2021-04-27 17:36:03 +03:00
|
|
|
elif not config_disabled:
|
2020-04-29 05:50:12 +03:00
|
|
|
config_data = request.form.to_dict()
|
2020-05-10 22:27:02 +03:00
|
|
|
if 'url' not in config_data or not config_data['url']:
|
2020-05-16 01:29:22 +03:00
|
|
|
config_data['url'] = g.user_config.url
|
2020-05-10 22:27:02 +03:00
|
|
|
|
2020-06-06 00:24:44 +03:00
|
|
|
# Save config by name to allow a user to easily load later
|
2020-06-02 21:54:47 +03:00
|
|
|
if 'name' in request.args:
|
2020-12-18 00:06:47 +03:00
|
|
|
pickle.dump(
|
|
|
|
config_data,
|
|
|
|
open(os.path.join(
|
|
|
|
app.config['CONFIG_PATH'],
|
|
|
|
request.args.get('name')), 'wb'))
|
2020-06-06 00:24:44 +03:00
|
|
|
|
|
|
|
# Overwrite default config if user has cookies disabled
|
|
|
|
if g.cookies_disabled:
|
2020-12-18 00:06:47 +03:00
|
|
|
open(app.config['DEFAULT_CONFIG'], 'w').write(
|
|
|
|
json.dumps(config_data, indent=4))
|
2020-04-06 02:59:50 +03:00
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
session['config'] = config_data
|
2020-05-10 22:27:02 +03:00
|
|
|
return redirect(config_data['url'])
|
2021-04-27 17:36:03 +03:00
|
|
|
else:
|
|
|
|
return redirect(url_for('.index'), code=403)
|
2020-04-06 02:59:50 +03:00
|
|
|
|
|
|
|
|
2020-01-21 23:26:49 +03:00
|
|
|
@app.route('/url', methods=['GET'])
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-01-21 23:26:49 +03:00
|
|
|
def url():
|
2020-01-23 09:19:17 +03:00
|
|
|
if 'url' in request.args:
|
|
|
|
return redirect(request.args.get('url'))
|
|
|
|
|
2020-01-21 23:26:49 +03:00
|
|
|
q = request.args.get('q')
|
|
|
|
if len(q) > 0 and 'http' in q:
|
|
|
|
return redirect(q)
|
|
|
|
else:
|
2021-03-07 22:04:05 +03:00
|
|
|
return render_template(
|
|
|
|
'error.html',
|
|
|
|
error_message='Unable to resolve query: ' + q)
|
2020-01-21 23:26:49 +03:00
|
|
|
|
|
|
|
|
2020-01-23 09:19:17 +03:00
|
|
|
@app.route('/imgres')
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-01-23 09:19:17 +03:00
|
|
|
def imgres():
|
|
|
|
return redirect(request.args.get('imgurl'))
|
|
|
|
|
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
@app.route('/element')
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-06-02 21:54:47 +03:00
|
|
|
def element():
|
2021-04-01 07:23:30 +03:00
|
|
|
cipher_suite = Fernet(session['key'])
|
2020-06-02 21:54:47 +03:00
|
|
|
src_url = cipher_suite.decrypt(request.args.get('url').encode()).decode()
|
|
|
|
src_type = request.args.get('type')
|
2020-05-23 23:27:23 +03:00
|
|
|
|
|
|
|
try:
|
2020-06-02 21:54:47 +03:00
|
|
|
file_data = g.user_request.send(base_url=src_url).content
|
2020-05-23 23:27:23 +03:00
|
|
|
tmp_mem = io.BytesIO()
|
|
|
|
tmp_mem.write(file_data)
|
|
|
|
tmp_mem.seek(0)
|
|
|
|
|
2020-06-02 21:54:47 +03:00
|
|
|
return send_file(tmp_mem, mimetype=src_type)
|
|
|
|
except exceptions.RequestException:
|
2020-05-23 23:27:23 +03:00
|
|
|
pass
|
|
|
|
|
2020-12-18 00:06:47 +03:00
|
|
|
empty_gif = base64.b64decode(
|
|
|
|
'R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==')
|
2020-05-23 23:27:23 +03:00
|
|
|
return send_file(io.BytesIO(empty_gif), mimetype='image/gif')
|
2020-04-28 05:21:36 +03:00
|
|
|
|
|
|
|
|
2020-02-22 02:52:29 +03:00
|
|
|
@app.route('/window')
|
2020-05-18 19:30:32 +03:00
|
|
|
@auth_required
|
2020-02-22 02:52:29 +03:00
|
|
|
def window():
|
2020-06-02 21:54:47 +03:00
|
|
|
get_body = g.user_request.send(base_url=request.args.get('location')).text
|
2020-12-18 00:06:47 +03:00
|
|
|
get_body = get_body.replace('src="/',
|
|
|
|
'src="' + request.args.get('location') + '"')
|
|
|
|
get_body = get_body.replace('href="/',
|
|
|
|
'href="' + request.args.get('location') + '"')
|
2020-02-22 02:52:29 +03:00
|
|
|
|
2020-12-18 00:06:47 +03:00
|
|
|
results = bsoup(get_body, 'html.parser')
|
2020-02-22 02:52:29 +03:00
|
|
|
|
2020-12-18 00:06:47 +03:00
|
|
|
for script in results('script'):
|
|
|
|
script.decompose()
|
2020-02-22 02:52:29 +03:00
|
|
|
|
2020-04-24 05:59:43 +03:00
|
|
|
return render_template('display.html', response=results)
|
2020-02-22 02:52:29 +03:00
|
|
|
|
|
|
|
|
2021-03-24 22:13:52 +03:00
|
|
|
def run_app() -> None:
|
2020-12-18 00:06:47 +03:00
|
|
|
parser = argparse.ArgumentParser(
|
|
|
|
description='Whoogle Search console runner')
|
|
|
|
parser.add_argument(
|
|
|
|
'--port',
|
|
|
|
default=5000,
|
|
|
|
metavar='<port number>',
|
|
|
|
help='Specifies a port to run on (default 5000)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--host',
|
|
|
|
default='127.0.0.1',
|
|
|
|
metavar='<ip address>',
|
|
|
|
help='Specifies the host address to use (default 127.0.0.1)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--debug',
|
|
|
|
default=False,
|
|
|
|
action='store_true',
|
|
|
|
help='Activates debug mode for the server (default False)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--https-only',
|
|
|
|
default=False,
|
|
|
|
action='store_true',
|
|
|
|
help='Enforces HTTPS redirects for all requests')
|
|
|
|
parser.add_argument(
|
|
|
|
'--userpass',
|
|
|
|
default='',
|
|
|
|
metavar='<username:password>',
|
|
|
|
help='Sets a username/password basic auth combo (default None)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--proxyauth',
|
|
|
|
default='',
|
|
|
|
metavar='<username:password>',
|
|
|
|
help='Sets a username/password for a HTTP/SOCKS proxy (default None)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--proxytype',
|
|
|
|
default='',
|
|
|
|
metavar='<socks4|socks5|http>',
|
|
|
|
help='Sets a proxy type for all connections (default None)')
|
|
|
|
parser.add_argument(
|
|
|
|
'--proxyloc',
|
|
|
|
default='',
|
|
|
|
metavar='<location:port>',
|
|
|
|
help='Sets a proxy location for all connections (default None)')
|
2020-05-07 03:13:02 +03:00
|
|
|
args = parser.parse_args()
|
2020-05-18 19:30:32 +03:00
|
|
|
|
|
|
|
if args.userpass:
|
|
|
|
user_pass = args.userpass.split(':')
|
|
|
|
os.environ['WHOOGLE_USER'] = user_pass[0]
|
|
|
|
os.environ['WHOOGLE_PASS'] = user_pass[1]
|
|
|
|
|
Add tor and http/socks proxy support (#137)
* Add tor and http/socks proxy support
Allows users to enable/disable tor from the config menu, which will
forward all requests through Tor.
Also adds support for setting environment variables for alternative
proxy support. Setting the following variables will forward requests
through the proxy:
- WHOOGLE_PROXY_USER (optional)
- WHOOGLE_PROXY_PASS (optional)
- WHOOGLE_PROXY_TYPE (required)
- Can be "http", "socks4", or "socks5"
- WHOOGLE_PROXY_LOC (required)
- Format: "<ip address>:<port>"
See #30
* Refactor acquire_tor_conn -> acquire_tor_identity
Also updated travis CI to set up tor
* Add check for Tor socket on init, improve Tor error handling
Initializing the app sends a heartbeat request to Tor to check for
availability, and updates the home page config options accordingly. This
heartbeat is sent on every request, to ensure Tor support can be
reconfigured without restarting the entire app.
If Tor support is enabled, and a subsequent request fails, then a new
TorError exception is raised, and the Tor feature is disabled until a
valid connection is restored.
The max attempts has been updated to 10, since 5 seemed a bit too low
for how quickly the attempts go by.
* Change send_tor_signal arg type, update function doc
send_tor_signal now accepts a stem.Signal arg (a bit cleaner tbh). Also
added the doc string for the "disable" attribute in TorError.
* Fix tor identity logic in Request.send
* Update proxy init, change proxyloc var name
Proxy is now only initialized if both type and location are specified,
as neither have a default fallback and both are required. I suppose the
type could fall back to http, but seems safer this way.
Also refactored proxyurl -> proxyloc for the runtime args in order to
match the Dockerfile args.
* Add tor/proxy support for Docker builds, fix opensearch/init
The Dockerfile is now updated to include support for Tor configuration,
with a working torrc file included in the repo.
An issue with opensearch was fixed as well, which was uncovered during
testing and was simple enough to fix here. Likewise, DDG bang gen was
updated to only ever happen if the file didn't exist previously, as
testing with the file being regenerated every time was tedious.
* Add missing "@" for socks proxy requests
2020-10-29 03:47:42 +03:00
|
|
|
if args.proxytype and args.proxyloc:
|
|
|
|
if args.proxyauth:
|
|
|
|
proxy_user_pass = args.proxyauth.split(':')
|
|
|
|
os.environ['WHOOGLE_PROXY_USER'] = proxy_user_pass[0]
|
|
|
|
os.environ['WHOOGLE_PROXY_PASS'] = proxy_user_pass[1]
|
|
|
|
os.environ['WHOOGLE_PROXY_TYPE'] = args.proxytype
|
|
|
|
os.environ['WHOOGLE_PROXY_LOC'] = args.proxyloc
|
|
|
|
|
2020-05-16 00:44:50 +03:00
|
|
|
os.environ['HTTPS_ONLY'] = '1' if args.https_only else ''
|
|
|
|
|
2020-05-13 02:14:55 +03:00
|
|
|
if args.debug:
|
|
|
|
app.run(host=args.host, port=args.port, debug=args.debug)
|
|
|
|
else:
|
|
|
|
waitress.serve(app, listen="{}:{}".format(args.host, args.port))
|