From dd9d87d25bacc48e0287650c46cea02b71ace81d Mon Sep 17 00:00:00 2001 From: Marvin Borner Date: Fri, 26 Jun 2020 00:26:02 +0200 Subject: [PATCH 1/3] Added ddg-style !bang-operators This is a proof of concept! The code works, but uses hardcoded operators and may be placed in the wrong file/class. The best-case scenario would be the possibility to use the 13.000+ ddg operators, but I don't know if that's possible without having to redirect to duckduckgo first. --- app/routes.py | 4 ++++ app/utils/routing_utils.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/app/routes.py b/app/routes.py index ed288c0..7a58e06 100644 --- a/app/routes.py +++ b/app/routes.py @@ -144,6 +144,10 @@ def search(): search_util = RoutingUtils(request, g.user_config, session, cookies_disabled=g.cookies_disabled) query = search_util.new_search_query() + resolved_bangs = search_util.bang_operator() + if resolved_bangs != '': + return redirect(resolved_bangs) + # Redirect to home if invalid/blank search if not query: return redirect('/') diff --git a/app/utils/routing_utils.py b/app/utils/routing_utils.py index 40f8a90..deb0444 100644 --- a/app/utils/routing_utils.py +++ b/app/utils/routing_utils.py @@ -17,6 +17,14 @@ class RoutingUtils: self.query = '' self.cookies_disabled = cookies_disabled self.search_type = self.request_params.get('tbm') if 'tbm' in self.request_params else '' + self.bang_operators = { + '!gh': 'https://github.com/search?q=', + '!ddg': 'https://duckduckgo.com/?q=', + '!w': 'https://wikipedia.com/wiki/', + '!so': 'https://stackoverflow.com/search?q=', + '!a': 'https://amazon.com/s?k=', + '!ebay': 'https://ebay.com/sch/i.html?_nkw=', + } def __getitem__(self, name): return getattr(self, name) @@ -55,6 +63,15 @@ class RoutingUtils: self.query = q[2:] if self.feeling_lucky else q return self.query + + def bang_operator(self) -> str: + print(self.query) + for operator, url in self.bang_operators.items(): + if operator in self.query: + return url + self.query.replace(operator, '').strip() + return '' + + def generate_response(self) -> Tuple[Any, int]: mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent From 348301f201651c7ebc04db2cfbee8f9ebaa5c1ea Mon Sep 17 00:00:00 2001 From: Marvin Borner Date: Sun, 28 Jun 2020 12:03:23 +0200 Subject: [PATCH 2/3] Added bang operator list generator This is, again, just a proof of concept. --- gen_ops.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 gen_ops.py diff --git a/gen_ops.py b/gen_ops.py new file mode 100644 index 0000000..6ecedb6 --- /dev/null +++ b/gen_ops.py @@ -0,0 +1,19 @@ +import csv, json, sys +import requests +import collections + +# Request list +try: + r = requests.get('https://duckduckgo.com/bang.v255.js') + r.raise_for_status() +except requests.exceptions.HTTPError as err: + raise SystemExit(err) + +# Convert to json +data = json.loads(r.text) + +# Output CSV +output = csv.writer(sys.stdout) +output.writerow(['tag', 'url', 'domain', 'name']) +for row in data: + output.writerow([row['t'], row['u'], row['d'], row['s']]) From ae05e8ff8b4100f32990ef4bf6d918046ac15e9e Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Sat, 10 Oct 2020 15:55:14 -0400 Subject: [PATCH 3/3] Finished basic implementation of DDG bang feature Initialization of the app now includes generation of a ddg-bang json file, which is used for all bang style searches afterwards. Also added search suggestion handling for bang json lookup. Queries beginning with "!" now reference the bang json file to pull all keys that match. Updated test suite to include basic tests for bang functionality. Updated gitignore to exclude bang subdir. --- .gitignore | 1 + app/__init__.py | 8 ++++++++ app/routes.py | 9 ++++++++- app/static/js/autocomplete.js | 10 ++++++++-- app/utils/gen_ddg_bangs.py | 26 ++++++++++++++++++++++++++ app/utils/routing_utils.py | 19 ++++--------------- gen_ops.py | 19 ------------------- test/test_routes.py | 10 ++++++++++ 8 files changed, 65 insertions(+), 37 deletions(-) create mode 100644 app/utils/gen_ddg_bangs.py delete mode 100644 gen_ops.py diff --git a/.gitignore b/.gitignore index bbffdb4..caa4595 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ test/static flask_session/ app/static/config app/static/custom_config +app/static/bangs # pip stuff build/ diff --git a/app/__init__.py b/app/__init__.py index 8293c44..820edb0 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,5 @@ from app.utils.session_utils import generate_user_keys +from app.utils.gen_ddg_bangs import gen_bangs_json from flask import Flask from flask_session import Session import os @@ -15,6 +16,8 @@ app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config')) app.config['DEFAULT_CONFIG'] = os.path.join(app.config['CONFIG_PATH'], 'config.json') app.config['SESSION_FILE_DIR'] = os.path.join(app.config['CONFIG_PATH'], 'session') +app.config['BANG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'bangs')) +app.config['BANG_FILE'] = os.path.join(app.config['BANG_PATH'], 'bangs.json') if not os.path.exists(app.config['CONFIG_PATH']): os.makedirs(app.config['CONFIG_PATH']) @@ -22,6 +25,11 @@ if not os.path.exists(app.config['CONFIG_PATH']): if not os.path.exists(app.config['SESSION_FILE_DIR']): os.makedirs(app.config['SESSION_FILE_DIR']) +# (Re)generate DDG bang filter, and create path if it doesn't exist yet +if not os.path.exists(app.config['BANG_PATH']): + os.makedirs(app.config['BANG_PATH']) +gen_bangs_json(app.config['BANG_FILE']) + Session(app) from app import routes diff --git a/app/routes.py b/app/routes.py index 198dc4f..da4be87 100644 --- a/app/routes.py +++ b/app/routes.py @@ -18,6 +18,9 @@ from app.request import Request from app.utils.session_utils import valid_user_session from app.utils.routing_utils import * +# Load DDG bang json files only on init +bang_json = json.load(open(app.config['BANG_FILE'])) + def auth_required(f): @wraps(f) @@ -126,6 +129,10 @@ def opensearch(): def autocomplete(): q = g.request_params.get('q') + # Search bangs if the query begins with "!", but not "! " (feeling lucky) + if q.startswith('!') and len(q) > 1 and not q.startswith('! '): + return jsonify([q, [bang_json[_]['suggestion'] for _ in bang_json if _.startswith(q)]]) + if not q and not request.data: return jsonify({'?': []}) elif request.data: @@ -143,7 +150,7 @@ def search(): search_util = RoutingUtils(request, g.user_config, session, cookies_disabled=g.cookies_disabled) query = search_util.new_search_query() - resolved_bangs = search_util.bang_operator() + resolved_bangs = search_util.bang_operator(bang_json) if resolved_bangs != '': return redirect(resolved_bangs) diff --git a/app/static/js/autocomplete.js b/app/static/js/autocomplete.js index 3d179ca..b8f8bf6 100644 --- a/app/static/js/autocomplete.js +++ b/app/static/js/autocomplete.js @@ -93,8 +93,14 @@ const autocomplete = (searchInput, autocompleteResults) => { removeActive(suggestion); suggestion[currentFocus].classList.add("autocomplete-active"); - // Autofill search bar with suggestion content - searchBar.value = suggestion[currentFocus].textContent; + // Autofill search bar with suggestion content (minus the "bang name" if using a bang operator) + let searchContent = suggestion[currentFocus].textContent; + if (searchContent.indexOf('(') > 0) { + searchBar.value = searchContent.substring(0, searchContent.indexOf('(')); + } else { + searchBar.value = searchContent; + } + searchBar.focus(); }; diff --git a/app/utils/gen_ddg_bangs.py b/app/utils/gen_ddg_bangs.py new file mode 100644 index 0000000..0ed3953 --- /dev/null +++ b/app/utils/gen_ddg_bangs.py @@ -0,0 +1,26 @@ +import json +import requests + + +def gen_bangs_json(bangs_file): + # Request list + try: + r = requests.get('https://duckduckgo.com/bang.v255.js') + r.raise_for_status() + except requests.exceptions.HTTPError as err: + raise SystemExit(err) + + # Convert to json + data = json.loads(r.text) + + # Set up a json object (with better formatting) for all available bangs + bangs_data = {} + + for row in data: + bang_command = '!' + row['t'] + bangs_data[bang_command] = { + 'url': row['u'].replace('{{{s}}}', '{}'), + 'suggestion': bang_command + ' (' + row['s'] + ')' + } + + json.dump(bangs_data, open(bangs_file, 'w')) diff --git a/app/utils/routing_utils.py b/app/utils/routing_utils.py index 3dac09a..c6c960b 100644 --- a/app/utils/routing_utils.py +++ b/app/utils/routing_utils.py @@ -17,14 +17,6 @@ class RoutingUtils: self.query = '' self.cookies_disabled = cookies_disabled self.search_type = self.request_params.get('tbm') if 'tbm' in self.request_params else '' - self.bang_operators = { - '!gh': 'https://github.com/search?q=', - '!ddg': 'https://duckduckgo.com/?q=', - '!w': 'https://wikipedia.com/wiki/', - '!so': 'https://stackoverflow.com/search?q=', - '!a': 'https://amazon.com/s?k=', - '!ebay': 'https://ebay.com/sch/i.html?_nkw=', - } def __getitem__(self, name): return getattr(self, name) @@ -63,15 +55,12 @@ class RoutingUtils: self.query = q[2:] if self.feeling_lucky else q return self.query - - def bang_operator(self) -> str: - print(self.query) - for operator, url in self.bang_operators.items(): - if operator in self.query: - return url + self.query.replace(operator, '').strip() + def bang_operator(self, bangs_dict: dict) -> str: + for operator in bangs_dict.keys(): + if self.query.split(' ')[0] == operator: + return bangs_dict[operator]['url'].format(self.query.replace(operator, '').strip()) return '' - def generate_response(self) -> Tuple[Any, int]: mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent diff --git a/gen_ops.py b/gen_ops.py deleted file mode 100644 index 6ecedb6..0000000 --- a/gen_ops.py +++ /dev/null @@ -1,19 +0,0 @@ -import csv, json, sys -import requests -import collections - -# Request list -try: - r = requests.get('https://duckduckgo.com/bang.v255.js') - r.raise_for_status() -except requests.exceptions.HTTPError as err: - raise SystemExit(err) - -# Convert to json -data = json.loads(r.text) - -# Output CSV -output = csv.writer(sys.stdout) -output.writerow(['tag', 'url', 'domain', 'name']) -for row in data: - output.writerow([row['t'], row['u'], row['d'], row['s']]) diff --git a/test/test_routes.py b/test/test_routes.py index 3d08f0a..995e3c7 100644 --- a/test/test_routes.py +++ b/test/test_routes.py @@ -27,6 +27,16 @@ def test_feeling_lucky(client): assert rv._status_code == 303 +def test_ddg_bang(client): + rv = client.get('/search?q=!gh%20whoogle') + assert rv._status_code == 302 + assert rv.headers.get('Location').startswith('https://github.com') + + rv = client.get('/search?q=!w%20github') + assert rv._status_code == 302 + assert rv.headers.get('Location').startswith('https://en.wikipedia.org') + + def test_config(client): rv = client.post('/config', data=demo_config) assert rv._status_code == 302