Added ddg-style !bang-operators #96

Adds support for ~12K ddg-style !bang-operators -- for example "!gh <query>" to search GitHub, "!w <query>" to search Wikipedia, etc.

Bang operators are loosely supported in the search suggestion API, but should be improved upon eventually to prioritize more popular bangs. At the moment, most bang suggestions are obscure results that likely aren't being used by the vast majority of users. This is simply due to the fact that no intelligent filtering occurs between matching the input text and the results, it's simply a string comparison against the available bang operator keys.

The full list of bang operators is generated on initialization of the app, under a new and separate directory (`app/static/bangs/`).

Authored by: @marvinborner 
Co-authored by: @benbusby
main
Ben Busby 2020-10-13 10:05:37 -04:00 committed by GitHub
commit 57ca6e99ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 70 additions and 2 deletions

1
.gitignore vendored
View File

@ -9,6 +9,7 @@ test/static
flask_session/
app/static/config
app/static/custom_config
app/static/bangs
# pip stuff
build/

View File

@ -1,4 +1,5 @@
from app.utils.session_utils import generate_user_keys
from app.utils.gen_ddg_bangs import gen_bangs_json
from flask import Flask
from flask_session import Session
import os
@ -15,6 +16,8 @@ app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config
app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config'))
app.config['DEFAULT_CONFIG'] = os.path.join(app.config['CONFIG_PATH'], 'config.json')
app.config['SESSION_FILE_DIR'] = os.path.join(app.config['CONFIG_PATH'], 'session')
app.config['BANG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'bangs'))
app.config['BANG_FILE'] = os.path.join(app.config['BANG_PATH'], 'bangs.json')
if not os.path.exists(app.config['CONFIG_PATH']):
os.makedirs(app.config['CONFIG_PATH'])
@ -22,6 +25,11 @@ if not os.path.exists(app.config['CONFIG_PATH']):
if not os.path.exists(app.config['SESSION_FILE_DIR']):
os.makedirs(app.config['SESSION_FILE_DIR'])
# (Re)generate DDG bang filter, and create path if it doesn't exist yet
if not os.path.exists(app.config['BANG_PATH']):
os.makedirs(app.config['BANG_PATH'])
gen_bangs_json(app.config['BANG_FILE'])
Session(app)
from app import routes

View File

@ -18,6 +18,9 @@ from app.request import Request
from app.utils.session_utils import valid_user_session
from app.utils.routing_utils import *
# Load DDG bang json files only on init
bang_json = json.load(open(app.config['BANG_FILE']))
def auth_required(f):
@wraps(f)
@ -126,6 +129,10 @@ def opensearch():
def autocomplete():
q = g.request_params.get('q')
# Search bangs if the query begins with "!", but not "! " (feeling lucky)
if q.startswith('!') and len(q) > 1 and not q.startswith('! '):
return jsonify([q, [bang_json[_]['suggestion'] for _ in bang_json if _.startswith(q)]])
if not q and not request.data:
return jsonify({'?': []})
elif request.data:
@ -143,6 +150,10 @@ def search():
search_util = RoutingUtils(request, g.user_config, session, cookies_disabled=g.cookies_disabled)
query = search_util.new_search_query()
resolved_bangs = search_util.bang_operator(bang_json)
if resolved_bangs != '':
return redirect(resolved_bangs)
# Redirect to home if invalid/blank search
if not query:
return redirect('/')

View File

@ -93,8 +93,14 @@ const autocomplete = (searchInput, autocompleteResults) => {
removeActive(suggestion);
suggestion[currentFocus].classList.add("autocomplete-active");
// Autofill search bar with suggestion content
searchBar.value = suggestion[currentFocus].textContent;
// Autofill search bar with suggestion content (minus the "bang name" if using a bang operator)
let searchContent = suggestion[currentFocus].textContent;
if (searchContent.indexOf('(') > 0) {
searchBar.value = searchContent.substring(0, searchContent.indexOf('('));
} else {
searchBar.value = searchContent;
}
searchBar.focus();
};

View File

@ -0,0 +1,26 @@
import json
import requests
def gen_bangs_json(bangs_file):
# Request list
try:
r = requests.get('https://duckduckgo.com/bang.v255.js')
r.raise_for_status()
except requests.exceptions.HTTPError as err:
raise SystemExit(err)
# Convert to json
data = json.loads(r.text)
# Set up a json object (with better formatting) for all available bangs
bangs_data = {}
for row in data:
bang_command = '!' + row['t']
bangs_data[bang_command] = {
'url': row['u'].replace('{{{s}}}', '{}'),
'suggestion': bang_command + ' (' + row['s'] + ')'
}
json.dump(bangs_data, open(bangs_file, 'w'))

View File

@ -55,6 +55,12 @@ class RoutingUtils:
self.query = q[2:] if self.feeling_lucky else q
return self.query
def bang_operator(self, bangs_dict: dict) -> str:
for operator in bangs_dict.keys():
if self.query.split(' ')[0] == operator:
return bangs_dict[operator]['url'].format(self.query.replace(operator, '').strip())
return ''
def generate_response(self) -> Tuple[Any, int]:
mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent

View File

@ -27,6 +27,16 @@ def test_feeling_lucky(client):
assert rv._status_code == 303
def test_ddg_bang(client):
rv = client.get('/search?q=!gh%20whoogle')
assert rv._status_code == 302
assert rv.headers.get('Location').startswith('https://github.com')
rv = client.get('/search?q=!w%20github')
assert rv._status_code == 302
assert rv.headers.get('Location').startswith('https://en.wikipedia.org')
def test_config(client):
rv = client.post('/config', data=demo_config)
assert rv._status_code == 302