Added ddg-style !bang-operators #96

Adds support for ~12K ddg-style !bang-operators -- for example "!gh <query>" to search GitHub, "!w <query>" to search Wikipedia, etc.

Bang operators are loosely supported in the search suggestion API, but should be improved upon eventually to prioritize more popular bangs. At the moment, most bang suggestions are obscure results that likely aren't being used by the vast majority of users. This is simply due to the fact that no intelligent filtering occurs between matching the input text and the results, it's simply a string comparison against the available bang operator keys.

The full list of bang operators is generated on initialization of the app, under a new and separate directory (`app/static/bangs/`).

Authored by: @marvinborner 
Co-authored by: @benbusby

.gitignore

@@ -9,6 +9,7 @@ test/static
 flask_session/
 app/static/config
 app/static/custom_config
+app/static/bangs
 
 # pip stuff
 build/

app/__init__.py

@@ -1,4 +1,5 @@
 from app.utils.session_utils import generate_user_keys
+from app.utils.gen_ddg_bangs import gen_bangs_json
 from flask import Flask
 from flask_session import Session
 import os
@@ -15,6 +16,8 @@ app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config
 app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config'))
 app.config['DEFAULT_CONFIG'] = os.path.join(app.config['CONFIG_PATH'], 'config.json')
 app.config['SESSION_FILE_DIR'] = os.path.join(app.config['CONFIG_PATH'], 'session')
+app.config['BANG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'bangs'))
+app.config['BANG_FILE'] = os.path.join(app.config['BANG_PATH'], 'bangs.json')
 
 if not os.path.exists(app.config['CONFIG_PATH']):
     os.makedirs(app.config['CONFIG_PATH'])
@@ -22,6 +25,11 @@ if not os.path.exists(app.config['CONFIG_PATH']):
 if not os.path.exists(app.config['SESSION_FILE_DIR']):
     os.makedirs(app.config['SESSION_FILE_DIR'])
 
+# (Re)generate DDG bang filter, and create path if it doesn't exist yet
+if not os.path.exists(app.config['BANG_PATH']):
+    os.makedirs(app.config['BANG_PATH'])
+gen_bangs_json(app.config['BANG_FILE'])
+
 Session(app)
 
 from app import routes

app/routes.py

@@ -18,6 +18,9 @@ from app.request import Request
 from app.utils.session_utils import valid_user_session
 from app.utils.routing_utils import *
 
+# Load DDG bang json files only on init
+bang_json = json.load(open(app.config['BANG_FILE']))
+
 
 def auth_required(f):
     @wraps(f)
@@ -126,6 +129,10 @@ def opensearch():
 def autocomplete():
     q = g.request_params.get('q')
 
+    # Search bangs if the query begins with "!", but not "! " (feeling lucky)
+    if q.startswith('!') and len(q) > 1 and not q.startswith('! '):
+        return jsonify([q, [bang_json[_]['suggestion'] for _ in bang_json if _.startswith(q)]])
+
     if not q and not request.data:
         return jsonify({'?': []})
     elif request.data:
@@ -143,6 +150,10 @@ def search():
     search_util = RoutingUtils(request, g.user_config, session, cookies_disabled=g.cookies_disabled)
     query = search_util.new_search_query()
 
+    resolved_bangs = search_util.bang_operator(bang_json)
+    if resolved_bangs != '':
+        return redirect(resolved_bangs)
+
     # Redirect to home if invalid/blank search
     if not query:
         return redirect('/')

app/static/js/autocomplete.js

@@ -93,8 +93,14 @@ const autocomplete = (searchInput, autocompleteResults) => {
         removeActive(suggestion);
         suggestion[currentFocus].classList.add("autocomplete-active");
 
-        // Autofill search bar with suggestion content
+        // Autofill search bar with suggestion content (minus the "bang name" if using a bang operator)
-        searchBar.value = suggestion[currentFocus].textContent;
+        let searchContent = suggestion[currentFocus].textContent;
+        if (searchContent.indexOf('(') > 0) {
+            searchBar.value = searchContent.substring(0, searchContent.indexOf('('));
+        } else {
+            searchBar.value = searchContent;
+        }
+
         searchBar.focus();
     };
 

app/utils/gen_ddg_bangs.py

@@ -0,0 +1,26 @@
+import json
+import requests
+
+
+def gen_bangs_json(bangs_file):
+    # Request list
+    try:
+        r = requests.get('https://duckduckgo.com/bang.v255.js')
+        r.raise_for_status()
+    except requests.exceptions.HTTPError as err:
+        raise SystemExit(err)
+
+    # Convert to json
+    data = json.loads(r.text)
+
+    # Set up a json object (with better formatting) for all available bangs
+    bangs_data = {}
+
+    for row in data:
+        bang_command = '!' + row['t']
+        bangs_data[bang_command] = {
+            'url': row['u'].replace('{{{s}}}', '{}'),
+            'suggestion': bang_command + ' (' + row['s'] + ')'
+        }
+
+    json.dump(bangs_data, open(bangs_file, 'w'))

app/utils/routing_utils.py

@@ -55,6 +55,12 @@ class RoutingUtils:
         self.query = q[2:] if self.feeling_lucky else q
         return self.query
 
+    def bang_operator(self, bangs_dict: dict) -> str:
+        for operator in bangs_dict.keys():
+            if self.query.split(' ')[0] == operator:
+                return bangs_dict[operator]['url'].format(self.query.replace(operator, '').strip())
+        return ''
+
     def generate_response(self) -> Tuple[Any, int]:
         mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent
 

test/test_routes.py

@@ -27,6 +27,16 @@ def test_feeling_lucky(client):
     assert rv._status_code == 303
 
 
+def test_ddg_bang(client):
+    rv = client.get('/search?q=!gh%20whoogle')
+    assert rv._status_code == 302
+    assert rv.headers.get('Location').startswith('https://github.com')
+
+    rv = client.get('/search?q=!w%20github')
+    assert rv._status_code == 302
+    assert rv.headers.get('Location').startswith('https://en.wikipedia.org')
+
+
 def test_config(client):
     rv = client.post('/config', data=demo_config)
     assert rv._status_code == 302