Finished basic implementation of DDG bang feature

Initialization of the app now includes generation of a ddg-bang json
file, which is used for all bang style searches afterwards.

Also added search suggestion handling for bang json lookup. Queries
beginning with "!" now reference the bang json file to pull all keys
that match.

Updated test suite to include basic tests for bang functionality.

Updated gitignore to exclude bang subdir.

.gitignore

@@ -9,6 +9,7 @@ test/static
 flask_session/
 app/static/config
 app/static/custom_config
+app/static/bangs
 
 # pip stuff
 build/

app/__init__.py

@@ -1,4 +1,5 @@
 from app.utils.session_utils import generate_user_keys
+from app.utils.gen_ddg_bangs import gen_bangs_json
 from flask import Flask
 from flask_session import Session
 import os
@@ -15,6 +16,8 @@ app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config
 app.config['CONFIG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'config'))
 app.config['DEFAULT_CONFIG'] = os.path.join(app.config['CONFIG_PATH'], 'config.json')
 app.config['SESSION_FILE_DIR'] = os.path.join(app.config['CONFIG_PATH'], 'session')
+app.config['BANG_PATH'] = os.getenv('CONFIG_VOLUME', os.path.join(app.config['STATIC_FOLDER'], 'bangs'))
+app.config['BANG_FILE'] = os.path.join(app.config['BANG_PATH'], 'bangs.json')
 
 if not os.path.exists(app.config['CONFIG_PATH']):
     os.makedirs(app.config['CONFIG_PATH'])
@@ -22,6 +25,11 @@ if not os.path.exists(app.config['CONFIG_PATH']):
 if not os.path.exists(app.config['SESSION_FILE_DIR']):
     os.makedirs(app.config['SESSION_FILE_DIR'])
 
+# (Re)generate DDG bang filter, and create path if it doesn't exist yet
+if not os.path.exists(app.config['BANG_PATH']):
+    os.makedirs(app.config['BANG_PATH'])
+gen_bangs_json(app.config['BANG_FILE'])
+
 Session(app)
 
 from app import routes

app/routes.py

@@ -18,6 +18,9 @@ from app.request import Request
 from app.utils.session_utils import valid_user_session
 from app.utils.routing_utils import *
 
+# Load DDG bang json files only on init
+bang_json = json.load(open(app.config['BANG_FILE']))
+
 
 def auth_required(f):
     @wraps(f)
@@ -126,6 +129,10 @@ def opensearch():
 def autocomplete():
     q = g.request_params.get('q')
 
+    # Search bangs if the query begins with "!", but not "! " (feeling lucky)
+    if q.startswith('!') and len(q) > 1 and not q.startswith('! '):
+        return jsonify([q, [bang_json[_]['suggestion'] for _ in bang_json if _.startswith(q)]])
+
     if not q and not request.data:
         return jsonify({'?': []})
     elif request.data:
@@ -143,7 +150,7 @@ def search():
     search_util = RoutingUtils(request, g.user_config, session, cookies_disabled=g.cookies_disabled)
     query = search_util.new_search_query()
 
-    resolved_bangs = search_util.bang_operator()
+    resolved_bangs = search_util.bang_operator(bang_json)
     if resolved_bangs != '':
         return redirect(resolved_bangs)
 

app/static/js/autocomplete.js

@@ -93,8 +93,14 @@ const autocomplete = (searchInput, autocompleteResults) => {
         removeActive(suggestion);
         suggestion[currentFocus].classList.add("autocomplete-active");
 
-        // Autofill search bar with suggestion content
-        searchBar.value = suggestion[currentFocus].textContent;
+        // Autofill search bar with suggestion content (minus the "bang name" if using a bang operator)
+        let searchContent = suggestion[currentFocus].textContent;
+        if (searchContent.indexOf('(') > 0) {
+            searchBar.value = searchContent.substring(0, searchContent.indexOf('('));
+        } else {
+            searchBar.value = searchContent;
+        }
+
         searchBar.focus();
     };
 

app/utils/gen_ddg_bangs.py

@@ -0,0 +1,26 @@
+import json
+import requests
+
+
+def gen_bangs_json(bangs_file):
+    # Request list
+    try:
+        r = requests.get('https://duckduckgo.com/bang.v255.js')
+        r.raise_for_status()
+    except requests.exceptions.HTTPError as err:
+        raise SystemExit(err)
+
+    # Convert to json
+    data = json.loads(r.text)
+
+    # Set up a json object (with better formatting) for all available bangs
+    bangs_data = {}
+
+    for row in data:
+        bang_command = '!' + row['t']
+        bangs_data[bang_command] = {
+            'url': row['u'].replace('{{{s}}}', '{}'),
+            'suggestion': bang_command + ' (' + row['s'] + ')'
+        }
+
+    json.dump(bangs_data, open(bangs_file, 'w'))

app/utils/routing_utils.py

@@ -17,14 +17,6 @@ class RoutingUtils:
         self.query = ''
         self.cookies_disabled = cookies_disabled
         self.search_type = self.request_params.get('tbm') if 'tbm' in self.request_params else ''
-        self.bang_operators = {
-            '!gh': 'https://github.com/search?q=',
-            '!ddg': 'https://duckduckgo.com/?q=',
-            '!w': 'https://wikipedia.com/wiki/',
-            '!so': 'https://stackoverflow.com/search?q=',
-            '!a': 'https://amazon.com/s?k=',
-            '!ebay': 'https://ebay.com/sch/i.html?_nkw=',
-        }
 
     def __getitem__(self, name):
         return getattr(self, name)
@@ -63,15 +55,12 @@ class RoutingUtils:
         self.query = q[2:] if self.feeling_lucky else q
         return self.query
 
-
-    def bang_operator(self) -> str:
-        print(self.query)
-        for operator, url in self.bang_operators.items():
-            if operator in self.query:
-                return url + self.query.replace(operator, '').strip()
+    def bang_operator(self, bangs_dict: dict) -> str:
+        for operator in bangs_dict.keys():
+            if self.query.split(' ')[0] == operator:
+                return bangs_dict[operator]['url'].format(self.query.replace(operator, '').strip())
         return ''
 
-
     def generate_response(self) -> Tuple[Any, int]:
         mobile = 'Android' in self.user_agent or 'iPhone' in self.user_agent
 

gen_ops.py

@@ -1,19 +0,0 @@
-import csv, json, sys
-import requests
-import collections
-
-# Request list
-try:
-    r = requests.get('https://duckduckgo.com/bang.v255.js')
-    r.raise_for_status()
-except requests.exceptions.HTTPError as err:
-    raise SystemExit(err)
-
-# Convert to json
-data = json.loads(r.text)
-
-# Output CSV
-output = csv.writer(sys.stdout)
-output.writerow(['tag', 'url', 'domain', 'name'])
-for row in data:
-    output.writerow([row['t'], row['u'], row['d'], row['s']])

test/test_routes.py

@@ -27,6 +27,16 @@ def test_feeling_lucky(client):
     assert rv._status_code == 303
 
 
+def test_ddg_bang(client):
+    rv = client.get('/search?q=!gh%20whoogle')
+    assert rv._status_code == 302
+    assert rv.headers.get('Location').startswith('https://github.com')
+
+    rv = client.get('/search?q=!w%20github')
+    assert rv._status_code == 302
+    assert rv.headers.get('Location').startswith('https://en.wikipedia.org')
+
+
 def test_config(client):
     rv = client.post('/config', data=demo_config)
     assert rv._status_code == 302