Move bangs init to bg thread

Initializing the DDG bangs when running whoogle for the first time
creates an indeterminate amount of delay before the app becomes usable,
which makes usability tests (particularly w/ Docker) unreliable. This
moves the bang json init to a background thread and writes a temporary
empty dict to the bangs json file until the full bangs json can be used.

.github/workflows/docker_main.yml

@@ -17,11 +17,12 @@ jobs:
     - name: build and test (docker)
       run: |
         docker build --tag whoogle-search:test .
-        docker run --publish 5000:5000 --detach --name whoogle-search whoogle-search:test
+        docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test
         sleep 15
-        docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1
+        docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
     - name: build and test (docker-compose)
       run: |
+        docker rm -f whoogle-search-nocompose
         docker-compose up --detach
         sleep 15
         docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1

.github/workflows/docker_tests.yml

@@ -20,6 +20,7 @@ jobs:
         docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
     - name: build and test (docker-compose)
       run: |
+        docker rm -f whoogle-search-nocompose
         docker-compose up --detach
         sleep 15
         docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1

app/__init__.py

@@ -9,6 +9,7 @@ import json
 import logging.config
 import os
 from stem import Signal
+import threading
 from dotenv import load_dotenv
 
 app = Flask(__name__, static_folder=os.path.dirname(
@@ -97,7 +98,11 @@ if not os.path.exists(app.config['SESSION_FILE_DIR']):
 if not os.path.exists(app.config['BANG_PATH']):
     os.makedirs(app.config['BANG_PATH'])
 if not os.path.exists(app.config['BANG_FILE']):
-    gen_bangs_json(app.config['BANG_FILE'])
+    json.dump({}, open(app.config['BANG_FILE'], 'w'))
+    bangs_thread = threading.Thread(
+        target=gen_bangs_json,
+        args=(app.config['BANG_FILE'],))
+    bangs_thread.start()
 
 # Build new mapping of static files for cache busting
 if not os.path.exists(app.config['BUILD_FOLDER']):

app/routes.py

@@ -2,6 +2,7 @@ import argparse
 import base64
 import io
 import json
+import os
 import pickle
 import urllib.parse as urlparse
 import uuid
@@ -26,7 +27,7 @@ from requests import exceptions, get
 from requests.models import PreparedRequest
 
 # Load DDG bang json files only on init
-bang_json = json.load(open(app.config['BANG_FILE']))
+bang_json = json.load(open(app.config['BANG_FILE'])) or {}
 
 # Check the newest version of WHOOGLE
 update = bsoup(get(app.config['RELEASES_URL']).text, 'html.parser')
@@ -101,6 +102,8 @@ def session_required(f):
 
 @app.before_request
 def before_request_func():
+    global bang_json
+
     g.request_params = (
         request.args if request.method == 'GET' else request.form
     )
@@ -150,6 +153,15 @@ def before_request_func():
 
     g.app_location = g.user_config.url
 
+    # Attempt to reload bangs json if not generated yet
+    if not bang_json and os.path.getsize(app.config['BANG_FILE']) > 4:
+        try:
+            bang_json = json.load(open(app.config['BANG_FILE']))
+        except json.decoder.JSONDecodeError:
+            # Ignore decoding error, can occur if file is still 
+            # being written
+            pass
+
 
 @app.after_request
 def after_request_func(resp):

app/utils/bangs.py

@@ -35,6 +35,7 @@ def gen_bangs_json(bangs_file: str) -> None:
         }
 
     json.dump(bangs_data, open(bangs_file, 'w'))
+    print('* Finished creating ddg bangs json')
 
 
 def resolve_bang(query: str, bangs_dict: dict) -> str: