From 72e5a227c83a06117257d63a073e0ac8e50ed30e Mon Sep 17 00:00:00 2001
From: Ben Busby <contact@benbusby.com>
Date: Tue, 25 Jan 2022 12:28:06 -0700
Subject: [PATCH] Move bangs init to bg thread

Initializing the DDG bangs when running whoogle for the first time
creates an indeterminate amount of delay before the app becomes usable,
which makes usability tests (particularly w/ Docker) unreliable. This
moves the bang json init to a background thread and writes a temporary
empty dict to the bangs json file until the full bangs json can be used.
---
 .github/workflows/docker_main.yml  |  5 +++--
 .github/workflows/docker_tests.yml |  1 +
 app/__init__.py                    |  7 ++++++-
 app/routes.py                      | 14 +++++++++++++-
 app/utils/bangs.py                 |  1 +
 5 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/docker_main.yml b/.github/workflows/docker_main.yml
index 754ff44..c4ab678 100644
--- a/.github/workflows/docker_main.yml
+++ b/.github/workflows/docker_main.yml
@@ -17,11 +17,12 @@ jobs:
     - name: build and test (docker)
       run: |
         docker build --tag whoogle-search:test .
-        docker run --publish 5000:5000 --detach --name whoogle-search whoogle-search:test
+        docker run --publish 5000:5000 --detach --name whoogle-search-nocompose whoogle-search:test
         sleep 15
-        docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1
+        docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
     - name: build and test (docker-compose)
       run: |
+        docker rm -f whoogle-search-nocompose
         docker-compose up --detach
         sleep 15
         docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1
diff --git a/.github/workflows/docker_tests.yml b/.github/workflows/docker_tests.yml
index f0d7deb..7eabdd9 100644
--- a/.github/workflows/docker_tests.yml
+++ b/.github/workflows/docker_tests.yml
@@ -20,6 +20,7 @@ jobs:
         docker exec whoogle-search-nocompose curl -f http://localhost:5000/healthz || exit 1
     - name: build and test (docker-compose)
       run: |
+        docker rm -f whoogle-search-nocompose
         docker-compose up --detach
         sleep 15
         docker exec whoogle-search curl -f http://localhost:5000/healthz || exit 1
diff --git a/app/__init__.py b/app/__init__.py
index 82c66fb..99daa01 100644
--- a/app/__init__.py
+++ b/app/__init__.py
@@ -9,6 +9,7 @@ import json
 import logging.config
 import os
 from stem import Signal
+import threading
 from dotenv import load_dotenv
 
 app = Flask(__name__, static_folder=os.path.dirname(
@@ -97,7 +98,11 @@ if not os.path.exists(app.config['SESSION_FILE_DIR']):
 if not os.path.exists(app.config['BANG_PATH']):
     os.makedirs(app.config['BANG_PATH'])
 if not os.path.exists(app.config['BANG_FILE']):
-    gen_bangs_json(app.config['BANG_FILE'])
+    json.dump({}, open(app.config['BANG_FILE'], 'w'))
+    bangs_thread = threading.Thread(
+        target=gen_bangs_json,
+        args=(app.config['BANG_FILE'],))
+    bangs_thread.start()
 
 # Build new mapping of static files for cache busting
 if not os.path.exists(app.config['BUILD_FOLDER']):
diff --git a/app/routes.py b/app/routes.py
index 0cd9ecb..253db21 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -2,6 +2,7 @@ import argparse
 import base64
 import io
 import json
+import os
 import pickle
 import urllib.parse as urlparse
 import uuid
@@ -26,7 +27,7 @@ from requests import exceptions, get
 from requests.models import PreparedRequest
 
 # Load DDG bang json files only on init
-bang_json = json.load(open(app.config['BANG_FILE']))
+bang_json = json.load(open(app.config['BANG_FILE'])) or {}
 
 # Check the newest version of WHOOGLE
 update = bsoup(get(app.config['RELEASES_URL']).text, 'html.parser')
@@ -101,6 +102,8 @@ def session_required(f):
 
 @app.before_request
 def before_request_func():
+    global bang_json
+
     g.request_params = (
         request.args if request.method == 'GET' else request.form
     )
@@ -150,6 +153,15 @@ def before_request_func():
 
     g.app_location = g.user_config.url
 
+    # Attempt to reload bangs json if not generated yet
+    if not bang_json and os.path.getsize(app.config['BANG_FILE']) > 4:
+        try:
+            bang_json = json.load(open(app.config['BANG_FILE']))
+        except json.decoder.JSONDecodeError:
+            # Ignore decoding error, can occur if file is still 
+            # being written
+            pass
+
 
 @app.after_request
 def after_request_func(resp):
diff --git a/app/utils/bangs.py b/app/utils/bangs.py
index 23b3c04..4a1a3cb 100644
--- a/app/utils/bangs.py
+++ b/app/utils/bangs.py
@@ -35,6 +35,7 @@ def gen_bangs_json(bangs_file: str) -> None:
         }
 
     json.dump(bangs_data, open(bangs_file, 'w'))
+    print('* Finished creating ddg bangs json')
 
 
 def resolve_bang(query: str, bangs_dict: dict) -> str: