From b5b6e641778c39a2f4e2362e69d7326e2133d289 Mon Sep 17 00:00:00 2001
From: Ben Busby <ben.busby7@gmail.com>
Date: Wed, 15 Apr 2020 17:41:53 -0600
Subject: [PATCH] Added testing and ci build, refactored filter class,
 refactored project structure

---
 .gitignore                                  |   1 +
 .travis.yml                                 |   6 +
 Dockerfile                                  |   4 +-
 README.md                                   |  20 ++-
 app/filter.py                               | 179 ++++++++++----------
 app/routes.py                               |  37 ++--
 opensearch.py => config/opensearch.py       |   6 +-
 requirements.txt => config/requirements.txt |   1 +
 run                                         |  33 ++++
 run.sh                                      |  17 --
 server.py                                   |   1 -
 test/__init__.py                            |   0
 test/conftest.py                            |   8 +
 test/test_results.py                        |  54 ++++++
 test/test_routes.py                         |  30 ++++
 15 files changed, 269 insertions(+), 128 deletions(-)
 create mode 100644 .travis.yml
 rename opensearch.py => config/opensearch.py (73%)
 rename requirements.txt => config/requirements.txt (94%)
 create mode 100755 run
 delete mode 100755 run.sh
 delete mode 100644 server.py
 create mode 100644 test/__init__.py
 create mode 100644 test/conftest.py
 create mode 100644 test/test_results.py
 create mode 100644 test/test_routes.py

diff --git a/.gitignore b/.gitignore
index 469cf3d..1c70601 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,3 +5,4 @@ __pycache__/
 *.pem
 *.xml
 config.json
+test/static
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..b9200fe
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,6 @@
+language: python
+python: 3.6
+install:
+    - pip install -r config/requirements.txt
+script:
+    - ./run test
diff --git a/Dockerfile b/Dockerfile
index 2007e76..dc0e92c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -3,6 +3,6 @@ FROM python:3
 WORKDIR /usr/src/app
 COPY . .
 
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r config/requirements.txt
 
-CMD ["./run.sh"]
+CMD ["./run"]
diff --git a/README.md b/README.md
index db6340e..df38f69 100644
--- a/README.md
+++ b/README.md
@@ -29,14 +29,26 @@ heroku open
 Now you're done! This series of commands can take a while, but once you run it once, you shouldn't have to run it again. The final command, `heroku open` will launch a tab in your web browser, where you can test out Shoogle and even [set it as your primary search engine](https://github.com/benbusby/shoogle#set-shoogle-as-your-primary-search-engine).
 
 #### B) Using your own server, or alternative container deployment
-There are other methods for deploying docker containers that are well outlined in [this article](https://rollout.io/blog/the-shortlist-of-docker-hosting/), but there are too many to describe set up for each here. Generally it should be about the same amount of effort as the Heroku deployment. 
+There are other methods for deploying docker containers that are well outlined in [this article](https://rollout.io/blog/the-shortlist-of-docker-hosting/), but there are too many to describe set up for each here. Generally it should be about the same amount of effort as the Heroku deployment.
 
 Depending on your preferences, you can also deploy the app yourself on your own infrastructure. This route would require a few extra steps:
   - A server (I personally recommend [Digital Ocean](https://www.digitalocean.com/pricing/) or [Linode](https://www.linode.com/pricing/), their cheapest tiers will work fine)
   - Your own URL (I suppose this is optional, but recommended)
   - SSL certificates (free through [Let's Encrypt](https://letsencrypt.org/getting-started/))
   - A bit more experience or willingness to work through issues
-  
+
+## Setup (Local Only)
+If you want to test the app out on your own machine first, you can build it with the following instructions:
+
+```bash
+git clone https://github.com/benbusby/shoogle.git
+cd shoogle
+python3 -m venv venv
+source venv/bin/activate
+pip install -r config/requirements.txt
+./run
+```
+
 ## Usage
 Same as most search engines, with the exception of filtering by time range.
 
@@ -44,7 +56,7 @@ To filter by a range of time, append ":past <time>" to the end of your search, w
 
 ## Extra Steps
 ### Set Shoogle as your primary search engine
-1. From the main shoogle folder, run `python opensearch.py "<your app url>"`
+1. From the main shoogle folder, run `python config/opensearch.py "<your app url>"`
 2. Rebuild and release your updated app
   - `heroku container:push web` and then `heroku container:release web`
 3. Update browser settings
@@ -69,4 +81,4 @@ Part of the deal with Heroku's free tier is that you're allocated 550 hours/mont
 
 A good solution for this is to set up a simple cronjob on any device at your home that is consistently powered on and connected to the internet (in my case, a PiHole worked perfectly). All the device needs to do is fetch app content on a consistent basis to keep the app alive in whatever ~17 hour window you want it on (17 hrs * 31 days = 527, meaning you'd still have 23 leftover hours each month if you searched outside of your target window).
 
-For instance: `*/20 7-23 * * * curl https://<your heroku app name>.herokuapp.com > /home/<username>/shoogle-refresh` will fetch the home page of the app every 20 minutes between 7am and midnight, allowing for downtime from midnight to 7am. And again, this wouldn't be a hard limit - you'd still have plenty of remaining hours of uptime each month in case you were searching after this window has closed. 
+For instance: `*/20 7-23 * * * curl https://<your heroku app name>.herokuapp.com > /home/<username>/shoogle-refresh` will fetch the home page of the app every 20 minutes between 7am and midnight, allowing for downtime from midnight to 7am. And again, this wouldn't be a hard limit - you'd still have plenty of remaining hours of uptime each month in case you were searching after this window has closed.
diff --git a/app/filter.py b/app/filter.py
index 7e845d3..1fd8c1b 100644
--- a/app/filter.py
+++ b/app/filter.py
@@ -3,109 +3,118 @@ import re
 import urllib.parse as urlparse
 from urllib.parse import parse_qs
 
-AD_CLASS = 'ZINbbc'
-SPONS_CLASS = 'D1fz0e'
 
+class Filter:
+    def __init__(self, mobile=False, config=None):
+        self.mobile = False
+        self.dark_mode = False
+        self.nojs = False
+        self.near_city = None
 
-def reskin(page, dark_mode=False):
-    # Aesthetic only re-skinning
-    page = page.replace('>G<', '>Sh<')
-    pattern = re.compile('4285f4|ea4335|fbcc05|34a853|fbbc05', re.IGNORECASE)
-    page = pattern.sub('685e79', page)
-    if dark_mode:
-        page = page.replace('fff', '000').replace('202124', 'ddd').replace('1967D2', '3b85ea')
+        if config is None:
+            config = {}
 
-    return page
+        near_city = config['near'] if 'near' in config else None
+        dark_mode = config['dark_mode'] if 'dark_mode' in config else False
+        nojs = config['nojs'] if 'nojs' in config else False
+        mobile = mobile
 
+    def reskin(self, page):
+        # Aesthetic only re-skinning
+        page = page.replace('>G<', '>Sh<')
+        pattern = re.compile('4285f4|ea4335|fbcc05|34a853|fbbc05', re.IGNORECASE)
+        page = pattern.sub('685e79', page)
+        if self.dark_mode:
+            page = page.replace('fff', '000').replace('202124', 'ddd').replace('1967D2', '3b85ea')
 
-def gen_query(q, args, near_city=None):
-    # Use :past(hour/day/week/month/year) if available
-    # example search "new restaurants :past month"
-    tbs = ''
-    # if 'tbs' in request.args:
-    #     tbs = '&tbs=' + request.args.get('tbs')
-    #     q = q.replace(q.split(':past', 1)[-1], '').replace(':past', '')
-    if ':past' in q:
-        time_range = str.strip(q.split(':past', 1)[-1])
-        tbs = '&tbs=qdr:' + str.lower(time_range[0])
+        return page
 
-    # Ensure search query is parsable
-    q = urlparse.quote(q)
+    def gen_query(self, q, args):
+        # Use :past(hour/day/week/month/year) if available
+        # example search "new restaurants :past month"
+        tbs = ''
+        if ':past' in q:
+            time_range = str.strip(q.split(':past', 1)[-1])
+            tbs = '&tbs=qdr:' + str.lower(time_range[0])
 
-    # Pass along type of results (news, images, books, etc)
-    tbm = ''
-    if 'tbm' in args:
-        tbm = '&tbm=' + args.get('tbm')
+        # Ensure search query is parsable
+        q = urlparse.quote(q)
 
-    # Get results page start value (10 per page, ie page 2 start val = 20)
-    start = ''
-    if 'start' in args:
-        start = '&start=' + args.get('start')
+        # Pass along type of results (news, images, books, etc)
+        tbm = ''
+        if 'tbm' in args:
+            tbm = '&tbm=' + args.get('tbm')
 
-    # Grab city from config, if available
-    near = ''
-    if near_city:
-        near = '&near=' + urlparse.quote(near_city)
+        # Get results page start value (10 per page, ie page 2 start val = 20)
+        start = ''
+        if 'start' in args:
+            start = '&start=' + args.get('start')
 
-    return q + tbs + tbm + start + near
+        # Grab city from config, if available
+        near = ''
+        if self.near_city:
+            near = '&near=' + urlparse.quote(self.near_city)
 
+        return q + tbs + tbm + start + near
 
-def cook(soup, user_agent, nojs=False, dark_mode=False):
-    # Remove all ads (TODO: Ad specific div classes probably change over time, look into a more generic method)
-    main_divs = soup.find('div', {'id': 'main'})
-    if main_divs is not None:
-        ad_divs = main_divs.findAll('div', {'class': AD_CLASS}, recursive=False)
-        sponsored_divs = main_divs.findAll('div', {'class': SPONS_CLASS}, recursive=False)
-        for div in ad_divs + sponsored_divs:
-            div.decompose()
+    def clean(self, soup):
+        # Remove all ads
+        main_divs = soup.find('div', {'id': 'main'})
+        if main_divs is not None:
+            result_divs = main_divs.findAll('div', recursive=False)
 
-    # Remove unnecessary button(s)
-    for button in soup.find_all('button'):
-        button.decompose()
+            # Only ads/sponsored content use classes in the list of result divs
+            ad_divs = [ad_div for ad_div in result_divs if 'class' in ad_div.attrs]
+            for div in ad_divs:
+                div.decompose()
 
-    # Remove svg logos
-    for svg in soup.find_all('svg'):
-        svg.decompose()
+        # Remove unnecessary button(s)
+        for button in soup.find_all('button'):
+            button.decompose()
 
-    # Update logo
-    logo = soup.find('a', {'class': 'l'})
-    if logo is not None and ('Android' in user_agent or 'iPhone' in user_agent):
-        logo.insert(0, 'Shoogle')
-        logo['style'] = 'display: flex;justify-content: center;align-items: center;color: #685e79;font-size: 18px;'
+        # Remove svg logos
+        for svg in soup.find_all('svg'):
+            svg.decompose()
 
-    # Replace hrefs with only the intended destination (no "utm" type tags)
-    for a in soup.find_all('a', href=True):
-        href = a['href']
-        if '/advanced_search' in href:
-            a.decompose()
-            continue
+        # Update logo
+        logo = soup.find('a', {'class': 'l'})
+        if logo is not None and self.mobile:
+            logo.insert(0, 'Shoogle')
+            logo['style'] = 'display: flex;justify-content: center;align-items: center;color: #685e79;font-size: 18px;'
 
-        if 'url?q=' in href:
-            # Strip unneeded arguments
-            href = urlparse.urlparse(href)
-            href = parse_qs(href.query)['q'][0]
+        # Replace hrefs with only the intended destination (no "utm" type tags)
+        for a in soup.find_all('a', href=True):
+            href = a['href']
+            if '/advanced_search' in href:
+                a.decompose()
+                continue
 
-            # Add no-js option
-            if nojs:
-                nojs_link = soup.new_tag('a')
-                nojs_link['href'] = '/window?location=' + href
-                nojs_link['style'] = 'display:block;width:100%;'
-                nojs_link.string = 'NoJS Link: ' + nojs_link['href']
-                a.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
-                a.append(nojs_link)
+            if 'url?q=' in href:
+                # Strip unneeded arguments
+                href = urlparse.urlparse(href)
+                href = parse_qs(href.query)['q'][0]
 
-    # Set up dark mode if active
-    if dark_mode:
-        soup.find('html')['style'] = 'scrollbar-color: #333 #111;'
-        for input_element in soup.findAll('input'):
-            input_element['style'] = 'color:#fff;'
+                # Add no-js option
+                if self.nojs:
+                    nojs_link = soup.new_tag('a')
+                    nojs_link['href'] = '/window?location=' + href
+                    nojs_link['style'] = 'display:block;width:100%;'
+                    nojs_link.string = 'NoJS Link: ' + nojs_link['href']
+                    a.append(BeautifulSoup('<br><hr><br>', 'html.parser'))
+                    a.append(nojs_link)
 
-    # Ensure no extra scripts passed through
-    try:
-        for script in soup('script'):
-            script.decompose()
-        soup.find('div', id='sfooter').decompose()
-    except Exception:
-        pass
+        # Set up dark mode if active
+        if self.dark_mode:
+            soup.find('html')['style'] = 'scrollbar-color: #333 #111;'
+            for input_element in soup.findAll('input'):
+                input_element['style'] = 'color:#fff;'
 
-    return soup
+        # Ensure no extra scripts passed through
+        try:
+            for script in soup('script'):
+                script.decompose()
+            soup.find('div', id='sfooter').decompose()
+        except Exception:
+            pass
+
+        return soup
diff --git a/app/routes.py b/app/routes.py
index adedd90..ac82867 100644
--- a/app/routes.py
+++ b/app/routes.py
@@ -1,4 +1,5 @@
-from app import app, rhyme, filter
+from app import app, rhyme
+from app.filter import Filter
 from bs4 import BeautifulSoup
 from flask import request, redirect, render_template
 from io import BytesIO
@@ -7,8 +8,8 @@ import os
 import pycurl
 import urllib.parse as urlparse
 
-APP_ROOT = os.path.dirname(os.path.abspath(__file__))
-STATIC_FOLDER = os.path.join(APP_ROOT, 'static')
+app.config['APP_ROOT'] = os.getenv('APP_ROOT', os.path.dirname(os.path.abspath(__file__)))
+app.config['STATIC_FOLDER'] = os.getenv('STATIC_FOLDER', os.path.join(app.config['APP_ROOT'], 'static'))
 
 # Get Mozilla Firefox rhyme (important) and form a new user agent
 mozilla = rhyme.get_rhyme('Mo') + 'zilla'
@@ -20,7 +21,7 @@ DESKTOP_UA = mozilla + '/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/2010010
 # Base search url
 SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
 
-user_config = json.load(open(STATIC_FOLDER + '/config.json'))
+user_config = json.load(open(app.config['STATIC_FOLDER'] + '/config.json'))
 
 
 def get_ua(user_agent):
@@ -55,29 +56,31 @@ def search():
     if q is None or len(q) <= 0:
         return render_template('error.html')
 
-    full_query = filter.gen_query(q, request.args)
     user_agent = request.headers.get('User-Agent')
-    dark_mode = 'dark' in user_config and user_config['dark']
-    nojs = 'nojs' in user_config and user_config['nojs']
+    mobile = 'Android' in user_agent or 'iPhone' in user_agent
 
-    get_body = filter.reskin(send_request(
-        SEARCH_URL + full_query, get_ua(user_agent)), dark_mode=dark_mode)
-
-    soup = filter.cook(BeautifulSoup(get_body, 'html.parser'), user_agent, nojs=nojs, dark_mode=dark_mode)
+    content_filter = Filter(mobile, user_config)
+    full_query = content_filter.gen_query(q, request.args)
+    get_body = send_request(SEARCH_URL + full_query, get_ua(user_agent))
+    get_body = content_filter.reskin(get_body)
+    soup = content_filter.clean(BeautifulSoup(get_body, 'html.parser'))
 
     return render_template('display.html', query=urlparse.unquote(q), response=soup)
 
 
-@app.route('/config', methods=['POST'])
+@app.route('/config', methods=['GET', 'POST'])
 def config():
     global user_config
-    with open(STATIC_FOLDER + '/config.json', 'w') as config_file:
-        config_file.write(json.dumps(json.loads(request.data), indent=4))
-        config_file.close()
+    if request.method == 'GET':
+        return json.dumps(user_config)
+    else:
+        with open(app.config['STATIC_FOLDER'] + '/config.json', 'w') as config_file:
+            config_file.write(json.dumps(json.loads(request.data), indent=4))
+            config_file.close()
 
-        user_config = json.loads(request.data)
+            user_config = json.loads(request.data)
 
-    return 'New config: ' + str(request.data)
+        return 'New config: ' + str(request.data)
 
 
 @app.route('/url', methods=['GET'])
diff --git a/opensearch.py b/config/opensearch.py
similarity index 73%
rename from opensearch.py
rename to config/opensearch.py
index 420c776..31d612e 100644
--- a/opensearch.py
+++ b/config/opensearch.py
@@ -1,7 +1,9 @@
+import os
 import sys
 
-template_path = './app/static/opensearch.template'
-opensearch_path = './app/static/opensearch.xml'
+script_path = os.path.dirname(os.path.realpath(__file__))
+template_path = script_path + '/../app/static/opensearch.template'
+opensearch_path = script_path + '/../app/static/opensearch.xml'
 replace_tag = 'SHOOGLE_URL'
 
 if len(sys.argv) != 2:
diff --git a/requirements.txt b/config/requirements.txt
similarity index 94%
rename from requirements.txt
rename to config/requirements.txt
index ceb2637..2abc3b7 100644
--- a/requirements.txt
+++ b/config/requirements.txt
@@ -11,6 +11,7 @@ Phyme==0.0.9
 pycparser==2.19
 pycurl==7.43.0.4
 pyOpenSSL==19.1.0
+pytest==5.4.1
 six==1.14.0
 soupsieve==1.9.5
 Werkzeug==0.16.0
diff --git a/run b/run
new file mode 100755
index 0000000..289dce9
--- /dev/null
+++ b/run
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Usage:
+# ./run # Runs the full web app
+# ./run test # Runs the testing suite
+
+SCRIPT=`realpath $0`
+SCRIPT_DIR=`dirname $SCRIPT`
+
+# Set default port if unavailable
+if [[ -z "${PORT}" ]]; then
+    PORT=5000
+fi
+
+# Set directory to serve static content from
+[[ ! -z $1 ]] && SUBDIR="$1" || SUBDIR="app"
+export APP_ROOT=$SCRIPT_DIR/$SUBDIR
+export STATIC_FOLDER=$APP_ROOT/static
+
+mkdir -p $STATIC_FOLDER
+
+# Create default config json if it doesn't exist
+if [[ ! -f $STATIC_FOLDER/config.json ]]; then
+    echo "{}" > $STATIC_FOLDER/config.json
+fi
+
+pkill flask
+
+# Check for regular vs test run
+if [[ $SUBDIR == "test" ]]; then
+    pytest -sv
+else
+    flask run --host="0.0.0.0" --port=$PORT
+fi
diff --git a/run.sh b/run.sh
deleted file mode 100755
index f6e1ac4..0000000
--- a/run.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/bin/bash
-
-SCRIPT=`realpath $0`
-SCRIPT_DIR=`dirname $SCRIPT`
-
-if [[ -z "${PORT}" ]]; then
-    PORT=5000
-fi
-
-# Create config json if it doesn't exist
-if [[ ! -f $SCRIPT_DIR/app/static/config.json ]]; then
-    echo "{}" > $SCRIPT_DIR/app/static/config.json
-fi
-
-pkill flask
-
-flask run --host="0.0.0.0" --port=$PORT
diff --git a/server.py b/server.py
deleted file mode 100644
index d099b92..0000000
--- a/server.py
+++ /dev/null
@@ -1 +0,0 @@
-from app import app
diff --git a/test/__init__.py b/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/test/conftest.py b/test/conftest.py
new file mode 100644
index 0000000..3d2aa33
--- /dev/null
+++ b/test/conftest.py
@@ -0,0 +1,8 @@
+from app import app
+import pytest
+
+
+@pytest.fixture
+def client():
+    client = app.test_client()
+    yield client
diff --git a/test/test_results.py b/test/test_results.py
new file mode 100644
index 0000000..9ab32e1
--- /dev/null
+++ b/test/test_results.py
@@ -0,0 +1,54 @@
+from bs4 import BeautifulSoup
+from app.filter import Filter
+import json
+from datetime import datetime
+from dateutil.parser import *
+from test.conftest import client
+
+
+def get_search_results(data):
+    soup = Filter().clean(BeautifulSoup(rv.data, 'html.parser'))
+
+    main_divs = soup.find('div', {'id': 'main'})
+    assert len(main_divs) > 1
+
+    result_divs = []
+    for div in main_divs:
+        # Result divs should only have 1 inner div
+        if len(list(div.children)) != 1 or not div.findChild() or 'div' not in div.findChild().name:
+            continue
+
+        result_divs.append(div)
+
+    return result_divs
+
+
+def test_search_results(client):
+    rv = client.get('/search?q=test')
+    assert rv._status_code == 200
+
+    assert len(get_search_results(rv.data)) == 10
+
+
+def test_recent_results(client):
+    times = {
+        'pastyear': 365,
+        'pastmonth': 31,
+        'pastweek': 7
+    }
+
+    for time, num_days in times.items():
+        rv = client.get('/search?q=test%20%3A' + time)
+        result_divs = get_search_results(rv.data)
+
+        current_date = datetime.now()
+        for div in result_divs:
+            date_span = div.find('span').decode_contents()
+            if not date_span or len(date_span) > 15:
+                continue
+
+            try:
+                date = parse(date_span)
+                assert (current_date - date).days < num_days
+            except ParserError:
+                assert ' ago' in date_span
diff --git a/test/test_routes.py b/test/test_routes.py
new file mode 100644
index 0000000..bcf9991
--- /dev/null
+++ b/test/test_routes.py
@@ -0,0 +1,30 @@
+import json
+from test.conftest import client
+
+demo_config = {
+    'near': 'Seattle',
+    'dark_mode': 0,
+    'nojs': 0
+}
+
+
+def test_main(client):
+    rv = client.get('/')
+    assert rv._status_code == 200
+
+
+def test_search(client):
+    rv = client.get('/search?q=test')
+    assert rv._status_code == 200
+
+
+def test_config(client):
+    rv = client.post('/config', data=json.dumps(demo_config))
+    assert rv._status_code == 200
+
+    rv = client.get('/config')
+    assert rv._status_code == 200
+
+    config = json.loads(rv.data)
+    for key in demo_config.keys():
+        assert config[key] == demo_config[key]