From 8c426ab1803049c61c91070abeeba53512982aa1 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Tue, 7 Mar 2023 11:28:55 -0700 Subject: [PATCH] Suppress invalid warning from bs4, add 404 handler An invalid parsing warning was being thrown by the latest version of the bs4 library. This suppresses that warning from being shown in the console. A 404 handler was added to move logging from the console to the error template, since a lot of users assumed that 404 errors from the result page were problems with Whoogle itself. Fixes #967 --- app/routes.py | 5 +++++ app/utils/results.py | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/routes.py b/app/routes.py index 3b074fa..05abfe8 100644 --- a/app/routes.py +++ b/app/routes.py @@ -557,6 +557,11 @@ def window(): ) +@app.errorhandler(404) +def page_not_found(e): + return render_template('error.html', error_message=str(e)), 404 + + def run_app() -> None: parser = argparse.ArgumentParser( description='Whoogle Search console runner') diff --git a/app/utils/results.py b/app/utils/results.py index 5a76dc8..82c4d2b 100644 --- a/app/utils/results.py +++ b/app/utils/results.py @@ -1,6 +1,6 @@ from app.models.config import Config from app.models.endpoint import Endpoint -from bs4 import BeautifulSoup, NavigableString +from bs4 import BeautifulSoup, NavigableString, MarkupResemblesLocatorWarning import copy from flask import current_app import html @@ -8,6 +8,10 @@ import os import urllib.parse as urlparse from urllib.parse import parse_qs import re +import warnings + +# Suppress incorrect warnings from bs4 related to parsing HTML content +warnings.filterwarnings('ignore', category=MarkupResemblesLocatorWarning) SKIP_ARGS = ['ref_src', 'utm'] SKIP_PREFIX = ['//www.', '//mobile.', '//m.', 'www.', 'mobile.', 'm.']