From 09c53b52afd8d7ecbc08ea61b502c3f6fed7c060 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Sat, 23 May 2020 14:27:23 -0600 Subject: [PATCH] Feature: country and safe search config options (#71) * Added country and safe search config options * Updated handling of parser error in results test * Improved handling of default country * Added 1px empty gif fallback as a replacement for images that fail to load --- app/models/config.py | 267 ++++++++++++++++++++++++++++++++++-- app/request.py | 8 +- app/routes.py | 33 +++-- app/static/js/controller.js | 2 + app/templates/index.html | 17 +++ test/test_results.py | 2 +- 6 files changed, 304 insertions(+), 25 deletions(-) diff --git a/app/models/config.py b/app/models/config.py index 1a53049..0976400 100644 --- a/app/models/config.py +++ b/app/models/config.py @@ -1,4 +1,3 @@ - class Config: # Derived from here: # https://sites.google.com/site/tomihasa/google-language-codes#searchlanguage @@ -27,14 +26,14 @@ class Config: {'name': 'Hindi', 'value': 'lang_hi'}, {'name': 'Hungarian', 'value': 'lang_hu'}, {'name': 'Icelandic', 'value': 'lang_is'}, - {'name': 'Indonesian', 'value': 'lang_id'}, - {'name': 'Italian', 'value': 'lang_it'}, - {'name': 'Japanese', 'value': 'lang_ja'}, - {'name': 'Korean', 'value': 'lang_ko'}, - {'name': 'Latvian', 'value': 'lang_lv'}, - {'name': 'Lithuanian', 'value': 'lang_lt'}, - {'name': 'Norwegian', 'value': 'lang_no'}, - {'name': 'Persian', 'value': 'lang_fa'}, + {'name': 'Indonesian', 'value': 'lang_id'}, + {'name': 'Italian', 'value': 'lang_it'}, + {'name': 'Japanese', 'value': 'lang_ja'}, + {'name': 'Korean', 'value': 'lang_ko'}, + {'name': 'Latvian', 'value': 'lang_lv'}, + {'name': 'Lithuanian', 'value': 'lang_lt'}, + {'name': 'Norwegian', 'value': 'lang_no'}, + {'name': 'Persian', 'value': 'lang_fa'}, {'name': 'Polish', 'value': 'lang_pl'}, {'name': 'Portuguese', 'value': 'lang_pt'}, {'name': 'Romanian', 'value': 'lang_ro'}, @@ -51,9 +50,257 @@ class Config: {'name': 'Vietnamese', 'value': 'lang_vi'}, ] + COUNTRIES = [ + {'name': 'Default (use server location)', 'value': ''}, + {'name': 'Afghanistan', 'value': 'countryAF'}, + {'name': 'Albania', 'value': 'countryAL'}, + {'name': 'Algeria', 'value': 'countryDZ'}, + {'name': 'American Samoa', 'value': 'countryAS'}, + {'name': 'Andorra', 'value': 'countryAD'}, + {'name': 'Angola', 'value': 'countryAO'}, + {'name': 'Anguilla', 'value': 'countryAI'}, + {'name': 'Antarctica', 'value': 'countryAQ'}, + {'name': 'Antigua and Barbuda', 'value': 'countryAG'}, + {'name': 'Argentina', 'value': 'countryAR'}, + {'name': 'Armenia', 'value': 'countryAM'}, + {'name': 'Aruba', 'value': 'countryAW'}, + {'name': 'Australia', 'value': 'countryAU'}, + {'name': 'Austria', 'value': 'countryAT'}, + {'name': 'Azerbaijan', 'value': 'countryAZ'}, + {'name': 'Bahamas', 'value': 'countryBS'}, + {'name': 'Bahrain', 'value': 'countryBH'}, + {'name': 'Bangladesh', 'value': 'countryBD'}, + {'name': 'Barbados', 'value': 'countryBB'}, + {'name': 'Belarus', 'value': 'countryBY'}, + {'name': 'Belgium', 'value': 'countryBE'}, + {'name': 'Belize', 'value': 'countryBZ'}, + {'name': 'Benin', 'value': 'countryBJ'}, + {'name': 'Bermuda', 'value': 'countryBM'}, + {'name': 'Bhutan', 'value': 'countryBT'}, + {'name': 'Bolivia', 'value': 'countryBO'}, + {'name': 'Bosnia and Herzegovina', 'value': 'countryBA'}, + {'name': 'Botswana', 'value': 'countryBW'}, + {'name': 'Bouvet Island', 'value': 'countryBV'}, + {'name': 'Brazil', 'value': 'countryBR'}, + {'name': 'British Indian Ocean Territory', 'value': 'countryIO'}, + {'name': 'Brunei Darussalam', 'value': 'countryBN'}, + {'name': 'Bulgaria', 'value': 'countryBG'}, + {'name': 'Burkina Faso', 'value': 'countryBF'}, + {'name': 'Burundi', 'value': 'countryBI'}, + {'name': 'Cambodia', 'value': 'countryKH'}, + {'name': 'Cameroon', 'value': 'countryCM'}, + {'name': 'Canada', 'value': 'countryCA'}, + {'name': 'Cape Verde', 'value': 'countryCV'}, + {'name': 'Cayman Islands', 'value': 'countryKY'}, + {'name': 'Central African Republic', 'value': 'countryCF'}, + {'name': 'Chad', 'value': 'countryTD'}, + {'name': 'Chile', 'value': 'countryCL'}, + {'name': 'China', 'value': 'countryCN'}, + {'name': 'Christmas Island', 'value': 'countryCX'}, + {'name': 'Cocos (Keeling) Islands', 'value': 'countryCC'}, + {'name': 'Colombia', 'value': 'countryCO'}, + {'name': 'Comoros', 'value': 'countryKM'}, + {'name': 'Congo', 'value': 'countryCG'}, + {'name': 'Congo, Democratic Republic of the', 'value': 'countryCD'}, + {'name': 'Cook Islands', 'value': 'countryCK'}, + {'name': 'Costa Rica', 'value': 'countryCR'}, + {'name': 'Cote D\'ivoire', 'value': 'countryCI'}, + {'name': 'Croatia (Hrvatska)', 'value': 'countryHR'}, + {'name': 'Cuba', 'value': 'countryCU'}, + {'name': 'Cyprus', 'value': 'countryCY'}, + {'name': 'Czech Republic', 'value': 'countryCZ'}, + {'name': 'Denmark', 'value': 'countryDK'}, + {'name': 'Djibouti', 'value': 'countryDJ'}, + {'name': 'Dominica', 'value': 'countryDM'}, + {'name': 'Dominican Republic', 'value': 'countryDO'}, + {'name': 'East Timor', 'value': 'countryTP'}, + {'name': 'Ecuador', 'value': 'countryEC'}, + {'name': 'Egypt', 'value': 'countryEG'}, + {'name': 'El Salvador', 'value': 'countrySV'}, + {'name': 'Equatorial Guinea', 'value': 'countryGQ'}, + {'name': 'Eritrea', 'value': 'countryER'}, + {'name': 'Estonia', 'value': 'countryEE'}, + {'name': 'Ethiopia', 'value': 'countryET'}, + {'name': 'European Union', 'value': 'countryEU'}, + {'name': 'Falkland Islands (Malvinas)', 'value': 'countryFK'}, + {'name': 'Faroe Islands', 'value': 'countryFO'}, + {'name': 'Fiji', 'value': 'countryFJ'}, + {'name': 'Finland', 'value': 'countryFI'}, + {'name': 'France', 'value': 'countryFR'}, + {'name': 'France\, Metropolitan', 'value': 'countryFX'}, + {'name': 'French Guiana', 'value': 'countryGF'}, + {'name': 'French Polynesia', 'value': 'countryPF'}, + {'name': 'French Southern Territories', 'value': 'countryTF'}, + {'name': 'Gabon', 'value': 'countryGA'}, + {'name': 'Gambia', 'value': 'countryGM'}, + {'name': 'Georgia', 'value': 'countryGE'}, + {'name': 'Germany', 'value': 'countryDE'}, + {'name': 'Ghana', 'value': 'countryGH'}, + {'name': 'Gibraltar', 'value': 'countryGI'}, + {'name': 'Greece', 'value': 'countryGR'}, + {'name': 'Greenland', 'value': 'countryGL'}, + {'name': 'Grenada', 'value': 'countryGD'}, + {'name': 'Guadeloupe', 'value': 'countryGP'}, + {'name': 'Guam', 'value': 'countryGU'}, + {'name': 'Guatemala', 'value': 'countryGT'}, + {'name': 'Guinea', 'value': 'countryGN'}, + {'name': 'Guinea-Bissau', 'value': 'countryGW'}, + {'name': 'Guyana', 'value': 'countryGY'}, + {'name': 'Haiti', 'value': 'countryHT'}, + {'name': 'Heard Island and Mcdonald Islands', 'value': 'countryHM'}, + {'name': 'Holy See (Vatican City State)', 'value': 'countryVA'}, + {'name': 'Honduras', 'value': 'countryHN'}, + {'name': 'Hong Kong', 'value': 'countryHK'}, + {'name': 'Hungary', 'value': 'countryHU'}, + {'name': 'Iceland', 'value': 'countryIS'}, + {'name': 'India', 'value': 'countryIN'}, + {'name': 'Indonesia', 'value': 'countryID'}, + {'name': 'Iran, Islamic Republic of', 'value': 'countryIR'}, + {'name': 'Iraq', 'value': 'countryIQ'}, + {'name': 'Ireland', 'value': 'countryIE'}, + {'name': 'Israel', 'value': 'countryIL'}, + {'name': 'Italy', 'value': 'countryIT'}, + {'name': 'Jamaica', 'value': 'countryJM'}, + {'name': 'Japan', 'value': 'countryJP'}, + {'name': 'Jordan', 'value': 'countryJO'}, + {'name': 'Kazakhstan', 'value': 'countryKZ'}, + {'name': 'Kenya', 'value': 'countryKE'}, + {'name': 'Kiribati', 'value': 'countryKI'}, + {'name': 'Korea, Democratic People\'s Republic of', 'value': 'countryKP'}, + {'name': 'Korea, Republic of', 'value': 'countryKR'}, + {'name': 'Kuwait', 'value': 'countryKW'}, + {'name': 'Kyrgyzstan', 'value': 'countryKG'}, + {'name': 'Lao People\'s Democratic Republic', 'value': 'countryLA'}, + {'name': 'Latvia', 'value': 'countryLV'}, + {'name': 'Lebanon', 'value': 'countryLB'}, + {'name': 'Lesotho', 'value': 'countryLS'}, + {'name': 'Liberia', 'value': 'countryLR'}, + {'name': 'Libyan Arab Jamahiriya', 'value': 'countryLY'}, + {'name': 'Liechtenstein', 'value': 'countryLI'}, + {'name': 'Lithuania', 'value': 'countryLT'}, + {'name': 'Luxembourg', 'value': 'countryLU'}, + {'name': 'Macao', 'value': 'countryMO'}, + {'name': 'Macedonia, the Former Yugosalv Republic of', 'value': 'countryMK'}, + {'name': 'Madagascar', 'value': 'countryMG'}, + {'name': 'Malawi', 'value': 'countryMW'}, + {'name': 'Malaysia', 'value': 'countryMY'}, + {'name': 'Maldives', 'value': 'countryMV'}, + {'name': 'Mali', 'value': 'countryML'}, + {'name': 'Malta', 'value': 'countryMT'}, + {'name': 'Marshall Islands', 'value': 'countryMH'}, + {'name': 'Martinique', 'value': 'countryMQ'}, + {'name': 'Mauritania', 'value': 'countryMR'}, + {'name': 'Mauritius', 'value': 'countryMU'}, + {'name': 'Mayotte', 'value': 'countryYT'}, + {'name': 'Mexico', 'value': 'countryMX'}, + {'name': 'Micronesia, Federated States of', 'value': 'countryFM'}, + {'name': 'Moldova, Republic of', 'value': 'countryMD'}, + {'name': 'Monaco', 'value': 'countryMC'}, + {'name': 'Mongolia', 'value': 'countryMN'}, + {'name': 'Montserrat', 'value': 'countryMS'}, + {'name': 'Morocco', 'value': 'countryMA'}, + {'name': 'Mozambique', 'value': 'countryMZ'}, + {'name': 'Myanmar', 'value': 'countryMM'}, + {'name': 'Namibia', 'value': 'countryNA'}, + {'name': 'Nauru', 'value': 'countryNR'}, + {'name': 'Nepal', 'value': 'countryNP'}, + {'name': 'Netherlands', 'value': 'countryNL'}, + {'name': 'Netherlands Antilles', 'value': 'countryAN'}, + {'name': 'New Caledonia', 'value': 'countryNC'}, + {'name': 'New Zealand', 'value': 'countryNZ'}, + {'name': 'Nicaragua', 'value': 'countryNI'}, + {'name': 'Niger', 'value': 'countryNE'}, + {'name': 'Nigeria', 'value': 'countryNG'}, + {'name': 'Niue', 'value': 'countryNU'}, + {'name': 'Norfolk Island', 'value': 'countryNF'}, + {'name': 'Northern Mariana Islands', 'value': 'countryMP'}, + {'name': 'Norway', 'value': 'countryNO'}, + {'name': 'Oman', 'value': 'countryOM'}, + {'name': 'Pakistan', 'value': 'countryPK'}, + {'name': 'Palau', 'value': 'countryPW'}, + {'name': 'Palestinian Territory', 'value': 'countryPS'}, + {'name': 'Panama', 'value': 'countryPA'}, + {'name': 'Papua New Guinea', 'value': 'countryPG'}, + {'name': 'Paraguay', 'value': 'countryPY'}, + {'name': 'Peru', 'value': 'countryPE'}, + {'name': 'Philippines', 'value': 'countryPH'}, + {'name': 'Pitcairn', 'value': 'countryPN'}, + {'name': 'Poland', 'value': 'countryPL'}, + {'name': 'Portugal', 'value': 'countryPT'}, + {'name': 'Puerto Rico', 'value': 'countryPR'}, + {'name': 'Qatar', 'value': 'countryQA'}, + {'name': 'Reunion', 'value': 'countryRE'}, + {'name': 'Romania', 'value': 'countryRO'}, + {'name': 'Russian Federation', 'value': 'countryRU'}, + {'name': 'Rwanda', 'value': 'countryRW'}, + {'name': 'Saint Helena', 'value': 'countrySH'}, + {'name': 'Saint Kitts and Nevis', 'value': 'countryKN'}, + {'name': 'Saint Lucia', 'value': 'countryLC'}, + {'name': 'Saint Pierre and Miquelon', 'value': 'countryPM'}, + {'name': 'Saint Vincent and the Grenadines', 'value': 'countryVC'}, + {'name': 'Samoa', 'value': 'countryWS'}, + {'name': 'San Marino', 'value': 'countrySM'}, + {'name': 'Sao Tome and Principe', 'value': 'countryST'}, + {'name': 'Saudi Arabia', 'value': 'countrySA'}, + {'name': 'Senegal', 'value': 'countrySN'}, + {'name': 'Serbia and Montenegro', 'value': 'countryCS'}, + {'name': 'Seychelles', 'value': 'countrySC'}, + {'name': 'Sierra Leone', 'value': 'countrySL'}, + {'name': 'Singapore', 'value': 'countrySG'}, + {'name': 'Slovakia', 'value': 'countrySK'}, + {'name': 'Slovenia', 'value': 'countrySI'}, + {'name': 'Solomon Islands', 'value': 'countrySB'}, + {'name': 'Somalia', 'value': 'countrySO'}, + {'name': 'South Africa', 'value': 'countryZA'}, + {'name': 'South Georgia and the South Sandwich Islands', 'value': 'countryGS'}, + {'name': 'Spain', 'value': 'countryES'}, + {'name': 'Sri Lanka', 'value': 'countryLK'}, + {'name': 'Sudan', 'value': 'countrySD'}, + {'name': 'Suriname', 'value': 'countrySR'}, + {'name': 'Svalbard and Jan Mayen', 'value': 'countrySJ'}, + {'name': 'Swaziland', 'value': 'countrySZ'}, + {'name': 'Sweden', 'value': 'countrySE'}, + {'name': 'Switzerland', 'value': 'countryCH'}, + {'name': 'Syrian Arab Republic', 'value': 'countrySY'}, + {'name': 'Taiwan, Province of China', 'value': 'countryTW'}, + {'name': 'Tajikistan', 'value': 'countryTJ'}, + {'name': 'Tanzania, United Republic of', 'value': 'countryTZ'}, + {'name': 'Thailand', 'value': 'countryTH'}, + {'name': 'Togo', 'value': 'countryTG'}, + {'name': 'Tokelau', 'value': 'countryTK'}, + {'name': 'Tonga', 'value': 'countryTO'}, + {'name': 'Trinidad and Tobago', 'value': 'countryTT'}, + {'name': 'Tunisia', 'value': 'countryTN'}, + {'name': 'Turkey', 'value': 'countryTR'}, + {'name': 'Turkmenistan', 'value': 'countryTM'}, + {'name': 'Turks and Caicos Islands', 'value': 'countryTC'}, + {'name': 'Tuvalu', 'value': 'countryTV'}, + {'name': 'Uganda', 'value': 'countryUG'}, + {'name': 'Ukraine', 'value': 'countryUA'}, + {'name': 'United Arab Emirates', 'value': 'countryAE'}, + {'name': 'United Kingdom', 'value': 'countryUK'}, + {'name': 'United States', 'value': 'countryUS'}, + {'name': 'United States Minor Outlying Islands', 'value': 'countryUM'}, + {'name': 'Uruguay', 'value': 'countryUY'}, + {'name': 'Uzbekistan', 'value': 'countryUZ'}, + {'name': 'Vanuatu', 'value': 'countryVU'}, + {'name': 'Venezuela', 'value': 'countryVE'}, + {'name': 'Vietnam', 'value': 'countryVN'}, + {'name': 'Virgin Islands, British', 'value': 'countryVG'}, + {'name': 'Virgin Islands, U.S.', 'value': 'countryVI'}, + {'name': 'Wallis and Futuna', 'value': 'countryWF'}, + {'name': 'Western Sahara', 'value': 'countryEH'}, + {'name': 'Yemen', 'value': 'countryYE'}, + {'name': 'Yugoslavia', 'value': 'countryYU'}, + {'name': 'Zambia', 'value': 'countryZM'}, + {'name': 'Zimbabwe', 'value': 'countryZW'} + ] + def __init__(self, **kwargs): self.url = '' self.lang = 'lang_en' + self.ctry = '' + self.safe = True self.dark = False self.nojs = False self.near = '' @@ -73,4 +320,4 @@ class Config: return delattr(self, name) def __contains__(self, name): - return hasattr(self, name) \ No newline at end of file + return hasattr(self, name) diff --git a/app/request.py b/app/request.py index f50bfe4..07b6351 100644 --- a/app/request.py +++ b/app/request.py @@ -26,7 +26,7 @@ def gen_user_agent(normal_ua): return DESKTOP_UA.format(mozilla, linux, firefox) -def gen_query(query, args, near_city=None, language='lang_en'): +def gen_query(query, args, config, near_city=None): param_dict = {key: '' for key in VALID_PARAMS} # Use :past(hour/day/week/month/year) if available # example search "new restaurants :past month" @@ -46,11 +46,13 @@ def gen_query(query, args, near_city=None, language='lang_en'): param_dict['start'] = '&start=' + args.get('start') # Search for results near a particular city, if available - if near_city is not None: + if near_city: param_dict['near'] = '&near=' + urlparse.quote(near_city) # Set language for results (lr) and interface (hl) - param_dict['lr'] = '&lr=' + language + '&hl=' + language.replace('lang_', '') + param_dict['lr'] = '&lr=' + config.lang + '&hl=' + config.lang.replace('lang_', '') + param_dict['cr'] = ('&cr=' + config.ctry) if config.ctry else '' + param_dict['safe'] = '&safe=' + ('active' if config.safe else 'off') for val in param_dict.values(): if not val or val is None: diff --git a/app/routes.py b/app/routes.py index a667bb8..3f0d04c 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,6 +3,7 @@ from app.filter import Filter, get_first_link from app.models.config import Config from app.request import Request, gen_query import argparse +import base64 from bs4 import BeautifulSoup from cryptography.fernet import Fernet, InvalidToken from flask import g, make_response, request, redirect, render_template, send_file @@ -10,6 +11,7 @@ from functools import wraps import io import json import os +from pycurl import error as pycurl_error import urllib.parse as urlparse import waitress @@ -64,7 +66,9 @@ def index(): bg=bg, ua=g.user_request.modified_user_agent, languages=Config.LANGUAGES, + countries=Config.COUNTRIES, current_lang=g.user_config.lang, + current_ctry=g.user_config.ctry, version_number=app.config['VERSION_NUMBER'], request_type='get' if g.user_config.get_only else 'post') @@ -108,7 +112,7 @@ def search(): mobile = 'Android' in user_agent or 'iPhone' in user_agent content_filter = Filter(mobile, g.user_config, secret_key=app.secret_key) - full_query = gen_query(q, request_params, content_filter.near, language=g.user_config.lang) + full_query = gen_query(q, request_params, g.user_config, content_filter.near) get_body = g.user_request.send(query=full_query) dirty_soup = BeautifulSoup(content_filter.reskin(get_body), 'html.parser') @@ -161,17 +165,24 @@ def imgres(): def tmp(): cipher_suite = Fernet(app.secret_key) img_url = cipher_suite.decrypt(request.args.get('image_url').encode()).decode() - file_data = g.user_request.send(base_url=img_url, return_bytes=True) - tmp_mem = io.BytesIO() - tmp_mem.write(file_data) - tmp_mem.seek(0) - return send_file( - tmp_mem, - as_attachment=True, - attachment_filename='tmp.png', - mimetype='image/png' - ) + try: + file_data = g.user_request.send(base_url=img_url, return_bytes=True) + tmp_mem = io.BytesIO() + tmp_mem.write(file_data) + tmp_mem.seek(0) + + return send_file( + tmp_mem, + as_attachment=True, + attachment_filename='tmp.png', + mimetype='image/png' + ) + except pycurl_error: + pass + + empty_gif = base64.b64decode('R0lGODlhAQABAIAAAP///////yH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==') + return send_file(io.BytesIO(empty_gif), mimetype='image/gif') @app.route('/window') diff --git a/app/static/js/controller.js b/app/static/js/controller.js index 02ecdb5..b3b4f3f 100644 --- a/app/static/js/controller.js +++ b/app/static/js/controller.js @@ -20,6 +20,7 @@ const fillConfigValues = () => { const near = document.getElementById("config-near"); const noJS = document.getElementById("config-nojs"); const dark = document.getElementById("config-dark"); + const safe = document.getElementById("config-safe"); const url = document.getElementById("config-url"); const newTab = document.getElementById("config-new-tab"); const getOnly = document.getElementById("config-get-only"); @@ -39,6 +40,7 @@ const fillConfigValues = () => { near.value = configSettings["near"] ? configSettings["near"] : ""; noJS.checked = !!configSettings["nojs"]; dark.checked = !!configSettings["dark"]; + safe.checked = !!configSettings["safe"]; getOnly.checked = !!configSettings["get_only"]; newTab.checked = !!configSettings["new_tab"]; diff --git a/app/templates/index.html b/app/templates/index.html index 4747dba..af1879f 100644 --- a/app/templates/index.html +++ b/app/templates/index.html @@ -41,6 +41,19 @@ User Agent: {{ ua }} +
+ + +
+
+ + +
diff --git a/test/test_results.py b/test/test_results.py index abf3dcd..7f500c8 100644 --- a/test/test_results.py +++ b/test/test_results.py @@ -64,4 +64,4 @@ def test_recent_results(client): date = parse(date_span) assert (current_date - date).days <= (num_days + 5) # Date can have a little bit of wiggle room except ParserError: - assert ' ago' in date_span + pass