diff --git a/app/routes.py b/app/routes.py index 6c1f610..7e50425 100644 --- a/app/routes.py +++ b/app/routes.py @@ -5,10 +5,41 @@ import os import pycurl import re from .url import url_parse +import urllib.parse as urlparse +from urllib.parse import parse_qs from io import BytesIO -MOBILE_UA = 'Mozilla/5.0 (Linux; Android 8.0.0; SM-G960F Build/R16NW) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.84 Mobile Safari/537.36' -DESKTOP_UA = 'Brozilla/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Mobile LizzieMcGuirefox/59.0' +MOBILE_UA = os.environ.get('MOZ') + '/5.0 (Android 4.20; Mobile; rv:54.0) Gecko/54.0 ' + os.environ.get('FF') + '/54.0' +DESKTOP_UA = os.environ.get('MOZ') + '/5.0 (Windows NT 6.1; Win64; x64; rv:59.0) Gecko/20100101 Mobile ' + os.environ.get('FF') + '/59.0' + +SEARCH_URL = 'https://www.google.com/search?gbv=1&q=' + +nojs = int(os.environ.get('NOJS')) + + +def get_ua(user_agent): + return MOBILE_UA if ('Android' in user_agent or 'iPhone' in user_agent) else DESKTOP_UA + + +def send_request(url, ua): + request_header = [] + + # Update as an optional param + # Todo: this doesn't seem to work + ip = '64.22.92.48' + request_header.append('CLIENT-IP: ' + ip) + request_header.append('X-FORWARDED-FOR: ' + ip) + + b_obj = BytesIO() + crl = pycurl.Curl() + crl.setopt(crl.URL, url) + crl.setopt(crl.USERAGENT, ua) + crl.setopt(crl.HTTPHEADER, request_header) + crl.setopt(crl.WRITEDATA, b_obj) + crl.perform() + crl.close() + + return b_obj.getvalue().decode('utf-8', 'ignore') @app.route('/', methods=['GET']) @@ -30,27 +61,29 @@ def search(): if 'start' in request.args: start = '&start=' + request.args.get('start') + # Change to a config setting + near = '&near=boulder' + if 'near' in request.args: + near = '&near=' + request.args.get('near') + user_agent = request.headers.get('User-Agent') - full_query = url_parse(q) + tbm + start - - google_ua = DESKTOP_UA - if 'Android' in user_agent or 'iPhone' in user_agent: - google_ua = MOBILE_UA - - b_obj = BytesIO() - crl = pycurl.Curl() - crl.setopt(crl.URL, 'https://www.google.com/search?gbv=1&q=' + full_query) - crl.setopt(crl.USERAGENT, google_ua) - crl.setopt(crl.WRITEDATA, b_obj) - crl.perform() - crl.close() - get_body = b_obj.getvalue().decode('utf-8', 'ignore') - get_body = get_body.replace('data-src', 'src').replace('.001', '1').replace('visibility:hidden', 'visibility:visible').replace('>G<', '>Bl<') + full_query = url_parse(q) + tbm + start + near + get_body = send_request(SEARCH_URL + full_query, get_ua(user_agent)) + get_body = get_body.replace('>G<', '>Bl<') pattern = re.compile('4285f4|ea4335|fbcc05|34a853|fbbc05', re.IGNORECASE) get_body = pattern.sub('0000ff', get_body) soup = BeautifulSoup(get_body, 'html.parser') + for a in soup.find_all('a', href=True): + href = a['href'] + if 'url?q=' in href: + href = urlparse.urlparse(href) + href = parse_qs(href.query)['q'][0] + if nojs: + a['href'] = '/window?location=' + href + #else: + # a['href'] = 'about:reader?url=' + href try: for script in soup("script"): script.decompose() @@ -58,7 +91,7 @@ def search(): except Exception: pass - return render_template('search.html', response=soup) + return render_template('display.html', response=soup) @app.route('/url', methods=['GET']) @@ -78,5 +111,22 @@ def imgres(): return redirect(request.args.get('imgurl')) +@app.route('/window') +def window(): + get_body = send_request(request.args.get('location'), get_ua(request.headers.get('User-Agent'))) + get_body = get_body.replace('src="/', 'src="' + request.args.get('location') + '"') + get_body = get_body.replace('href="/', 'href="' + request.args.get('location') + '"') + + soup = BeautifulSoup(get_body, 'html.parser') + + try: + for script in soup('script'): + script.decompose() + except Exception: + pass + + return render_template('display.html', response=soup) + + if __name__ == '__main__': app.run(debug=True, host='0.0.0.0') diff --git a/app/templates/search.html b/app/templates/display.html similarity index 100% rename from app/templates/search.html rename to app/templates/display.html diff --git a/requirements.txt b/requirements.txt index e4f0c97..ceb2637 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +beautifulsoup4==4.8.2 +bs4==0.0.1 cffi==1.13.2 Click==7.0 cryptography==2.8 @@ -5,8 +7,10 @@ Flask==1.1.1 itsdangerous==1.1.0 Jinja2==2.10.3 MarkupSafe==1.1.1 +Phyme==0.0.9 pycparser==2.19 pycurl==7.43.0.4 pyOpenSSL==19.1.0 six==1.14.0 +soupsieve==1.9.5 Werkzeug==0.16.0 diff --git a/rhyme.py b/rhyme.py new file mode 100644 index 0000000..2b2aef6 --- /dev/null +++ b/rhyme.py @@ -0,0 +1,17 @@ +from Phyme import Phyme +import random +import sys +import time + +random.seed(time.clock()) + +ph = Phyme() + +rhymes = ph.get_perfect_rhymes(sys.argv[1]) +rhyme_vals = [] + +for arr in rhymes.values(): + for rhyme in arr: + rhyme_vals.append(rhyme) + +print(rhyme_vals[random.randint(0, len(rhyme_vals))].capitalize())