Feature: autocomplete/search suggestions (#72)
Basic autocomplete/search suggestion functionality added * Adds new GET and POST routes for '/autocomplete' that accept a string query and returns an array of suggestions * Adds new autoscript.js file for handling queries on the main page and results view * Updated requests class to include autocomplete method * Updated opensearch template to handle search suggestions * Added header template to allow for autocomplete on results view * Updated readme to mention autocomplete featuremain
Ben Busby
GitHub
parent
3dbe51e9e7
commit
21012f5265
|
|
@ -25,7 +25,8 @@ Contents
|
|||
- No AMP links
|
||||
- No URL tracking tags (i.e. utm=%s)
|
||||
- No referrer header
|
||||
- POST request search queries (when possible)
|
||||
- Autocomplete/search suggestions
|
||||
- POST request search and suggestion queries (when possible)
|
||||
- View images at full res without site redirect (currently mobile only)
|
||||
- Dark mode
|
||||
- Randomly generated User Agent
|
||||
|
|
|
|||
|
|
@ -96,9 +96,13 @@ class Filter:
|
|||
st_card.decompose()
|
||||
|
||||
footer = soup.find('div', id='sfooter')
|
||||
if footer is not None:
|
||||
if footer:
|
||||
footer.decompose()
|
||||
|
||||
header = soup.find('header')
|
||||
if header:
|
||||
header.decompose()
|
||||
|
||||
return soup
|
||||
|
||||
def remove_ads(self, soup):
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
from io import BytesIO
|
||||
from lxml import etree
|
||||
import pycurl
|
||||
import random
|
||||
import urllib.parse as urlparse
|
||||
|
||||
# Base search url
|
||||
# Core Google search URLs
|
||||
SEARCH_URL = 'https://www.google.com/search?gbv=1&q='
|
||||
AUTOCOMPLETE_URL = 'https://suggestqueries.google.com/complete/search?client=toolbar&'
|
||||
|
||||
MOBILE_UA = '{}/5.0 (Android 0; Mobile; rv:54.0) Gecko/54.0 {}/59.0'
|
||||
DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
|
||||
|
|
@ -13,9 +15,7 @@ DESKTOP_UA = '{}/5.0 (X11; {} x86_64; rv:75.0) Gecko/20100101 {}/75.0'
|
|||
VALID_PARAMS = ['tbs', 'tbm', 'start', 'near']
|
||||
|
||||
|
||||
def gen_user_agent(normal_ua):
|
||||
is_mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
|
||||
|
||||
def gen_user_agent(normal_ua, is_mobile):
|
||||
mozilla = random.choice(['Moo', 'Woah', 'Bro', 'Slow']) + 'zilla'
|
||||
firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox'
|
||||
linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux'
|
||||
|
|
@ -64,8 +64,9 @@ def gen_query(query, args, config, near_city=None):
|
|||
|
||||
class Request:
|
||||
def __init__(self, normal_ua, language='lang_en'):
|
||||
self.modified_user_agent = gen_user_agent(normal_ua)
|
||||
self.language = language
|
||||
self.mobile = 'Android' in normal_ua or 'iPhone' in normal_ua
|
||||
self.modified_user_agent = gen_user_agent(normal_ua, self.mobile)
|
||||
|
||||
def __getitem__(self, name):
|
||||
return getattr(self, name)
|
||||
|
|
@ -76,6 +77,16 @@ class Request:
|
|||
else:
|
||||
return 'unicode-escape'
|
||||
|
||||
def autocomplete(self, query):
|
||||
ac_query = dict(hl=self.language, q=query)
|
||||
response = self.send(base_url=AUTOCOMPLETE_URL, query=urlparse.urlencode(ac_query))
|
||||
|
||||
if response:
|
||||
dom = etree.fromstring(response)
|
||||
return dom.xpath('//suggestion/@data')
|
||||
|
||||
return []
|
||||
|
||||
def send(self, base_url=SEARCH_URL, query='', return_bytes=False):
|
||||
response_header = []
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import argparse
|
|||
import base64
|
||||
from bs4 import BeautifulSoup
|
||||
from cryptography.fernet import Fernet, InvalidToken
|
||||
from flask import g, make_response, request, redirect, render_template, send_file
|
||||
from flask import g, jsonify, make_response, request, redirect, render_template, send_file
|
||||
from functools import wraps
|
||||
import io
|
||||
import json
|
||||
|
|
@ -88,6 +88,19 @@ def opensearch():
|
|||
return response
|
||||
|
||||
|
||||
@app.route('/autocomplete', methods=['GET', 'POST'])
|
||||
def autocomplete():
|
||||
request_params = request.args if request.method == 'GET' else request.form
|
||||
q = request_params.get('q')
|
||||
|
||||
if not q and not request.data:
|
||||
return jsonify({'?': []})
|
||||
elif request.data:
|
||||
q = urlparse.unquote_plus(request.data.decode('utf-8').replace('q=', ''))
|
||||
|
||||
return jsonify([q, g.user_request.autocomplete(q)])
|
||||
|
||||
|
||||
@app.route('/search', methods=['GET', 'POST'])
|
||||
@auth_required
|
||||
def search():
|
||||
|
|
@ -121,7 +134,14 @@ def search():
|
|||
else:
|
||||
formatted_results = content_filter.clean(dirty_soup)
|
||||
|
||||
return render_template('display.html', query=urlparse.unquote(q), response=formatted_results)
|
||||
return render_template(
|
||||
'display.html',
|
||||
query=urlparse.unquote(q),
|
||||
response=formatted_results,
|
||||
search_header=render_template(
|
||||
'header.html',
|
||||
q=urlparse.unquote(q),
|
||||
mobile=g.user_request.mobile))
|
||||
|
||||
|
||||
@app.route('/config', methods=['GET', 'POST'])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
header {
|
||||
font-family: Roboto,HelveticaNeue,Arial,sans-serif;
|
||||
font-size: 14px;
|
||||
line-height: 20px;
|
||||
color: #3C4043;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
|
||||
.logo-link, .logo-letter {
|
||||
text-decoration: none !important;
|
||||
letter-spacing: -1px;
|
||||
text-align: center;
|
||||
border-radius: 2px 0 0 0;
|
||||
}
|
||||
|
||||
.mobile-logo {
|
||||
font: 22px/36px Futura, Arial, sans-serif;
|
||||
padding-left: 5px;
|
||||
}
|
||||
|
||||
.logo-div {
|
||||
letter-spacing: -1px;
|
||||
text-align: center;
|
||||
font: 22pt Futura, Arial, sans-serif;
|
||||
padding: 10px 0 5px 0;
|
||||
height: 37px;
|
||||
font-smoothing: antialiased;
|
||||
}
|
||||
|
||||
.search-div {
|
||||
border-radius: 8px 8px 0 0;
|
||||
box-shadow: 0 1px 6px rgba(32, 33, 36, 0.18);
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.search-form {
|
||||
height: 39px;
|
||||
display: flex;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.search-input {
|
||||
background: none;
|
||||
margin: 2px 4px 2px 8px;
|
||||
display: block;
|
||||
font-size: 16px;
|
||||
padding: 0 0 0 8px;
|
||||
flex: 1;
|
||||
height: 35px;
|
||||
outline: none;
|
||||
border: none;
|
||||
width: 100%;
|
||||
-webkit-tap-highlight-color: rgba(0,0,0,0);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
.autocomplete {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.autocomplete-items {
|
||||
position: absolute;
|
||||
border: 1px solid #d4d4d4;
|
||||
border-bottom: none;
|
||||
border-top: none;
|
||||
z-index: 99;
|
||||
|
||||
/*position the autocomplete items to be the same width as the container:*/
|
||||
top: 100%;
|
||||
left: 0;
|
||||
right: 0;
|
||||
}
|
||||
|
||||
.autocomplete-items div {
|
||||
padding: 10px;
|
||||
cursor: pointer;
|
||||
background-color: #fff;
|
||||
border-bottom: 1px solid #d4d4d4;
|
||||
}
|
||||
|
||||
.autocomplete-items div:hover {
|
||||
background-color: #e9e9e9;
|
||||
}
|
||||
|
||||
.autocomplete-active {
|
||||
background-color: #685e79 !important;
|
||||
color: #ffffff;
|
||||
}
|
||||
|
|
@ -0,0 +1,98 @@
|
|||
const handleUserInput = searchBar => {
|
||||
let xhrRequest = new XMLHttpRequest();
|
||||
xhrRequest.open("POST", "/autocomplete");
|
||||
xhrRequest.setRequestHeader("Content-type", "application/x-www-form-urlencoded");
|
||||
xhrRequest.onload = function() {
|
||||
if (xhrRequest.readyState === 4 && xhrRequest.status !== 200) {
|
||||
alert("Error fetching autocomplete results");
|
||||
return;
|
||||
}
|
||||
|
||||
// Fill autocomplete with fetched results
|
||||
let autocompleteResults = JSON.parse(xhrRequest.responseText);
|
||||
autocomplete(searchBar, autocompleteResults[1]);
|
||||
};
|
||||
|
||||
xhrRequest.send('q=' + searchBar.value);
|
||||
};
|
||||
|
||||
const autocomplete = (searchInput, autocompleteResults) => {
|
||||
let currentFocus;
|
||||
|
||||
searchInput.addEventListener("input", function () {
|
||||
let autocompleteList, autocompleteItem, i, val = this.value;
|
||||
closeAllLists();
|
||||
|
||||
if (!val || !autocompleteResults) {
|
||||
return false;
|
||||
}
|
||||
|
||||
currentFocus = -1;
|
||||
autocompleteList = document.createElement("div");
|
||||
autocompleteList.setAttribute("id", this.id + "-autocomplete-list");
|
||||
autocompleteList.setAttribute("class", "autocomplete-items");
|
||||
this.parentNode.appendChild(autocompleteList);
|
||||
|
||||
for (i = 0; i < autocompleteResults.length; i++) {
|
||||
if (autocompleteResults[i].substr(0, val.length).toUpperCase() === val.toUpperCase()) {
|
||||
autocompleteItem = document.createElement("div");
|
||||
autocompleteItem.innerHTML = "<strong>" + autocompleteResults[i].substr(0, val.length) + "</strong>";
|
||||
autocompleteItem.innerHTML += autocompleteResults[i].substr(val.length);
|
||||
autocompleteItem.innerHTML += "<input type=\"hidden\" value=\"" + autocompleteResults[i] + "\">";
|
||||
autocompleteItem.addEventListener("click", function () {
|
||||
searchInput.value = this.getElementsByTagName("input")[0].value;
|
||||
closeAllLists();
|
||||
document.getElementById("search-form").submit();
|
||||
});
|
||||
autocompleteList.appendChild(autocompleteItem);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
searchInput.addEventListener("keydown", function (e) {
|
||||
let suggestion = document.getElementById(this.id + "-autocomplete-list");
|
||||
if (suggestion) suggestion = suggestion.getElementsByTagName("div");
|
||||
if (e.keyCode === 40) { // down
|
||||
currentFocus++;
|
||||
addActive(suggestion);
|
||||
} else if (e.keyCode === 38) { //up
|
||||
currentFocus--;
|
||||
addActive(suggestion);
|
||||
} else if (e.keyCode === 13) { // enter
|
||||
e.preventDefault();
|
||||
if (currentFocus > -1) {
|
||||
if (suggestion) suggestion[currentFocus].click();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
const addActive = suggestion => {
|
||||
if (!suggestion || !suggestion[currentFocus]) return false;
|
||||
removeActive(suggestion);
|
||||
|
||||
if (currentFocus >= suggestion.length) currentFocus = 0;
|
||||
if (currentFocus < 0) currentFocus = (suggestion.length - 1);
|
||||
|
||||
suggestion[currentFocus].classList.add("autocomplete-active");
|
||||
};
|
||||
|
||||
const removeActive = suggestion => {
|
||||
for (let i = 0; i < suggestion.length; i++) {
|
||||
suggestion[i].classList.remove("autocomplete-active");
|
||||
}
|
||||
};
|
||||
|
||||
const closeAllLists = el => {
|
||||
let suggestions = document.getElementsByClassName("autocomplete-items");
|
||||
for (let i = 0; i < suggestions.length; i++) {
|
||||
if (el !== suggestions[i] && el !== searchInput) {
|
||||
suggestions[i].parentNode.removeChild(suggestions[i]);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Close lists and search when user selects a suggestion
|
||||
document.addEventListener("click", function (e) {
|
||||
closeAllLists(e.target);
|
||||
});
|
||||
};
|
||||
|
|
@ -11,6 +11,8 @@ const setupSearchLayout = () => {
|
|||
if (event.keyCode === 13) {
|
||||
event.preventDefault();
|
||||
searchBtn.click();
|
||||
} else {
|
||||
handleUserInput(searchBar);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
|
|
|||
|
|
@ -5,9 +5,13 @@
|
|||
<link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<meta name="referrer" content="no-referrer">
|
||||
<script type="text/javascript" src="/static/js/autocomplete.js"></script>
|
||||
<link rel="stylesheet" href="/static/css/search.css">
|
||||
<link rel="stylesheet" href="/static/css/header.css">
|
||||
<title>{{ query }} - Whoogle Search</title>
|
||||
</head>
|
||||
<body>
|
||||
{{ search_header|safe }}
|
||||
{{ response|safe }}
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,53 @@
|
|||
{% if mobile %}
|
||||
<header>
|
||||
<div class="bz1lBb">
|
||||
<form class="Pg70bf" id="search-form" method="POST">
|
||||
<a class="logo-link mobile-logo"
|
||||
href="/"
|
||||
style="display:flex; justify-content:center; align-items:center; color:#685e79; font-size:18px; ">
|
||||
<span class="V6gwVd">Wh</span><span class="iWkuvd">o</span><span class="cDrQ7">o</span><span
|
||||
class="V6gwVd">g</span><span class="ntlR9">l</span><span
|
||||
class="iWkuvd tJ3Myc">e</span>
|
||||
</a>
|
||||
<div class="H0PQec" style="width: 100%;">
|
||||
<div class="sbc esbc autocomplete">
|
||||
<input id="search-bar" autocapitalize="none" autocomplete="off" class="noHIxc" name="q"
|
||||
spellcheck="false" type="text" value="{{ q }}">
|
||||
<div class="sc"></div>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</header>
|
||||
{% else %}
|
||||
<header>
|
||||
<div class="logo-div">
|
||||
<a class="logo-link" href="/">
|
||||
<span class="V6gwVd logo-letter">Wh</span><span class="iWkuvd logo-letter">o</span><span
|
||||
class="cDrQ7 logo-letter">o</span><span class="V6gwVd logo-letter">g</span><span
|
||||
class="ntlR9 logo-letter">l</span><span class="iWkuvd tJ3Myc logo-letter">e</span>
|
||||
</a>
|
||||
</div>
|
||||
<div class="search-div">
|
||||
<form id="search-form" class="search-form" id="sf" method="POST">
|
||||
<div class="autocomplete" style="width: 100%; flex: 1">
|
||||
<div style="width: 100%; display: flex">
|
||||
<input id="search-bar" autocapitalize="none" autocomplete="off" class="noHIxc" name="q"
|
||||
spellcheck="false" type="text" value="{{ q }}">
|
||||
<div class="sc"></div>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</header>
|
||||
{% endif %}
|
||||
|
||||
<script>
|
||||
const searchBar = document.getElementById("search-bar");
|
||||
|
||||
searchBar.addEventListener("keyup", function (event) {
|
||||
if (event.keyCode !== 13) {
|
||||
handleUserInput(searchBar);
|
||||
}
|
||||
});
|
||||
</script>
|
||||
|
|
@ -17,18 +17,22 @@
|
|||
<meta name="referrer" content="no-referrer">
|
||||
<meta name="msapplication-TileColor" content="#ffffff">
|
||||
<meta name="msapplication-TileImage" content="/static/img/favicon/ms-icon-144x144.png">
|
||||
<script type="text/javascript" src="/static/js/autocomplete.js"></script>
|
||||
<script type="text/javascript" src="/static/js/controller.js"></script>
|
||||
<link rel="search" href="/opensearch.xml" type="application/opensearchdescription+xml" title="Whoogle Search">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<link rel="stylesheet" href="/static/css/search.css">
|
||||
<link rel="stylesheet" href="/static/css/main.css">
|
||||
<title>Whoogle Search</title>
|
||||
</head>
|
||||
<body id="main" style="display: none; background-color: {{ bg }}">
|
||||
<div class="search-container">
|
||||
<img class="logo" src="/static/img/logo.png">
|
||||
<form action="/search" method="{{ request_type }}">
|
||||
<form id="search-form" action="/search" method="{{ request_type }}">
|
||||
<div class="search-fields">
|
||||
<div class="autocomplete">
|
||||
<input type="text" name="q" id="search-bar" autofocus="autofocus">
|
||||
</div>
|
||||
<input type="submit" id="search-submit" value="Search">
|
||||
</div>
|
||||
</form>
|
||||
|
|
|
|||
|
|
@ -7,7 +7,9 @@
|
|||
<Url type="text/html" method="{{ request_type }}" template="{{ main_url }}/search">
|
||||
<Param name="q" value="{searchTerms}"/>
|
||||
</Url>
|
||||
<Url type="application/x-suggestions+json" template="{{ main_url }}/search"/>
|
||||
<Url type="application/x-suggestions+json" method="{{ request_type }}" template="{{ main_url }}/autocomplete">
|
||||
<Param name="q" value="{searchTerms}"/>
|
||||
</Url>
|
||||
<moz:SearchForm>{{ main_url }}/search</moz:SearchForm>
|
||||
</OpenSearchDescription>
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ cryptography==2.8
|
|||
Flask==1.1.1
|
||||
itsdangerous==1.1.0
|
||||
Jinja2==2.10.3
|
||||
lxml==4.5.1
|
||||
MarkupSafe==1.1.1
|
||||
pycparser==2.19
|
||||
pycurl==7.43.0.4
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
def test_autocomplete_get(client):
|
||||
rv = client.get('/autocomplete?q=green+eggs+and')
|
||||
assert rv._status_code == 200
|
||||
assert len(rv.data) >= 1
|
||||
assert b'green eggs and ham' in rv.data
|
||||
|
||||
|
||||
def test_autocomplete_post(client):
|
||||
rv = client.post('/autocomplete', data=dict(q='the+cat+in+the'))
|
||||
assert rv._status_code == 200
|
||||
assert len(rv.data) >= 1
|
||||
assert b'the cat in the hat' in rv.data
|
||||
Loading…
Reference in New Issue