From 2cb4b9e3cac5a17faa31de919cec5f6cf26af854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roman=20=C5=A0tefko?= Date: Fri, 19 May 2023 19:32:05 +0200 Subject: [PATCH] Allow setting mobile/desktop UAs using env vars (#1003) Defines separate environment variables for setting mobile vs desktop user agents Defines an environment variable for using the client's User-Agent Co-authored-by: Ben Busby --- README.md | 3 +++ app/request.py | 20 ++++++++++++++++---- app/utils/search.py | 3 ++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6c12e43..6efe7d3 100644 --- a/README.md +++ b/README.md @@ -398,6 +398,9 @@ There are a few optional environment variables available for customizing a Whoog | WHOOGLE_PROXY_PASS | The password of the proxy server. | | WHOOGLE_PROXY_TYPE | The type of the proxy server. Can be "socks5", "socks4", or "http". | | WHOOGLE_PROXY_LOC | The location of the proxy server (host or ip). | +| WHOOGLE_USER_AGENT | The desktop user agent to use. Defaults to a randomly generated one. | +| WHOOGLE_USER_AGENT_MOBILE | The mobile user agent to use. Defaults to a randomly generated one. | +| WHOOGLE_USE_CLIENT_USER_AGENT | Enable to use your own user agent for all requests. Defaults to false. | | EXPOSE_PORT | The port where Whoogle will be exposed. | | HTTPS_ONLY | Enforce HTTPS. (See [here](https://github.com/benbusby/whoogle-search#https-enforcement)) | | WHOOGLE_ALT_TW | The twitter.com alternative to use when site alternatives are enabled in the config. Set to "" to disable. | diff --git a/app/request.py b/app/request.py index 7bce2e5..5a61071 100644 --- a/app/request.py +++ b/app/request.py @@ -73,6 +73,14 @@ def send_tor_signal(signal: Signal) -> bool: def gen_user_agent(is_mobile) -> str: + user_agent = os.environ.get('WHOOGLE_USER_AGENT', '') + user_agent_mobile = os.environ.get('WHOOGLE_USER_AGENT_MOBILE', '') + if user_agent and not is_mobile: + return user_agent + + if user_agent_mobile and is_mobile: + return user_agent_mobile + firefox = random.choice(['Choir', 'Squier', 'Higher', 'Wire']) + 'fox' linux = random.choice(['Win', 'Sin', 'Gin', 'Fin', 'Kin']) + 'ux' @@ -261,7 +269,7 @@ class Request: return [] def send(self, base_url='', query='', attempt=0, - force_mobile=False) -> Response: + force_mobile=False, user_agent='') -> Response: """Sends an outbound request to a URL. Optionally sends the request using Tor, if enabled by the user. @@ -277,10 +285,14 @@ class Request: Response: The Response object returned by the requests call """ - if force_mobile and not self.mobile: - modified_user_agent = self.modified_user_agent_mobile + use_client_user_agent = int(os.environ.get('WHOOGLE_USE_CLIENT_USER_AGENT', '0')) + if user_agent and use_client_user_agent == 1: + modified_user_agent = user_agent else: - modified_user_agent = self.modified_user_agent + if force_mobile and not self.mobile: + modified_user_agent = self.modified_user_agent_mobile + else: + modified_user_agent = self.modified_user_agent headers = { 'User-Agent': modified_user_agent diff --git a/app/utils/search.py b/app/utils/search.py index 8de53f8..f643cbe 100644 --- a/app/utils/search.py +++ b/app/utils/search.py @@ -144,7 +144,8 @@ class Search: and not g.user_request.mobile) get_body = g.user_request.send(query=full_query, - force_mobile=view_image) + force_mobile=view_image, + user_agent=self.user_agent) # Produce cleanable html soup from response get_body_safed = get_body.text.replace("<","andlt;").replace(">","andgt;")