From 0dfab2d1bcb89de0d994d14cc03b91fe14930449 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 3 Mar 2025 15:55:04 +0000 Subject: [PATCH] Add some code to attempt to click the cookies banners on various websites --- src/auto_archiver/utils/webdriver.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index 50a7b94..c6ad341 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -72,6 +72,8 @@ class CookieSettingDriver(webdriver.Firefox): time.sleep(2) except Exception as e: logger.warning(f'Failed on fb accept cookies.', e) + + # now get the actual URL super(CookieSettingDriver, self).get(url) if self.facebook_accept_cookies: @@ -79,7 +81,17 @@ class CookieSettingDriver(webdriver.Firefox): close_button = self.find_element(By.XPATH, "//div[@role='dialog']//div[@aria-label='Close']") if close_button: close_button.click() + else: + # for all other sites, try and use some common button text to reject/accept cookies + for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Accept all cookies"]: + try: + accept_button = self.find_element(By.XPATH, f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]") + if accept_button: + accept_button.click() + break + except Exception as e: + pass class Webdriver: