handling selenium better

configs/config.py

@@ -4,6 +4,7 @@ import gspread
 from loguru import logger
 from selenium import webdriver
 from dataclasses import asdict
+from selenium.common.exceptions import TimeoutException
 
 from utils import GWorksheet, getattr_or
 from .wayback_config import WaybackConfig
@@ -210,16 +211,23 @@ class Config:
     def destroy_webdriver(self):
         if self.webdriver is not None and type(self.webdriver) != str:
             self.webdriver.quit()
+            del self.webdriver
 
     def recreate_webdriver(self):
-        self.destroy_webdriver()
         options = webdriver.FirefoxOptions()
         options.headless = True
         options.set_preference('network.protocol-handler.external.tg', False)
-        self.webdriver = webdriver.Firefox(options=options)
+        try:
-        self.webdriver.set_window_size(self.selenium_config.window_width,
+            new_webdriver = webdriver.Firefox(options=options)
+            # only destroy if creation is successful
+            self.destroy_webdriver()
+            self.webdriver = new_webdriver
+            self.webdriver.set_window_size(self.selenium_config.window_width,
                                        self.selenium_config.window_height)
-        self.webdriver.set_page_load_timeout(self.selenium_config.timeout_seconds)
+            self.webdriver.set_page_load_timeout(self.selenium_config.timeout_seconds)
+        except TimeoutException as e:
+            logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}")
+
 
     def __str__(self) -> str:
         return json.dumps({