Merge branch 'main' into linting_etc

tests/extractors/test_tiktok_tikwm_extractor.py

@@ -1,67 +1,62 @@
 from datetime import datetime, timezone
 import time
 import pytest
+import yt_dlp
 
-from auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor import TiktokTikwmExtractor
+from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor
 from .test_extractor_base import TestExtractorBase
 
 
+@pytest.fixture(autouse=True)
+def skip_ytdlp_own_methods(mocker):
+    # mock this method, so that we skip the ytdlp download in these tests
+    mocker.patch("auto_archiver.modules.generic_extractor.tiktok.Tiktok.skip_ytdlp_download", return_value=True)
+    mocker.patch(
+        "auto_archiver.modules.generic_extractor.generic_extractor.GenericExtractor.suitable_extractors",
+        return_value=[e for e in yt_dlp.YoutubeDL()._ies.values() if e.IE_NAME == "TikTok"],
+    )
+
+
+@pytest.fixture()
+def mock_get(mocker):
+    return mocker.patch("auto_archiver.modules.generic_extractor.tiktok.requests.get")
+
+
 class TestTiktokTikwmExtractor(TestExtractorBase):
     """
     Test suite for TestTiktokTikwmExtractor.
     """
 
-    extractor_module = "tiktok_tikwm_extractor"
-    extractor: TiktokTikwmExtractor
+    extractor_module = "generic_extractor"
+    extractor: GenericExtractor
 
     config = {}
 
     VALID_EXAMPLE_URL = "https://www.tiktok.com/@example/video/1234"
 
-    @staticmethod
-    def get_mockers(mocker):
-        mock_get = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.requests.get")
-        mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger")
-        return mock_get, mock_logger
-
-    @pytest.mark.parametrize(
-        "url,valid_url",
-        [
-            ("https://bellingcat.com", False),
-            ("https://youtube.com", False),
-            ("https://tiktok.co/", False),
-            ("https://tiktok.com/", False),
-            ("https://www.tiktok.com/", False),
-            ("https://api.cool.tiktok.com/", False),
-            (VALID_EXAMPLE_URL, True),
-            ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
-            ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
-        ],
-    )
-    def test_valid_urls(self, mocker, make_item, url, valid_url):
-        mock_get, mock_logger = self.get_mockers(mocker)
-        if valid_url:
-            mock_get.return_value.status_code = 404
-        assert self.extractor.download(make_item(url)) is False
-        assert mock_get.call_count == int(valid_url)
-        assert mock_logger.error.call_count == int(valid_url)
-
-    def test_invalid_json_responses(self, mocker, make_item):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_invalid_json_responses(self, mock_get, make_item, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = ValueError
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
+        with caplog.at_level("DEBUG"):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            # first message is just the 'Skipping using ytdlp to download files for TikTok' message
+            assert (
+                "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'"
+                in caplog.text
+            )
 
         mock_get.return_value.json.side_effect = Exception
-        with pytest.raises(Exception):
-            self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
-        mock_get.assert_called()
-        assert mock_get.call_count == 2
-        assert mock_get.return_value.json.call_count == 2
+        with caplog.at_level("ERROR"):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called()
+            assert mock_get.call_count == 2
+            assert mock_get.return_value.json.call_count == 2
+            assert (
+                "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'"
+                in caplog.text
+            )
 
     @pytest.mark.parametrize(
         "response",
@@ -70,15 +65,14 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             ({"msg": "success"}),
         ],
     )
-    def test_unsuccessful_responses(self, mocker, make_item, response):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_unsuccessful_responses(self, mock_get, make_item, response, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = response
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response")
+        with caplog.at_level("DEBUG"):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            assert "failed to get a valid response from tikwm.com" in caplog.text
 
     @pytest.mark.parametrize(
         "response,has_vid",
@@ -88,8 +82,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             ({"data": {"play": "url"}}, True),
         ],
     )
-    def test_correct_extraction(self, mocker, make_item, response, has_vid):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item, response, has_vid):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", **response}
 
@@ -108,8 +101,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         else:
             mock_logger.error.assert_not_called()
 
-    def test_correct_data_extracted(self, mocker, make_item):
-        mock_get, _ = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {
             "msg": "success",