Move tikwm extractor into a droping for the generic extractor

tests/extractors/test_tiktok_tikwm_extractor.py

@@ -1,86 +1,74 @@
 from datetime import datetime, timezone
 import time
 import pytest
+import yt_dlp
 
-from auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor import TiktokTikwmExtractor
+from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor
 from .test_extractor_base import TestExtractorBase
 
 
+@pytest.fixture(autouse=True)
+def skip_ytdlp_own_methods(mocker):
+    # mock this method, so that we skip the ytdlp download in these tests
+    mocker.patch("auto_archiver.modules.generic_extractor.tiktok.Tiktok.skip_ytdlp_download", return_value=True)
+    mocker.patch("auto_archiver.modules.generic_extractor.generic_extractor.GenericExtractor.suitable_extractors",
+                 return_value=[e for e in yt_dlp.YoutubeDL()._ies.values() if e.IE_NAME == 'TikTok'])
+
+@pytest.fixture()
+def mock_get(mocker):
+    return mocker.patch("auto_archiver.modules.generic_extractor.tiktok.requests.get")
+
 class TestTiktokTikwmExtractor(TestExtractorBase):
     """
     Test suite for TestTiktokTikwmExtractor.
     """
 
-    extractor_module = "tiktok_tikwm_extractor"
-    extractor: TiktokTikwmExtractor
+    extractor_module = "generic_extractor"
+    extractor: GenericExtractor
 
     config = {}
 
     VALID_EXAMPLE_URL = "https://www.tiktok.com/@example/video/1234"
 
-    @staticmethod
-    def get_mockers(mocker):
-        mock_get = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.requests.get")
-        mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger")
-        return mock_get, mock_logger
-
-    @pytest.mark.parametrize("url,valid_url", [
-        ("https://bellingcat.com", False),
-        ("https://youtube.com", False),
-        ("https://tiktok.co/", False),
-        ("https://tiktok.com/", False),
-        ("https://www.tiktok.com/", False),
-        ("https://api.cool.tiktok.com/", False),
-        (VALID_EXAMPLE_URL, True),
-        ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
-        ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
-    ])
-    def test_valid_urls(self, mocker, make_item, url, valid_url):
-        mock_get, mock_logger = self.get_mockers(mocker)
-        if valid_url:
-            mock_get.return_value.status_code = 404
-        assert self.extractor.download(make_item(url)) == False
-        assert mock_get.call_count == int(valid_url)
-        assert mock_logger.error.call_count == int(valid_url)
-
-    def test_invalid_json_responses(self, mocker, make_item):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_invalid_json_responses(self, mock_get, make_item, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = ValueError
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
+        with caplog.at_level('DEBUG'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            assert len(caplog.records) == 2
+            # first message is just the 'Skipping using ytdlp to download files for TikTok' message
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
 
         mock_get.return_value.json.side_effect = Exception
-        with pytest.raises(Exception):
-            self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
-        mock_get.assert_called()
-        assert mock_get.call_count == 2
-        assert mock_get.return_value.json.call_count == 2
+        with caplog.at_level('ERROR'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called()
+            assert mock_get.call_count == 2
+            assert mock_get.return_value.json.call_count == 2
+            assert len(caplog.records) == 2
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
 
     @pytest.mark.parametrize("response", [
         ({"msg": "failure"}),
         ({"msg": "success"}),
     ])
-    def test_unsuccessful_responses(self, mocker, make_item, response):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_unsuccessful_responses(self, mock_get, make_item, response, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = response
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response")
+        with caplog.at_level('DEBUG'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            assert "failed to get a valid response from tikwm.com" in caplog.records[1].message
 
     @pytest.mark.parametrize("response,has_vid", [
         ({"data": {"id": 123}}, False),
         ({"data": {"wmplay": "url"}}, True),
         ({"data": {"play": "url"}}, True),
     ])
-    def test_correct_extraction(self, mocker, make_item, response, has_vid):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item, response, has_vid):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", **response}
 
@@ -99,8 +87,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         else:
             mock_logger.error.assert_not_called()
 
-    def test_correct_extraction(self, mocker, make_item):
-        mock_get, _ = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", "data": {
             "wmplay": "url",