auto-archiver/tests/archivers/test_twitter_archiver.py

import unittest
import datetime

from auto_archiver.archivers.twitter_archiver import TwitterArchiver

from .test_archiver_base import TestArchiverBase


class TestTwitterArchiver(TestArchiverBase, unittest.TestCase):

    archiver_class = TwitterArchiver
    config = {}
    
    def test_sanitize_url(self):

        # should expand t.co URLs
        t_co_url = "https://t.co/yl3oOJatFp"
        t_co_resolved_url = "https://www.bellingcat.com/category/resources/"
        self.assertEqual(t_co_resolved_url, self.archiver.sanitize_url(t_co_url))

        # shouldn't alter valid x URLs
        x_url = "https://x.com/bellingcat/status/1874097816571961839"
        self.assertEqual(x_url, self.archiver.sanitize_url(x_url))

        # shouldn't alter valid twitter.com URLs
        twitter_url = "https://twitter.com/bellingcat/status/1874097816571961839"
        self.assertEqual(twitter_url, self.archiver.sanitize_url(twitter_url))

        # should strip tracking params
        tracking_url = "https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"
        self.assertEqual("https://twitter.com/bellingcat/status/1874097816571961839", self.archiver.sanitize_url(tracking_url))

        # shouldn't alter non-twitter/x URLs
        test_url = "https://www.bellingcat.com/category/resources/"
        self.assertEqual(test_url, self.archiver.sanitize_url(test_url))

        # shouldn't strip params from non-twitter/x URLs
        test_url = "https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"
        self.assertEqual(test_url, self.archiver.sanitize_url(test_url))

    def test_get_username_tweet_id_from_url(self):

        # test valid twitter URL
        url = "https://twitter.com/bellingcat/status/1874097816571961839"
        username, tweet_id = self.archiver.get_username_tweet_id(url)
        self.assertEqual("bellingcat", username)
        self.assertEqual("1874097816571961839", tweet_id)

        # test valid x URL
        url = "https://x.com/bellingcat/status/1874097816571961839"
        username, tweet_id = self.archiver.get_username_tweet_id(url)
        self.assertEqual("bellingcat", username)
        self.assertEqual("1874097816571961839", tweet_id)

        # test invalid URL
        # TODO: should this return None, False or raise an exception? Right now it returns False
        url = "https://www.bellingcat.com/category/resources/"
        username, tweet_id = self.archiver.get_username_tweet_id(url)
        self.assertFalse(username)
        self.assertFalse(tweet_id)

    def test_youtube_dlp_archiver(self):

        url = "https://x.com/bellingcat/status/1874097816571961839"
        post = self.archiver.download_yt_dlp(self.create_item(url), url, "1874097816571961839")
        self.assertTrue(post)
        self.assertValidResponseMetadata(
            post,
            "As 2024 comes to a close, here’s some examples of what Bellingcat investigated per month in our 10th year! 🧵",
            datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)
        )
    
    def test_reverse_engineer_token(self):
        # see Vercel's implementation here: https://github.com/vercel/react-tweet/blob/main/packages/react-tweet/src/api/fetch-tweet.ts#L27C1-L31C2
        # and the discussion here: https://github.com/JustAnotherArchivist/snscrape/issues/996#issuecomment-2211358215

        for tweet_id, real_token in [
            ("1874097816571961839", "4jjngwkifa"),
            ("1674700676612386816", "42586mwa3uv"),
            ("1877747914073620506", "4jv4aahw36n"),
            ("1876710769913450647", "4jruzjz5lux"),
            ("1346554693649113090", "39ibqxei7mo"),]:
            generated_token = self.archiver.generate_token(tweet_id)
            self.assertEqual(real_token, generated_token)
                         
    def test_syndication_archiver(self):

        url = "https://x.com/bellingcat/status/1874097816571961839"
        post = self.archiver.download_syndication(self.create_item(url), url, "1874097816571961839")
        self.assertTrue(post)
        self.assertValidResponseMetadata(
            post,
            "As 2024 comes to a close, here’s some examples of what Bellingcat investigated per month in our 10th year! 🧵",
            datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)
        )

    def test_download_nonexistend_tweet(self):
        # this tweet does not exist
        url = "https://x.com/Bellingcat/status/17197025860711058"
        response = self.archiver.download(self.create_item(url))
        self.assertFalse(response)

    def test_download_malformed_tweetid(self):
        # this tweet does not exist
        url = "https://x.com/Bellingcat/status/1719702586071100058"
        response = self.archiver.download(self.create_item(url))
        self.assertFalse(response)

    def test_download_media_with_images(self):
        # url https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w
        
        post = self.archiver.download()

        # just make sure twitter haven't changed their format, images should be under "record/embed/media/images"
        # there should be 2 images
        self.assertTrue("record" in post)
        self.assertTrue("embed" in post["record"])
        self.assertTrue("media" in post["record"]["embed"])
        self.assertTrue("images" in post["record"]["embed"]["media"])
        self.assertEqual(len(post["record"]["embed"]["media"]["images"]), 2)

        # try downloading the media files
        media = self.archiver.download(post)
        self.assertEqual(len(media), 2)

        # check the IDs
        self.assertTrue("bafkreiflrkfihcvwlhka5tb2opw2qog6gfvywsdzdlibveys2acozh75tq" in media[0].get('src'))
        self.assertTrue("bafkreibsprmwchf7r6xcstqkdvvuj3ijw7efciw7l3y4crxr4cmynseo7u" in media[1].get('src'))