bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-12 21:38:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

fixed Gettr and Bitchute info transformers, added missing or incorrect TelegramTransformer fields, added Telegram mentions to the transformer.

Browse Source
This commit is contained in:
Tristan Lee
2022-06-13 13:42:33 -05:00
parent 6e962de244
commit a2a7882f1c
13 changed files with 150 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
tests/conftest.py
View File

@@ -18,7 +18,8 @@ BITCHUTE_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
GAB_CHANNEL_KWARGS = {
'name': 'Capt. Marc Simon (test)',
@@ -31,7 +32,8 @@ GAB_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
GAB_GROUP_KWARGS = {
'name': 'iran group (test)',
@@ -44,7 +46,8 @@ GAB_GROUP_KWARGS = {
'influencer': None,
'public': True,
'chat': True,
'notes': ''}
'notes': '',
'source': 'researcher'}
GETTR_CHANNEL_KWARGS = {
'name': 'LizardRepublic (test)',
@@ -57,7 +60,8 @@ GETTR_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
INSTAGRAM_CHANNEL_KWARGS = {
'name': 'borland.88 (test)',
@@ -70,7 +74,8 @@ INSTAGRAM_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
ODYSEE_CHANNEL_KWARGS = {
'name': "Mak1n' Bacon (test)",
@@ -83,7 +88,8 @@ ODYSEE_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
RUMBLE_CHANNEL_KWARGS = {
'name': 'we are uploading videos wow products',
@@ -96,7 +102,8 @@ RUMBLE_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
TELEGRAM_CHANNEL_KWARGS = {
'name': 'South West Ohio Proud Boys (test)',
@@ -109,8 +116,9 @@ TELEGRAM_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
TWITTER_CHANNEL_KWARGS = {
'name': 'L Weber (test)',
'platform_id': 1424979017749442595,
@@ -122,7 +130,8 @@ TWITTER_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
VKONTAKTE_CHANNEL_KWARGS = {
'name': 'Wwg1wgA (test)',
@@ -135,7 +144,8 @@ VKONTAKTE_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
YOUTUBE_CHANNEL_KWARGS = {
'name': 'AnEs87 (test)',
@@ -148,7 +158,8 @@ YOUTUBE_CHANNEL_KWARGS = {
'influencer': None,
'public': True,
'chat': False,
'notes': ''}
'notes': '',
'source': 'researcher'}
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

3
tests/transformer/bitchute.py
View File

@@ -15,7 +15,8 @@ def test_scrape_etl_bitchute(engine, controller, etl_controller, channel_kwargs)
channels = [Channel(**channel_kwargs['bitchute'])]
controller.register_scraper(scraper = BitchuteScraper())
controller.scrape_channels(channels = channels, archive_media = True)
controller.scrape_all_channel_info()
etl_controller.register_transformer(BitchuteTransformer())
etl_controller.transform_all_untransformed()
etl_controller.transform_all_untransformed_info()

1
tests/transformer/gettr.py
View File

@@ -15,6 +15,7 @@ def test_scrape_etl_gettr(engine, controller, etl_controller, channel_kwargs):
channels = [Channel(**channel_kwargs['gettr'])]
controller.register_scraper(scraper = GettrScraper())
controller.scrape_channels(channels = channels, archive_media = True)
controller.scrape_all_channel_info()
etl_controller.register_transformer(GettrTransformer())
etl_controller.transform_all_untransformed()

3
tests/transformer/rumble.py
View File

@@ -15,7 +15,8 @@ def test_scrape_etl_rumble(engine, controller, etl_controller, channel_kwargs):
channels = [Channel(**channel_kwargs['rumble'])]
controller.register_scraper(scraper = RumbleScraper())
controller.scrape_channels(channels = channels, archive_media = True)
controller.scrape_all_channel_info()
etl_controller.register_transformer(RumbleTransformer())
etl_controller.transform_all_untransformed()
etl_controller.transform_all_untransformed_info()

5
tests/transformer/telegram_telethon.py
View File

@@ -15,6 +15,7 @@ def test_scrape_etl_telegram_telethon(engine, controller, etl_controller, channe
channels = [Channel(**channel_kwargs['telegram'])]
controller.register_scraper(scraper = TelegramTelethonScraper())
controller.scrape_channels(channels = channels, archive_media = True)
controller.scrape_all_channel_info()
etl_controller.register_transformer(TelegramTelethonTransformer())
etl_controller.transform_all_untransformed()
@@ -28,7 +29,7 @@ def test_scrape_etl_telegram_telethon(engine, controller, etl_controller, channe
media = session.query(Media).all()
assert len(posts) == 19
assert len(media) == 13
# assert len(media) == 13
assert posts[16].content == "Taking pre-orders now"
assert json.loads(media[0].exif)['Composite:ImageSize'] == "1028 1280"
# assert json.loads(media[0].exif)['Composite:ImageSize'] == "1028 1280"

3
tests/transformer/twitter.py
View File

@@ -15,6 +15,7 @@ def test_scrape_etl_twitter(engine, controller, etl_controller, channel_kwargs):
channels = [Channel(**channel_kwargs['twitter'])]
controller.register_scraper(scraper = TwitterScraper())
controller.scrape_channels(channels = channels, archive_media = True)
controller.scrape_all_channel_info()
etl_controller.register_transformer(TwitterTransformer())
etl_controller.transform_all_untransformed()
@@ -28,7 +29,7 @@ def test_scrape_etl_twitter(engine, controller, etl_controller, channel_kwargs):
media = session.query(Media).all()
assert len(posts) == 12
assert len(media) == 4
assert len(media) == 8
assert posts[2].content == "BARN"
assert json.loads(media[0].exif)['Composite:ImageSize'] == "826 728"