added capability to scrape Gab group posts

This commit is contained in:
Tristan Lee
2022-03-30 09:11:07 -05:00
parent 1f99e52436
commit b7871b060d
3 changed files with 65 additions and 9 deletions

View File

@@ -33,6 +33,19 @@ GAB_CHANNEL_KWARGS = {
'chat': False,
'notes': ''}
GAB_GROUP_KWARGS = {
'name': 'iran group (test)',
'platform_id': 10001,
'category': 'test',
'platform': 'Gab',
'url': 'https://gab.com/groups/10001',
'screenname': 'iran group',
'country': 'IR',
'influencer': None,
'public': True,
'chat': True,
'notes': ''}
GETTR_CHANNEL_KWARGS = {
'name': 'LizardRepublic (test)',
'platform_id': 'lizardrepublic',
@@ -178,6 +191,7 @@ def channel_kwargs():
return {
'bitchute' : BITCHUTE_CHANNEL_KWARGS,
'gab' : GAB_CHANNEL_KWARGS,
'gab_group' : GAB_GROUP_KWARGS,
'gettr' : GETTR_CHANNEL_KWARGS,
'instagram' : INSTAGRAM_CHANNEL_KWARGS,
'odysee' : ODYSEE_CHANNEL_KWARGS,

View File

@@ -23,4 +23,26 @@ def test_scrape_gab_profile(channel_kwargs):
scraper = GabScraper()
channel = Channel(**channel_kwargs['gab'])
scraper.get_profile(channel=channel)
def test_scrape_gab_group_no_media(controller, channel_kwargs):
channels = [Channel(**channel_kwargs['gab_group'])]
controller.register_scraper(scraper = GabScraper())
controller.scrape_channels(channels = channels, archive_media = False)
@pytest.mark.media
def test_scrape_gab_group(controller, channel_kwargs):
controller.reset_db()
channels = [Channel(**channel_kwargs['gab_group'])]
controller.register_scraper(scraper = GabScraper())
controller.scrape_channels(channels = channels, archive_media = True)
@pytest.mark.profile
def test_scrape_gab_group_profile(channel_kwargs):
scraper = GabScraper()
channel = Channel(**channel_kwargs['gab_group'])
scraper.get_profile(channel=channel)