bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-08 03:18:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix type hints

Browse Source
This commit is contained in:
Logan Williams
2022-02-24 20:36:23 +01:00
parent 1ad7c8bc11
commit 3480452fac
5 changed files with 10 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cisticola/scraper/base.py
View File

@@ -1,4 +1,4 @@
from typing import List
from typing import Generator
import cisticola.base
import requests
import os
@@ -55,5 +55,5 @@ class Scraper:
def can_handle(self, channel: cisticola.base.Channel) -> bool:
pass
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> List[cisticola.base.ScraperResult]:
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> Generator[cisticola.base.ScraperResult, None, None]:
pass

4
cisticola/scraper/bitchute.py
View File

@@ -4,7 +4,7 @@ import re
from html.parser import HTMLParser
import dateparser
import json
from typing import List
from typing import Generator
import requests
from bs4 import BeautifulSoup
@@ -23,7 +23,7 @@ class BitchuteScraper(cisticola.scraper.Scraper):
return username
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> List[cisticola.base.ScraperResult]:
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> Generator[cisticola.base.ScraperResult, None, None]:
session = requests.Session()
session.headers["User-Agent"] = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:84.0) Gecko/20100101 Firefox/84.0"

4
cisticola/scraper/gettr.py
View File

@@ -2,7 +2,7 @@ import cisticola.base
import cisticola.scraper.base
from datetime import datetime
import json
from typing import List
from typing import Generator
from gogettr import PublicClient
class GettrScraper(cisticola.scraper.base.Scraper):
@@ -16,7 +16,7 @@ class GettrScraper(cisticola.scraper.base.Scraper):
return username
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> List[cisticola.base.ScraperResult]:
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> Generator[cisticola.base.ScraperResult, None, None]:
client = PublicClient()
username = GettrScraper.get_username_from_url(channel.url)
scraper = client.user_activity(username=username, type="posts")

4
cisticola/scraper/telegram_snscrape.py
View File

@@ -1,6 +1,6 @@
import cisticola.base
import cisticola.scraper.base
from typing import List
from typing import Generator
import snscrape.modules
from datetime import datetime, timezone
@@ -12,7 +12,7 @@ class TelegramSnscrapeScraper(cisticola.scraper.base.Scraper):
if channel.platform == "Telegram" and channel.public and not channel.chat:
return True
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None):
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> Generator[cisticola.base.ScraperResult, None, None]:
scr = snscrape.modules.telegram.TelegramChannelScraper(
channel.screenname)

5
cisticola/scraper/twitter.py
View File

@@ -1,7 +1,7 @@
import cisticola.base
import cisticola.scraper.base
from datetime import datetime, timezone
from typing import List
from typing import Generator
import snscrape.modules
from loguru import logger
@@ -10,7 +10,7 @@ class TwitterScraper(cisticola.scraper.base.Scraper):
"""An implementation of a Scraper for Twitter, using snscrape library"""
__version__ = "TwitterScraper 0.0.1"
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> List[cisticola.base.ScraperResult]:
def get_posts(self, channel: cisticola.base.Channel, since: cisticola.base.ScraperResult = None) -> Generator[cisticola.base.ScraperResult, None, None]:
scraper = snscrape.modules.twitter.TwitterProfileScraper(channel.platform_id)
first = True
@@ -22,7 +22,6 @@ class TwitterScraper(cisticola.scraper.base.Scraper):
first = False
continue
else:
print('too far')
break
archived_urls = {}