From b8efce2a127af07f03d83000cb08f4cbc4a45029 Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Tue, 8 Mar 2022 15:10:15 +0100 Subject: [PATCH 1/5] Clean up unnecessary imports --- snscrape/modules/telegram.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 8b0ece7..ed81222 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -9,7 +9,6 @@ import re import snscrape.base import typing import urllib.parse -import base64 _logger = logging.getLogger(__name__) _SINGLE_MEDIA_LINK_PATTERN = re.compile(r'^https://t\.me/[^/]+/\d+\?single$') @@ -123,9 +122,6 @@ class TelegramChannelScraper(snscrape.base.Scraper): image = re.findall('url\(\'(.*?)\'\)', style) if len(image) == 1: images.append(image[0]) - # resp = self._get(image[0]) - # encoded_string = base64.b64encode(resp.content) - # Individual photo or video link continue href = urllib.parse.urljoin(pageUrl, link['href']) if href not in outlinks: From 65723f10ff7afbba075e8077e65c8836bceab2ef Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 25 May 2022 06:47:47 -0500 Subject: [PATCH 2/5] fixed merge --- snscrape/modules/telegram.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 1008328..395e9a3 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -203,36 +203,6 @@ class TelegramChannelScraper(snscrape.base.Scraper): mKwargs['duration'] = durationStrToSeconds(durationStr) media.append(cls(**mKwargs)) -<<<<<<< HEAD - outlinks = [] - for link in post.find_all('a'): - if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): - # Author links at the top (avatar and name) - continue - if link['href'] == rawUrl or link['href'] == url: - style = link.attrs.get('style', '') - # Generic filter of links to the post itself, catches videos, photos, and the date link - if style != '': - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) - continue - if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): - style = link.attrs.get('style', '') - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) - continue - href = urllib.parse.urljoin(pageUrl, link['href']) - if href not in outlinks: - outlinks.append(href) - else: - content = None - outlinks = [] - images = [] - video = None -======= ->>>>>>> master linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} From 56e4232083879c624954a9d8c8953112f25b7bff Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 23 Jun 2022 11:51:13 -0500 Subject: [PATCH 3/5] fixed typo --- snscrape/modules/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 395e9a3..3e3a512 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -321,7 +321,7 @@ def parse_num(s): def durationStrToSeconds(durationStr): durationList = durationStr.split(':') - return sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationList))]) + return sum([int(s) * int(g) for s, g in zip([1, 60, 3600], reversed(durationList))]) def telegramResponseOkCallback(r): if r.status_code == 200: From 056cd6215cf96af517711bb07c46f46078233d39 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 23 Jun 2022 15:47:18 -0500 Subject: [PATCH 4/5] incorporated requested changes from maintainer, removed modifications to VK module --- snscrape/modules/telegram.py | 23 ++++++++++++++--------- snscrape/modules/vkontakte.py | 8 ++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 3e3a512..16f2dc5 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -56,7 +56,7 @@ class TelegramPost(snscrape.base.Item): forwarded: typing.Optional['Channel'] = None forwardedUrl: typing.Optional[str] = None media: typing.Optional[typing.List['Medium']] = None - views: typing.Optional[int] = None + views: typing.Optional[snscrape.base.IntWithGranularity] = None linkPreview: typing.Optional[LinkPreview] = None outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') @@ -175,7 +175,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): for voicePlayer in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): audioUrl = voicePlayer.find('audio')['src'] durationStr = voicePlayer.find('time').text - duration = durationStrToSeconds(durationStr) + duration = _durationStrToSeconds(durationStr) barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voicePlayer.find('div', {'class': 'bar'}).find_all('s')] media.append(VoiceMessage(url = audioUrl, duration = duration, bars = barHeights)) @@ -200,7 +200,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): else: cls = Video durationStr = videoPlayer.find('time').text - mKwargs['duration'] = durationStrToSeconds(durationStr) + mKwargs['duration'] = _durationStrToSeconds(durationStr) media.append(cls(**mKwargs)) linkPreview = None @@ -223,7 +223,12 @@ class TelegramChannelScraper(snscrape.base.Scraper): outlinks.remove(kwargs['href']) viewsSpan = post.find('span', class_ = 'tgme_widget_message_views') - views = None if viewsSpan is None else parse_num(viewsSpan.text) + views = None if viewsSpan is None else _parse_num(viewsSpan.text) + + outlinks = outlinks if outlinks else None + media = media if media else None + mentions = mentions if mentions else None + hashtags = hashtags if hashtags else None yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, mentions = mentions, hashtags = hashtags, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) @@ -252,7 +257,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): else: break nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) - r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) + r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = _telegramResponseOkCallback) if r.status_code != 200: raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') @@ -293,7 +298,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): kwargs['description'] = descriptionDiv.text for div in channelInfoDiv.find_all('div', class_ = 'tgme_channel_info_counter'): - value, granularity = parse_num(div.find('span', class_ = 'counter_value').text) + value, granularity = _parse_num(div.find('span', class_ = 'counter_value').text) type_ = div.find('span', class_ = 'counter_type').text if type_ == 'members': # Already extracted more accurately from /channel, skip @@ -311,7 +316,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): def _cli_from_args(cls, args): return cls._cli_construct(args, args.channel) -def parse_num(s): +def _parse_num(s): s = s.replace(' ', '') if s.endswith('M'): return int(float(s[:-1]) * 1e6), 10 ** (6 if '.' not in s else 6 - len(s[:-1].split('.')[1])) @@ -319,11 +324,11 @@ def parse_num(s): return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) return int(s), 1 -def durationStrToSeconds(durationStr): +def _durationStrToSeconds(durationStr): durationList = durationStr.split(':') return sum([int(s) * int(g) for s, g in zip([1, 60, 3600], reversed(durationList))]) -def telegramResponseOkCallback(r): +def _telegramResponseOkCallback(r): if r.status_code == 200: return (True, None) return (False, f'{r.status_code=}') diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index 3193abe..9353cd5 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -177,15 +177,11 @@ class VKontakteUserScraper(snscrape.base.Scraper): continue if 'data-video' in a.attrs: # Video - if 'data-link-attr' in a.attrs: - hrefUrl = urllib.parse.unquote(a.attrs['data-link-attr'].split('to=')[1].split('&')[0]) - else: - hrefUrl = f'https://vk.com{a["href"]}' video = Video( id = a['data-video'], list = a['data-list'], duration = int(a['data-duration']), - url = hrefUrl, + url = f'https://vk.com{a["href"]}', thumbUrl = a['style'][(begin := a['style'].find('background-image: url(') + 22) : a['style'].find(')', begin)], ) continue @@ -389,4 +385,4 @@ class VKontakteUserScraper(snscrape.base.Scraper): @classmethod def _cli_from_args(cls, args): - return cls._cli_construct(args, args.username) + return cls._cli_construct(args, args.username) \ No newline at end of file From 73f10a4f24a42e9642bcea9dc955ed91c92de41e Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Tue, 5 Jul 2022 10:23:26 -0500 Subject: [PATCH 5/5] fixed edge case where channel with no members fails _get_entity --- snscrape/modules/telegram.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 16f2dc5..2e1322f 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -270,8 +270,12 @@ class TelegramChannelScraper(snscrape.base.Scraper): raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') membersDiv = soup.find('div', class_ = 'tgme_page_extra') - if membersDiv.text.endswith((' members', ' subscribers')): - kwargs['members'] = int(''.join(membersDiv.text.split(' ')[:-1])) + if membersDiv.text.split(',')[0].endswith((' members', ' subscribers')): + membersStr = ''.join(membersDiv.text.split(',')[0].split(' ')[:-1]) + if membersStr == 'no': + kwargs['members'] = 0 + else: + kwargs['members'] = int(membersStr) photoImg = soup.find('img', class_ = 'tgme_page_photo_image') if photoImg is not None: kwargs['photo'] = photoImg.attrs['src']