From 119e53d07cf9a3bf2051c078dc349dee520b11c0 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Wed, 7 Oct 2020 01:15:51 +0000 Subject: [PATCH] Fix Telegram post URL extraction --- snscrape/modules/telegram.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index caece1a..8ecbb32 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -73,7 +73,12 @@ class TelegramChannelScraper(snscrape.base.Scraper): if onlyUsername: yield post['data-post'].split('/')[0] return - date = datetime.datetime.strptime(post.find('div', class_ = 'tgme_widget_message_footer').find('a', class_ = 'tgme_widget_message_date').find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') + dateDiv = post.find('div', class_ = 'tgme_widget_message_footer').find('a', class_ = 'tgme_widget_message_date') + rawUrl = dateDiv['href'] + if not rawUrl.startswith('https://t.me/') or sum(x == '/' for x in rawUrl) != 4 or rawUrl.rsplit('/', 1)[1].strip('0123456789') != '': + self.logger.warning(f'Possibly incorrect URL: {rawUrl!r}') + url = rawUrl.replace('//t.me/', '//t.me/s/') + date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.text outlinks = [] @@ -81,7 +86,7 @@ class TelegramChannelScraper(snscrape.base.Scraper): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): # Author links at the top (avatar and name) continue - if link['href'] == f'https://t.me/{post["data-post"]}': + if link['href'] == rawUrl or link['href'] == url: # Generic filter of links to the post itself, catches videos, photos, and the date link continue href = urllib.parse.urljoin(pageUrl, link['href']) @@ -106,9 +111,9 @@ class TelegramChannelScraper(snscrape.base.Scraper): if imageI['style'].startswith("background-image:url('"): kwargs['image'] = imageI['style'][22 : imageI['style'].index("'", 22)] else: - self.logger.warning(f'Could not process link preview image on https://t.me/s/{post["data-post"]}') + self.logger.warning(f'Could not process link preview image on {url}') linkPreview = LinkPreview(**kwargs) - yield TelegramPost(url = f'https://t.me/s/{post["data-post"]}', date = date, content = content, outlinks = outlinks, outlinksss = outlinksss, linkPreview = linkPreview) + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, outlinksss = outlinksss, linkPreview = linkPreview) def get_items(self): r, soup = self._initial_page()