bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-13 04:48:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix link previews for pure-image previews

Browse Source 
... and any other preview that doesn't have all the things for some reason.
This commit is contained in:
JustAnotherArchivist
2020-10-01 18:56:55 +00:00
parent c72bf3174f
commit 8b68f1a8af
1 changed files with 13 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
snscrape/modules/telegram.py
View File

@@ -11,9 +11,9 @@ logger = logging.getLogger(__name__)
class LinkPreview(typing.NamedTuple): class LinkPreview(typing.NamedTuple):
href: str href: str
siteName: str siteName: typing.Optional[str] = None
title: str title: typing.Optional[str] = None
description: str description: typing.Optional[str] = None
image: typing.Optional[str] = None image: typing.Optional[str] = None
@@ -94,19 +94,20 @@ class TelegramChannelScraper(snscrape.base.Scraper):
outlinksss = '' outlinksss = ''
linkPreview = None linkPreview = None
if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')):
image = None kwargs = {}
kwargs['href'] = urllib.parse.urljoin(pageUrl, linkPreviewA['href'])
if (siteNameDiv := linkPreviewA.find('div', class_ = 'link_preview_site_name')):
kwargs['siteName'] = siteNameDiv.text
if (titleDiv := linkPreviewA.find('div', class_ = 'link_preview_title')):
kwargs['title'] = titleDiv.text
if (descriptionDiv := linkPreviewA.find('div', class_ = 'link_preview_description')):
kwargs['description'] = descriptionDiv.text
if (imageI := linkPreviewA.find('i', class_ = 'link_preview_image')): if (imageI := linkPreviewA.find('i', class_ = 'link_preview_image')):
if imageI['style'].startswith("background-image:url('"): if imageI['style'].startswith("background-image:url('"):
image = imageI['style'][22 : imageI['style'].index("'", 22)] kwargs['image'] = imageI['style'][22 : imageI['style'].index("'", 22)]
else: else:
self.logger.warning(f'Could not process link preview image on https://t.me/s/{post["data-post"]}') self.logger.warning(f'Could not process link preview image on https://t.me/s/{post["data-post"]}')
linkPreview = LinkPreview( linkPreview = LinkPreview(**kwargs)
href = urllib.parse.urljoin(pageUrl, linkPreviewA['href']),
siteName = linkPreviewA.find('div', class_ = 'link_preview_site_name').text,
title = linkPreviewA.find('div', class_ = 'link_preview_title').text,
description = linkPreviewA.find('div', class_ = 'link_preview_description').text,
image = image,
)
yield TelegramPost(url = f'https://t.me/s/{post["data-post"]}', date = date, content = content, outlinks = outlinks, outlinksss = outlinksss, linkPreview = linkPreview) yield TelegramPost(url = f'https://t.me/s/{post["data-post"]}', date = date, content = content, outlinks = outlinks, outlinksss = outlinksss, linkPreview = linkPreview)
def get_items(self): def get_items(self):