From f3997ff6ae1f28a1aca8ed59c665457908e7ed62 Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Mon, 1 Aug 2022 09:58:52 +0000 Subject: [PATCH] Catch errors in Bitchute channel profile scraper; add multi index on posts forwarded from/channel --- cisticola/base.py | 4 +++- cisticola/scraper/bitchute.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cisticola/base.py b/cisticola/base.py index d1f80b8..01df6f4 100644 --- a/cisticola/base.py +++ b/cisticola/base.py @@ -6,7 +6,7 @@ import json import io from sqlalchemy.orm import registry -from sqlalchemy import Table, Column, Integer, String, JSON, DateTime, ForeignKey, Boolean +from sqlalchemy import Table, Column, Integer, String, JSON, DateTime, ForeignKey, Boolean, Index import pytesseract import PIL import exiftool @@ -515,6 +515,8 @@ post_table = Table('posts', mapper_registry.metadata, Column('normalized_content', String) ) +posts_forwarded_from_channel_index = Index('posts_channel_forwarded_from_idx', post_table.c.channel, post_table.c.forwarded_from) + media_table = Table('media', mapper_registry.metadata, Column('id', Integer, primary_key=True, autoincrement=True), diff --git a/cisticola/scraper/bitchute.py b/cisticola/scraper/bitchute.py index b640e1d..284b300 100644 --- a/cisticola/scraper/bitchute.py +++ b/cisticola/scraper/bitchute.py @@ -71,7 +71,7 @@ class BitchuteScraper(Scraper): if channel.platform == "Bitchute" and self.get_username_from_url(channel.url) is not None: return True - @logger.catch(reraise = True) + @logger.catch def get_profile(self, channel: Channel) -> RawChannelInfo: base_url = channel.url