Merge pull request #52 from bellingcat/next-release

Next release

app.py

@@ -32,24 +32,25 @@ def sync_channels(args):
     row = 2
 
     for c in channels:
-        # only adding channels, so skip everything with an ID
-        if c["id"] == "":
+        if c["public"] == "":
+            c["public"] = False
+        if c["chat"] == "":
+            c["chat"] = False
+
+        for k in c.keys():
+            if c[k] == "TRUE" or c[k] == "yes":
+                c[k] = True
+            if c[k] == "FALSE" or c[k] == "no":
+                c[k] = False
+
+            if c[k] == "":
+                c[k] = None
+
+        del c["followers"]
+
+        # add new channel
+        if c["id"] == "" or c["id"] is None:
             del c["id"]
-            del c["followers"]
-
-            if c["public"] == "":
-                c["public"] = False
-            if c["chat"] == "":
-                c["chat"] = False
-
-            for k in c.keys():
-                if c[k] == "TRUE" or c[k] == "yes":
-                    c[k] = True
-                if c[k] == "FALSE" or c[k] == "no":
-                    c[k] = False
-
-                if c[k] == "":
-                    c[k] = None
 
             # check to see if this already exists,
             platform_id = None
@@ -73,6 +74,23 @@ def sync_channels(args):
 
                 wks.update_cell(row, 1, channel.id)
                 time.sleep(1)
+        else:
+            channel = session.query(Channel).filter_by(id=int(c["id"])).first()
+
+            logger.info(f"Updating channel {channel}")
+            channel.name = c["name"]
+            channel.category = c["category"]
+            channel.platform = c["platform"]
+            channel.url = c["url"]
+            channel.screenname = c["screenname"]
+            channel.country = c["country"]
+            channel.influencer = c["influencer"]
+            channel.public = c["public"]
+            channel.chat = c["chat"]
+            channel.notes = c["notes"]
+
+            session.flush()
+            session.commit()
 
         row += 1
 

cisticola/scraper/bitchute.py

@@ -70,6 +70,7 @@ class BitchuteScraper(Scraper):
         if channel.platform == "Bitchute" and self.get_username_from_url(channel.url) is not None:
             return True
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         base_url = channel.url

cisticola/scraper/gab.py

@@ -89,6 +89,7 @@ class GabScraper(Scraper):
         if channel.platform == "Gab" and self.get_username_from_url(channel.url) is not None:
             return True
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         client = Client(

cisticola/scraper/gettr.py

@@ -72,6 +72,7 @@ class GettrScraper(Scraper):
         key = urlparse(url).path.split('/')[-2] + ext
         return key 
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
         client = PublicClient()
         username = self.get_username_from_url(channel.url)

cisticola/scraper/instagram.py

@@ -91,6 +91,7 @@ class InstagramScraper(Scraper):
         if channel.platform == "Instagram" and self.get_username_from_url(channel.url) is not None:
             return True
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         username = self.get_username_from_url(channel.url)

cisticola/scraper/odysee.py

@@ -105,6 +105,7 @@ class OdyseeScraper(Scraper):
 
         return f'{key}.{ext}'
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         username = self.get_username_from_url(channel.url)

cisticola/scraper/rumble.py

@@ -69,6 +69,7 @@ class RumbleScraper(Scraper):
         if channel.platform == "Rumble" and channel.url is not None:
             return True
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         profile = get_channel_profile(url = channel.url)

cisticola/scraper/telegram_snscrape.py

@@ -55,6 +55,7 @@ class TelegramSnscrapeScraper(Scraper):
                 media_archived=datetime.now(timezone.utc) if archive_media else None
             )
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         scr = snscrape.modules.telegram.TelegramChannelScraper(

cisticola/scraper/telegram_telethon.py

@@ -149,6 +149,7 @@ class TelegramTelethonScraper(Scraper):
                     archived_urls=archived_urls,
                     media_archived=datetime.now(timezone.utc) if archive_media else None)
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
         username = channel.screenname
         if username is None:

cisticola/scraper/twitter.py

@@ -97,6 +97,7 @@ class TwitterScraper(Scraper):
         key = parsed_url.path.split('/')[-1] + ext
         return key 
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         scraper = TwitterUserScraper(channel.screenname)

cisticola/scraper/vkontakte.py

@@ -103,6 +103,7 @@ class VkontakteScraper(Scraper):
             
         return key
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
 
         username = self.get_username_from_url(channel.url)

cisticola/scraper/youtube.py

@@ -138,6 +138,7 @@ class YoutubeScraper(Scraper):
         result.media_archived = datetime.now(timezone.utc)
         return result
 
+    @logger.catch
     def get_profile(self, channel: Channel) -> RawChannelInfo:
         
         ydl_opts = {