included comments in Odysee scraper

This commit is contained in:
Tristan Lee
2022-02-28 09:15:09 -06:00
parent 36fb95d9ae
commit 7a257ea9f5
2 changed files with 19 additions and 5 deletions

10
Pipfile.lock generated
View File

@@ -211,11 +211,11 @@
},
"importlib-metadata": {
"hashes": [
"sha256:175f4ee440a0317f6e8d81b7f8d4869f93316170a65ad2b007d2929186c8052c",
"sha256:e0bc84ff355328a4adfc5240c4f211e0ab386f80aa640d1b11f0618a1d282094"
"sha256:b36ffa925fe3139b2f6ff11d6925ffd4fa7bc47870165e3ac260ac7b4f91e6ac",
"sha256:d16e8c1deb60de41b8e8ed21c1a7b947b0bc62fab7e1d470bcdf331cea2e6735"
],
"markers": "python_version < '3.10'",
"version": "==4.11.1"
"version": "==4.11.2"
},
"jinja2": {
"hashes": [
@@ -416,7 +416,7 @@
},
"polyphemus": {
"git": "https://github.com/bellingcat/polyphemus.git",
"ref": "72ea0a63de4b40bf8038dfdb26cbbab87ba86da9"
"ref": "18b89f19ecdd32e7dc8b5564b258a67165e680ca"
},
"pygments": {
"hashes": [
@@ -709,7 +709,7 @@
"sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
"sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'",
"version": "==1.26.8"
},
"zipp": {

View File

@@ -32,6 +32,8 @@ class OdyseeScraper(cisticola.scraper.base.Scraper):
archived_url = self.archive_media(media_blob, content_type, key)
archived_urls[url] = archived_url
all_comments = video.get_all_comments()
yield cisticola.base.ScraperResult(
scraper=self.__version__,
platform="Odysee",
@@ -42,6 +44,18 @@ class OdyseeScraper(cisticola.scraper.base.Scraper):
raw_data=json.dumps(video.info),
archived_urls=archived_urls)
for comment in all_comments:
yield cisticola.base.ScraperResult(
scraper=self.__version__,
platform="Odysee",
channel=channel.id,
platform_id=comment.info['claim_id'],
date=datetime.fromtimestamp(comment.info['created']),
date_archived=datetime.now(),
raw_data=json.dumps(comment.info),
archived_urls=archived_urls)
def can_handle(self, channel):
if channel.platform == "Odysee" and OdyseeScraper.get_username_from_url(channel.url) is not None:
return True