bellingcat/polyphemus
1
0
Fork 0
mirror of https://github.com/bellingcat/polyphemus.git synced 2026-06-07 19:08:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

added unit tests and made keyword arguments more consistent

Browse Source
This commit is contained in:
Tristan Lee
2022-03-02 09:44:17 -06:00
parent 00799059b3
commit 8dc9a9b28e
8 changed files with 327 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
.gitignore vendored
View File

@@ -1,8 +1,83 @@
*.csv
**/data/
# Ignore all non-Python files in examples directory
examples/*
!examples/*.py
**/__pycache__/
*.pyc
# Sphinx documentation
docs/_build/
docs/source/
# Test and coverage reports
reports
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# pytype static type analyzer
.pytype/

44
polyphemus/api.py
View File

@@ -87,14 +87,14 @@ def get_channel_info(channel_name):
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def get_subscribers(claim_id):
def get_subscribers(channel_id):
"""Get the number of subscribers for a channel.
"""
json_data = {
'auth_token': AUTH_TOKEN,
'claim_id': claim_id }
'claim_id': channel_id }
response = make_request(
request = requests.post,
@@ -156,14 +156,14 @@ def get_all_videos(channel_id):
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def get_views(claim_id):
def get_views(video_id):
"""Get the number of views for a given video.
"""
params = {
'auth_token': AUTH_TOKEN,
'claim_id': claim_id }
'claim_id': video_id }
response = make_request(
request = requests.get,
@@ -177,14 +177,14 @@ def get_views(claim_id):
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def get_video_reactions(claim_id):
def get_video_reactions(video_id):
"""Get all reactions for a given video.
"""
post_data = {
'auth_token': AUTH_TOKEN,
'claim_ids': claim_id }
'claim_ids': video_id }
response = make_request(
request = requests.post,
@@ -195,20 +195,20 @@ def get_video_reactions(claim_id):
result = json.loads(response.text)
if result['success']:
reactions = result['data']['others_reactions'][claim_id ]
reactions = result['data']['others_reactions'][video_id]
return reactions['like'], reactions['dislike']
else:
return None, None
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def get_all_comments(claim_id):
def get_all_comments(video_id):
"""Get a list of all comments for a single video.
Parameters
----------
claim_id: str
video_id: str
Claim ID for the video whose comments are to be scraped
e.g. ``'84d2a91e910bee523af5422439a639f677b9c78f'``
@@ -231,7 +231,7 @@ def get_all_comments(claim_id):
"method":"comment.List",
"params":{
"page":page,
"claim_id":claim_id,
"claim_id":video_id,
"page_size":10,
"top_level":False,
"sort_by":3}}
@@ -248,7 +248,7 @@ def get_all_comments(claim_id):
break
else:
_comments = result['result']['items']
comments = append_comment_reactions(comments = _comments)
comments = append_comment_reactions(comment_info_list = _comments)
all_comments.extend(comments)
page += 1
@@ -256,14 +256,14 @@ def get_all_comments(claim_id):
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def append_comment_reactions(comments):
def append_comment_reactions(comment_info_list):
"""Get reaction data for each comment and insert ``'reactions'`` key into
dict for each comment.
Parameters
----------
comments: list<dict>
comment_info_list: list<dict>
List of dictionaries, with each dict corresponding to a JSON response
containing data about a single comment for the specified video.
@@ -277,7 +277,7 @@ def append_comment_reactions(comments):
"""
comment_ids = ','.join([c['comment_id'] for c in comments])
comment_ids = ','.join([c['comment_id'] for c in comment_info_list])
json_data = {
"jsonrpc":"2.0",
@@ -296,23 +296,23 @@ def append_comment_reactions(comments):
reactions = result['result']['others_reactions']
for comment in comments:
for comment in comment_info_list:
comment['likes'] = reactions[comment['comment_id']]['like']
comment['dislikes'] = reactions[comment['comment_id']]['dislike']
return comments
return comment_info_list
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def get_recommended(title, claim_id):
def get_recommended(video_title, video_id):
name = quote(title)
name = quote(video_title)
params = {
's':name,
'size':'20',
'from':'0',
'related_to':claim_id}
'related_to':video_id}
response = make_request(
request = requests.get,
@@ -322,16 +322,16 @@ def get_recommended(title, claim_id):
result = json.loads(response.text)
recommended_video_info = [ name_to_video_info(r['name']) for r in result]
recommended_video_info = [ normalized_name_to_video_info(r['name']) for r in result]
recommended_video_info = [vi for vi in recommended_video_info if ((vi.get('value_type') == 'stream') & any(key in vi.get('value', []) for key in ('video', 'audio')))]
return recommended_video_info
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
def name_to_video_info(name):
def normalized_name_to_video_info(normalized_name):
video_url = f"lbry://{name}"
video_url = f"lbry://{normalized_name}"
json_data = {
"jsonrpc":"2.0",

12
polyphemus/base.py
View File

@@ -30,7 +30,7 @@ class OdyseeChannel:
self.info = info
self._channel_id = self.info['channel_id']
self.info['subscribers'] = api.get_subscribers(claim_id = self.info['channel_id'])
self.info['subscribers'] = api.get_subscribers(channel_id = self.info['channel_id'])
#-------------------------------------------------------------------------#
@@ -127,12 +127,12 @@ class OdyseeVideo:
'is_comment' : False,
'raw' : json.dumps(full_video_info)}
self._claim_id = self.info['claim_id']
self.claim_id = self.info['claim_id']
self.info['views'] = api.get_views(claim_id=self._claim_id)
self.info['views'] = api.get_views(video_id=self.claim_id)
self.info['likes'], self.info['dislikes']= api.get_video_reactions(
claim_id = self._claim_id)
video_id = self.claim_id)
self.info['streaming_url'] = api.get_streaming_url(self.info['canonical_url'])
@@ -140,7 +140,7 @@ class OdyseeVideo:
def get_all_comments(self):
all_comment_info = api.get_all_comments(claim_id=self._claim_id)
all_comment_info = api.get_all_comments(video_id=self.claim_id)
self.all_comments = (OdyseeComment(comment) for comment in all_comment_info)
return self.all_comments
@@ -150,7 +150,7 @@ class OdyseeVideo:
def get_recommended(self):
recommended_video_info = api.get_recommended(
title=self.info['title'], claim_id=self._claim_id)
video_title=self.info['title'], video_id=self.claim_id)
recommended_videos = [OdyseeVideo(video_info) for video_info in recommended_video_info]
return recommended_videos

13
pytest.ini Normal file
View File

@@ -0,0 +1,13 @@
[pytest]
minversion =
6.0.2
testpaths =
tests/
python_files =
*.py
addopts =
-vvv
--cov='polyphemus'
--cov-report html:reports/coverage
--html='reports/tests.html'
--self-contained-html

9
setup.py
View File

@@ -30,6 +30,15 @@ setup(
'requests >= 2.27.0',
'beautifulsoup4 >= 4.10.0',
'pandas >= 1.4.0'],
extras_require = {
'docs': [
'sphinx >= 3.3.1',
'sphinx_rtd_theme >= 0.5',],
'tests': [
'pytest >= 6.1.2',
'pytest-cov >= 2.10.1',
'pytest-html >= 3.0.0',
'pytest-metadata >= 1.10.0']},
include_package_data = True,
zip_safe = False )

44
tests/api.py Normal file
View File

@@ -0,0 +1,44 @@
# -*- coding: UTF-8 -*-
"""Tests for to polyphemus.api module.
The full set of tests for this module can be evaluated by executing the
command::
$ python -m pytest tests/api.py
from the project root directory.
"""
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
import pytest
from polyphemus import api
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
KWARGS_LIST = [
('get_channel_info', ['channel_name']),
('get_subscribers', ['channel_id']),
('get_all_videos', ['channel_id']),
('get_views', ['video_id']),
('get_video_reactions', ['video_id']),
('get_all_comments', ['video_id']),
('append_comment_reactions', ['comment_info_list']),
('normalized_name_to_video_info', ['normalized_name']),
('get_streaming_url', ['canonical_url']),
('get_recommended', ['video_title', 'video_id']),]
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
@pytest.mark.parametrize( 'function_str,kwargs', KWARGS_LIST )
def test_minimal_init( resources, function_str, kwargs ):
function = eval( f'api.{function_str}')
function_kwargs = { kwarg : resources[ kwarg ] for kwarg in kwargs }
function( **function_kwargs )
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

56
tests/base.py Normal file
View File

@@ -0,0 +1,56 @@
# -*- coding: UTF-8 -*-
"""Tests for to polyphemus.base module.
The full set of tests for this module can be evaluated by executing the
command::
$ python -m pytest tests/base.py
from the project root directory.
"""
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
import pytest
from polyphemus import base
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
class TestOdyseeChannel:
@pytest.fixture(autouse=True)
def test_simple_init(self, resources):
self.channel = base.OdyseeChannel(channel_name = resources['channel_name'])
def test_get_all_videos(self):
self.channel.get_all_videos()
def test_get_all_videos_and_comments(self):
self.channel.get_all_videos_and_comments()
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
class TestOdyseeVideo:
@pytest.fixture(autouse=True)
def test_simple_init(self, resources):
self.video = base.OdyseeVideo(full_video_info = resources['full_video_info'])
def test_get_all_comments(self):
self.video.get_all_comments()
def test_get_recommended(self):
self.video.get_recommended()
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
class TestOdyseeComment:
@pytest.fixture(autouse=True)
def test_simple_init(self, resources):
self.comment = base.OdyseeComment(full_comment_info = resources['full_comment_info'])
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

97
tests/conftest.py Normal file
View File

@@ -0,0 +1,97 @@
# -*- coding: UTF-8 -*-
"""Configuration for pytest sessions
"""
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
import pytest
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
CHANNEL_NAME = 'Mak1nBacon'
CHANNEL_ID = 'fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b'
VIDEO_ID = 'a754344cd7887a15ab4fddaa893ff08926c63bf3'
VIDEO_TITLE = 'chips'
NORMALIZED_NAME = 'want-me-eat-all-chips-meme'
CANONICAL_URL = 'lbry://@Mak1nBacon#f/want-me-eat-all-chips-meme#a'
FULL_VIDEO_INFO = {
'address': 'bPfL73FnWqHMd9idqgGh2xbJfYu85MMMRw',
'canonical_url': 'lbry://@Mak1nBacon#f/doggo-meme-cute-funny#5',
'claim_id': '53e51a9417a8445de3c11af3d45412df9693d015',
'name': 'doggo-meme-cute-funny',
'normalized_name': 'doggo-meme-cute-funny',
'permanent_url': 'lbry://doggo-meme-cute-funny#53e51a9417a8445de3c11af3d45412df9693d015',
'short_url': 'lbry://doggo-meme-cute-funny#5',
'signing_channel': {
'address': 'bPfL73FnWqHMd9idqgGh2xbJfYu85MMMRw',
'canonical_url': 'lbry://@Mak1nBacon#f',
'claim_id': 'fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b',
'name': '@Mak1nBacon',
'normalized_name': '@mak1nbacon',
'permanent_url': 'lbry://@Mak1nBacon#fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b',
'short_url': 'lbry://@Mak1nBacon#f',
'timestamp': 1642268511,
'type': 'claim',
'value': {
'cover': {
'url': 'https://thumbs.odycdn.com/6b6e3f5ed6b62e96e8013bbcfa486896.png'},
'description': "Hello ladies and men! In case you're wondering, yes, i'm still a piece of pork.\n\nBasically, i'm a random animator trying out Odysee. I make an object show called Meanwhile in the Void and random memes and animations too!\n\nIf you like this type of content, you're welcome to watch, but if you don't like my content, you're also welcome to watch! I don't mind lol.\n\nIf you're considering helping the channel, feel free to follow me!\n\nBacon included. ;)\n\nSee ya soon, stay calm, stick around and stay alive!",
'tags': ['comedy', 'animation', 'art', 'funny', 'object show'],
'thumbnail': {
'url': 'https://spee.ch/b/e4e3a6562e4b1cd5.png'},
'title': "Mak1n' Bacon"},
'value_type': 'channel'},
'timestamp': 1645981620,
'type': 'claim',
'value': {
'description': 'dog',
'languages': ['en'],
'license': 'None',
'release_time': '1645981256',
'stream_type': 'video',
'tags': ['art', 'comedy', 'meme', 'memes', 'animals'],
'thumbnail': {
'url': 'https://thumbs.odycdn.com/719ad60363211ef047b18a8f354c2943.jpeg'},
'title': 'doggo',
'video': {
'duration': 15,
'height': 640,
'width': 640}},
'value_type': 'stream'}
COMMENT_INFO_LIST = [{
'comment': 'the man on the right has some nice feet',
'comment_id': '320a0823689b9dbefad768598d89816bda0a015b11ad4b522bc0112a8089b3f5',
'claim_id': 'a754344cd7887a15ab4fddaa893ff08926c63bf3',
'timestamp': 1644193831,
'signature': '444835698b1bfe160c775210b9542970b14c8dcb7b88118a367c2fe102bb2ddcc3fa3881827a789cb183f2e3fd5c8f263ec05d7c431cfe8e145d7f3f501c0668',
'signing_ts': '1644193830',
'channel_id': 'a641423e6e20718f3d59138a17cf530bb419d86b',
'channel_name': '@devnull',
'channel_url': 'lbry://@devnull#a641423e6e20718f3d59138a17cf530bb419d86b',
'replies': 1,}]
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
@pytest.fixture(scope = 'module')
def resources():
"""SetUp fixture to create constant valued resources for testing modules
"""
resources_dict = dict(
channel_name = CHANNEL_NAME,
channel_id = CHANNEL_ID,
video_id = VIDEO_ID,
video_title = VIDEO_TITLE,
normalized_name = NORMALIZED_NAME,
canonical_url = CANONICAL_URL,
full_video_info = FULL_VIDEO_INFO,
full_comment_info = {**COMMENT_INFO_LIST[0], **{'likes' : 8, 'dislikes' : 0}},
comment_info_list = COMMENT_INFO_LIST)
return resources_dict
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#