diff --git a/.gitignore b/.gitignore index fd633f6..3e410c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,83 @@ -*.csv -**/data/ +# Ignore all non-Python files in examples directory +examples/* +!examples/*.py -**/__pycache__/ -*.pyc +# Sphinx documentation +docs/_build/ +docs/source/ +# Test and coverage reports +reports + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Distribution / packaging +.Python build/ -*.egg-info/ \ No newline at end of file +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# pytype static type analyzer +.pytype/ \ No newline at end of file diff --git a/polyphemus/api.py b/polyphemus/api.py index 9ea859a..b4b1d1e 100644 --- a/polyphemus/api.py +++ b/polyphemus/api.py @@ -87,14 +87,14 @@ def get_channel_info(channel_name): #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def get_subscribers(claim_id): +def get_subscribers(channel_id): """Get the number of subscribers for a channel. """ json_data = { 'auth_token': AUTH_TOKEN, - 'claim_id': claim_id } + 'claim_id': channel_id } response = make_request( request = requests.post, @@ -156,14 +156,14 @@ def get_all_videos(channel_id): #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def get_views(claim_id): +def get_views(video_id): """Get the number of views for a given video. """ params = { 'auth_token': AUTH_TOKEN, - 'claim_id': claim_id } + 'claim_id': video_id } response = make_request( request = requests.get, @@ -177,14 +177,14 @@ def get_views(claim_id): #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def get_video_reactions(claim_id): +def get_video_reactions(video_id): """Get all reactions for a given video. """ post_data = { 'auth_token': AUTH_TOKEN, - 'claim_ids': claim_id } + 'claim_ids': video_id } response = make_request( request = requests.post, @@ -195,20 +195,20 @@ def get_video_reactions(claim_id): result = json.loads(response.text) if result['success']: - reactions = result['data']['others_reactions'][claim_id ] + reactions = result['data']['others_reactions'][video_id] return reactions['like'], reactions['dislike'] else: return None, None #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def get_all_comments(claim_id): +def get_all_comments(video_id): """Get a list of all comments for a single video. Parameters ---------- - claim_id: str + video_id: str Claim ID for the video whose comments are to be scraped e.g. ``'84d2a91e910bee523af5422439a639f677b9c78f'`` @@ -231,7 +231,7 @@ def get_all_comments(claim_id): "method":"comment.List", "params":{ "page":page, - "claim_id":claim_id, + "claim_id":video_id, "page_size":10, "top_level":False, "sort_by":3}} @@ -248,7 +248,7 @@ def get_all_comments(claim_id): break else: _comments = result['result']['items'] - comments = append_comment_reactions(comments = _comments) + comments = append_comment_reactions(comment_info_list = _comments) all_comments.extend(comments) page += 1 @@ -256,14 +256,14 @@ def get_all_comments(claim_id): #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def append_comment_reactions(comments): +def append_comment_reactions(comment_info_list): """Get reaction data for each comment and insert ``'reactions'`` key into dict for each comment. Parameters ---------- - comments: list + comment_info_list: list List of dictionaries, with each dict corresponding to a JSON response containing data about a single comment for the specified video. @@ -277,7 +277,7 @@ def append_comment_reactions(comments): """ - comment_ids = ','.join([c['comment_id'] for c in comments]) + comment_ids = ','.join([c['comment_id'] for c in comment_info_list]) json_data = { "jsonrpc":"2.0", @@ -296,23 +296,23 @@ def append_comment_reactions(comments): reactions = result['result']['others_reactions'] - for comment in comments: + for comment in comment_info_list: comment['likes'] = reactions[comment['comment_id']]['like'] comment['dislikes'] = reactions[comment['comment_id']]['dislike'] - return comments + return comment_info_list #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def get_recommended(title, claim_id): +def get_recommended(video_title, video_id): - name = quote(title) + name = quote(video_title) params = { 's':name, 'size':'20', 'from':'0', - 'related_to':claim_id} + 'related_to':video_id} response = make_request( request = requests.get, @@ -322,16 +322,16 @@ def get_recommended(title, claim_id): result = json.loads(response.text) - recommended_video_info = [ name_to_video_info(r['name']) for r in result] + recommended_video_info = [ normalized_name_to_video_info(r['name']) for r in result] recommended_video_info = [vi for vi in recommended_video_info if ((vi.get('value_type') == 'stream') & any(key in vi.get('value', []) for key in ('video', 'audio')))] return recommended_video_info #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# -def name_to_video_info(name): +def normalized_name_to_video_info(normalized_name): - video_url = f"lbry://{name}" + video_url = f"lbry://{normalized_name}" json_data = { "jsonrpc":"2.0", diff --git a/polyphemus/base.py b/polyphemus/base.py index 39633e4..1166c0c 100644 --- a/polyphemus/base.py +++ b/polyphemus/base.py @@ -30,7 +30,7 @@ class OdyseeChannel: self.info = info self._channel_id = self.info['channel_id'] - self.info['subscribers'] = api.get_subscribers(claim_id = self.info['channel_id']) + self.info['subscribers'] = api.get_subscribers(channel_id = self.info['channel_id']) #-------------------------------------------------------------------------# @@ -127,12 +127,12 @@ class OdyseeVideo: 'is_comment' : False, 'raw' : json.dumps(full_video_info)} - self._claim_id = self.info['claim_id'] + self.claim_id = self.info['claim_id'] - self.info['views'] = api.get_views(claim_id=self._claim_id) + self.info['views'] = api.get_views(video_id=self.claim_id) self.info['likes'], self.info['dislikes']= api.get_video_reactions( - claim_id = self._claim_id) + video_id = self.claim_id) self.info['streaming_url'] = api.get_streaming_url(self.info['canonical_url']) @@ -140,7 +140,7 @@ class OdyseeVideo: def get_all_comments(self): - all_comment_info = api.get_all_comments(claim_id=self._claim_id) + all_comment_info = api.get_all_comments(video_id=self.claim_id) self.all_comments = (OdyseeComment(comment) for comment in all_comment_info) return self.all_comments @@ -150,7 +150,7 @@ class OdyseeVideo: def get_recommended(self): recommended_video_info = api.get_recommended( - title=self.info['title'], claim_id=self._claim_id) + video_title=self.info['title'], video_id=self.claim_id) recommended_videos = [OdyseeVideo(video_info) for video_info in recommended_video_info] return recommended_videos diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..54548c8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,13 @@ +[pytest] +minversion = + 6.0.2 +testpaths = + tests/ +python_files = + *.py +addopts = + -vvv + --cov='polyphemus' + --cov-report html:reports/coverage + --html='reports/tests.html' + --self-contained-html \ No newline at end of file diff --git a/setup.py b/setup.py index 69637da..582ae88 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,15 @@ setup( 'requests >= 2.27.0', 'beautifulsoup4 >= 4.10.0', 'pandas >= 1.4.0'], + extras_require = { + 'docs': [ + 'sphinx >= 3.3.1', + 'sphinx_rtd_theme >= 0.5',], + 'tests': [ + 'pytest >= 6.1.2', + 'pytest-cov >= 2.10.1', + 'pytest-html >= 3.0.0', + 'pytest-metadata >= 1.10.0']}, include_package_data = True, zip_safe = False ) diff --git a/tests/api.py b/tests/api.py new file mode 100644 index 0000000..ecd3da8 --- /dev/null +++ b/tests/api.py @@ -0,0 +1,44 @@ +# -*- coding: UTF-8 -*- + +"""Tests for to polyphemus.api module. + +The full set of tests for this module can be evaluated by executing the +command:: + + $ python -m pytest tests/api.py + +from the project root directory. + +""" + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +import pytest + +from polyphemus import api + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +KWARGS_LIST = [ + ('get_channel_info', ['channel_name']), + ('get_subscribers', ['channel_id']), + ('get_all_videos', ['channel_id']), + ('get_views', ['video_id']), + ('get_video_reactions', ['video_id']), + ('get_all_comments', ['video_id']), + ('append_comment_reactions', ['comment_info_list']), + ('normalized_name_to_video_info', ['normalized_name']), + ('get_streaming_url', ['canonical_url']), + ('get_recommended', ['video_title', 'video_id']),] + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +@pytest.mark.parametrize( 'function_str,kwargs', KWARGS_LIST ) +def test_minimal_init( resources, function_str, kwargs ): + + function = eval( f'api.{function_str}') + function_kwargs = { kwarg : resources[ kwarg ] for kwarg in kwargs } + + function( **function_kwargs ) + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# \ No newline at end of file diff --git a/tests/base.py b/tests/base.py new file mode 100644 index 0000000..6da7031 --- /dev/null +++ b/tests/base.py @@ -0,0 +1,56 @@ +# -*- coding: UTF-8 -*- + +"""Tests for to polyphemus.base module. + +The full set of tests for this module can be evaluated by executing the +command:: + + $ python -m pytest tests/base.py + +from the project root directory. + +""" + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +import pytest + +from polyphemus import base + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +class TestOdyseeChannel: + + @pytest.fixture(autouse=True) + def test_simple_init(self, resources): + self.channel = base.OdyseeChannel(channel_name = resources['channel_name']) + + def test_get_all_videos(self): + self.channel.get_all_videos() + + def test_get_all_videos_and_comments(self): + self.channel.get_all_videos_and_comments() + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +class TestOdyseeVideo: + + @pytest.fixture(autouse=True) + def test_simple_init(self, resources): + self.video = base.OdyseeVideo(full_video_info = resources['full_video_info']) + + def test_get_all_comments(self): + self.video.get_all_comments() + + def test_get_recommended(self): + self.video.get_recommended() + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +class TestOdyseeComment: + + @pytest.fixture(autouse=True) + def test_simple_init(self, resources): + self.comment = base.OdyseeComment(full_comment_info = resources['full_comment_info']) + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..91b8018 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,97 @@ +# -*- coding: UTF-8 -*- + +"""Configuration for pytest sessions +""" + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +import pytest + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +CHANNEL_NAME = 'Mak1nBacon' +CHANNEL_ID = 'fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b' +VIDEO_ID = 'a754344cd7887a15ab4fddaa893ff08926c63bf3' +VIDEO_TITLE = 'chips' +NORMALIZED_NAME = 'want-me-eat-all-chips-meme' +CANONICAL_URL = 'lbry://@Mak1nBacon#f/want-me-eat-all-chips-meme#a' + +FULL_VIDEO_INFO = { + 'address': 'bPfL73FnWqHMd9idqgGh2xbJfYu85MMMRw', + 'canonical_url': 'lbry://@Mak1nBacon#f/doggo-meme-cute-funny#5', + 'claim_id': '53e51a9417a8445de3c11af3d45412df9693d015', + 'name': 'doggo-meme-cute-funny', + 'normalized_name': 'doggo-meme-cute-funny', + 'permanent_url': 'lbry://doggo-meme-cute-funny#53e51a9417a8445de3c11af3d45412df9693d015', + 'short_url': 'lbry://doggo-meme-cute-funny#5', + 'signing_channel': { + 'address': 'bPfL73FnWqHMd9idqgGh2xbJfYu85MMMRw', + 'canonical_url': 'lbry://@Mak1nBacon#f', + 'claim_id': 'fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b', + 'name': '@Mak1nBacon', + 'normalized_name': '@mak1nbacon', + 'permanent_url': 'lbry://@Mak1nBacon#fb2a33dc4252feb2e99c6d6949fbd3cc557cab2b', + 'short_url': 'lbry://@Mak1nBacon#f', + 'timestamp': 1642268511, + 'type': 'claim', + 'value': { + 'cover': { + 'url': 'https://thumbs.odycdn.com/6b6e3f5ed6b62e96e8013bbcfa486896.png'}, + 'description': "Hello ladies and men! In case you're wondering, yes, i'm still a piece of pork.\n\nBasically, i'm a random animator trying out Odysee. I make an object show called Meanwhile in the Void and random memes and animations too!\n\nIf you like this type of content, you're welcome to watch, but if you don't like my content, you're also welcome to watch! I don't mind lol.\n\nIf you're considering helping the channel, feel free to follow me!\n\nBacon included. ;)\n\nSee ya soon, stay calm, stick around and stay alive!", + 'tags': ['comedy', 'animation', 'art', 'funny', 'object show'], + 'thumbnail': { + 'url': 'https://spee.ch/b/e4e3a6562e4b1cd5.png'}, + 'title': "Mak1n' Bacon"}, + 'value_type': 'channel'}, + 'timestamp': 1645981620, + 'type': 'claim', + 'value': { + 'description': 'dog', + 'languages': ['en'], + 'license': 'None', + 'release_time': '1645981256', + 'stream_type': 'video', + 'tags': ['art', 'comedy', 'meme', 'memes', 'animals'], + 'thumbnail': { + 'url': 'https://thumbs.odycdn.com/719ad60363211ef047b18a8f354c2943.jpeg'}, + 'title': 'doggo', + 'video': { + 'duration': 15, + 'height': 640, + 'width': 640}}, + 'value_type': 'stream'} + +COMMENT_INFO_LIST = [{ + 'comment': 'the man on the right has some nice feet', + 'comment_id': '320a0823689b9dbefad768598d89816bda0a015b11ad4b522bc0112a8089b3f5', + 'claim_id': 'a754344cd7887a15ab4fddaa893ff08926c63bf3', + 'timestamp': 1644193831, + 'signature': '444835698b1bfe160c775210b9542970b14c8dcb7b88118a367c2fe102bb2ddcc3fa3881827a789cb183f2e3fd5c8f263ec05d7c431cfe8e145d7f3f501c0668', + 'signing_ts': '1644193830', + 'channel_id': 'a641423e6e20718f3d59138a17cf530bb419d86b', + 'channel_name': '@devnull', + 'channel_url': 'lbry://@devnull#a641423e6e20718f3d59138a17cf530bb419d86b', + 'replies': 1,}] + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# + +@pytest.fixture(scope = 'module') +def resources(): + + """SetUp fixture to create constant valued resources for testing modules + """ + + resources_dict = dict( + channel_name = CHANNEL_NAME, + channel_id = CHANNEL_ID, + video_id = VIDEO_ID, + video_title = VIDEO_TITLE, + normalized_name = NORMALIZED_NAME, + canonical_url = CANONICAL_URL, + full_video_info = FULL_VIDEO_INFO, + full_comment_info = {**COMMENT_INFO_LIST[0], **{'likes' : 8, 'dislikes' : 0}}, + comment_info_list = COMMENT_INFO_LIST) + + return resources_dict + +#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++# \ No newline at end of file