bellingcat/polyphemus
1
0
Fork 0
mirror of https://github.com/bellingcat/polyphemus.git synced 2026-06-11 21:08:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

updated examples with refactored scraper, increased speed of recommendation engine fetchibng by implementing normalized_names_to_video_info routine, that allows requesting multiple videos at a time

Browse Source
This commit is contained in:
Tristan Lee
2022-04-11 23:28:44 -05:00
parent 44a673f889
commit 0aac7493a4
6 changed files with 75 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
examples/generate_network.py
View File

@@ -20,21 +20,23 @@ OUTPUT_DIR = '../../data'
if __name__ == '__main__':
odysee_channel = polyphemus.base.OdyseeChannel(channel_name = CHANNEL_NAME)
auth_token = polyphemus.api.get_auth_token()
scraper = polyphemus.base.OdyseeChannelScraper(channel_name = CHANNEL_NAME, auth_token = auth_token)
edge_list = list()
already_done = list()
new_videos = odysee_channel.get_all_videos()
master_video_dict = dict(zip([v.info['claim_id'] for v in new_videos], new_videos))
new_videos = list(scraper.get_all_videos())
master_video_dict = dict(zip([v.claim_id for v in new_videos], new_videos))
for iteration in range(ITERATIONS):
print(f'\n\nITERATION: {iteration}, N_VIDEOS: {len(new_videos)}\n\n')
for i, video in enumerate(new_videos):
claim_id = video.info['claim_id']
title = video.info['title']
claim_id = video.claim_id
title = video.title
print(f'\nVIDEO: {i}; CLAIM_ID: {claim_id}\n')
@@ -47,20 +49,23 @@ if __name__ == '__main__':
edge_list.append((claim_id, rec_claim_id))
if rec_video_info['claim_id'] not in master_video_dict:
master_video_dict[rec_claim_id] = polyphemus.base.OdyseeVideo(rec_video_info)
master_video_dict[rec_claim_id] = polyphemus.base.process_raw_video_info(
raw_video_info = rec_video_info,
auth_token = auth_token,
additional_fields = False)
already_done.append(claim_id)
new_videos = [video for video in master_video_dict.values() if video.info['claim_id'] not in already_done]
new_videos = [video for video in master_video_dict.values() if video.claim_id not in already_done]
#-------------------------------------------------------------------------#
os.makedirs(OUTPUT_DIR, exist_ok = True)
with open(Path(OUTPUT_DIR, 'master_video_dict.pkl'), 'wb') as f:
with open(Path(OUTPUT_DIR, f'master_video_dict_iterations={ITERATIONS}.pkl'), 'wb') as f:
pickle.dump(master_video_dict, f)
with open(Path(OUTPUT_DIR, 'edge_list.pkl'), 'wb') as f:
pickle.dump(edge_list)
with open(Path(OUTPUT_DIR, f'edge_list_iterations={ITERATIONS}.pkl'), 'wb') as f:
pickle.dump(edge_list, f)
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#

10
examples/scrape.py
View File

@@ -11,7 +11,7 @@ import os
import pandas as pd
from polyphemus.base import OdyseeChannel
from polyphemus.base import OdyseeChannelScraper
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++#
@@ -22,13 +22,13 @@ OUTPUT_DIR = Path('.').resolve().parents[1]/'data'
if __name__ == '__main__':
odysee_channel = OdyseeChannel(channel_name = CHANNEL_NAME)
odysee_channel = OdyseeChannelScraper(channel_name = CHANNEL_NAME)
video_list, comment_list = odysee_channel.get_all_videos_and_comments()
channel_df = pd.DataFrame([odysee_channel.info])
video_df = pd.DataFrame([v.info for v in video_list])
comment_df = pd.DataFrame([c.info for c in comment_list])
channel_df = pd.DataFrame([odysee_channel.get_entity().__dict__])
video_df = pd.DataFrame([v.__dict__ for v in video_list])
comment_df = pd.DataFrame([c.__dict__ for c in comment_list])
output_subdir = Path(OUTPUT_DIR, CHANNEL_NAME)
os.makedirs(output_subdir, exist_ok = True)