From 4d18522a5496fbab5c79a3f5933f75a6f5435c40 Mon Sep 17 00:00:00 2001 From: X Date: Thu, 5 May 2022 17:07:33 +0200 Subject: [PATCH] incorporate Tristan's suggestions and further fix logging --- logging.config | 36 +++++++++++++++++++++++++++++ tiktok_downloader/data_methods.py | 14 +++++------ tiktok_downloader/file_methods.py | 16 +++++-------- tiktok_downloader/run_downloader.py | 7 ++++-- 4 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 logging.config diff --git a/logging.config b/logging.config new file mode 100644 index 0000000..b0f5c59 --- /dev/null +++ b/logging.config @@ -0,0 +1,36 @@ +[loggers] +keys=root,Logger + +[handlers] +keys=consoleHandler,fileHandler + +[formatters] +keys=consoleFormatter,fileFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_Logger] +level=DEBUG +handlers=consoleHandler,fileHandler +qualname=Logger +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=consoleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=FileHandler +level=WARNING +formatter=fileFormatter +args=("../logfile.log",) + +[formatter_consoleFormatter] +format=%(message)s + +[formatter_fileFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/tiktok_downloader/data_methods.py b/tiktok_downloader/data_methods.py index b638ff0..2211814 100644 --- a/tiktok_downloader/data_methods.py +++ b/tiktok_downloader/data_methods.py @@ -1,10 +1,10 @@ from collections import namedtuple -import warnings -import logging +import logging, logging.config import file_methods -logger = logging.getLogger() +logging.config.fileConfig("../logging.config") +logger = logging.getLogger("Logger") """ @@ -58,7 +58,7 @@ def extract_posts(settings, file_name, tag): ids.append(post["id"]) if not ids: - warnings.warn(f"No posts were found for {tag} in the file - {file_name}") + logger.warn(f"No posts were found for the hashtag: {tag}") status = file_methods.check_existence(settings["post_ids"], "file") if not status: @@ -67,7 +67,7 @@ def extract_posts(settings, file_name, tag): else: new_ids = get_difference(tag, settings["post_ids"], ids) if not new_ids: - warnings.warn(f"No new posts were found in the downloaded file - {file_name}") + logger.warn(f"No new posts were found for the hashtag: {tag}") elif new_ids.filter_posts: new_posts = [post for post in posts if post['id'] in new_ids.ids] new_data = (new_ids.ids, new_posts) @@ -88,7 +88,7 @@ def extract_videos(settings, tag, download_list): else: new_videos = get_difference(tag, settings["video_ids"], download_list) if not new_videos: - warnings.warn(f"No new videos were found for the {tag} in the downloaded folder.") + logger.warn(f"No new videos were found for the {tag} in the downloaded folder.") return None else: return new_videos.ids @@ -140,4 +140,4 @@ def print_total(file_path, tag, data_type): if (total.total == total.unique): logger.info(f"Scraped {total.total} {data_type} containing the hashtag '{tag}'") else: - warnings.warn(f"Out of total {data_type} for the hashtag {tag} {total.total}, only {total.unique} are unique. Something is going wrong...") + logger.warn(f"Out of total {data_type} for the hashtag {tag} {total.total}, only {total.unique} are unique. Something is going wrong...") diff --git a/tiktok_downloader/file_methods.py b/tiktok_downloader/file_methods.py index b041130..0b28314 100644 --- a/tiktok_downloader/file_methods.py +++ b/tiktok_downloader/file_methods.py @@ -3,14 +3,11 @@ import json import subprocess from datetime import datetime import shutil -import warnings -import logging +import logging, logging.config -logging.basicConfig( - level = logging.INFO, - format = '%(message)s') -logger = logging.getLogger() +logging.config.fileConfig("../logging.config") +logger = logging.getLogger("Logger") """ The file contains the functions that operate on files, such as writing or reading from files etc. @@ -65,11 +62,10 @@ def download_posts(settings, tag): os.chdir("../../../tiktok_downloader") return new_file else: - warnings.warn(f"Something's wrong with what is returned by tiktok-scraper for the hashtag {tag} - *{new_file}* is not a json file.\n\ntiktok-scraper returned {output}") + logger.warn(f"Something's wrong with what is returned by tiktok-scraper for the hashtag {tag} - *{new_file}* is not a json file.\n\ntiktok-scraper returned {output}") os.chdir("../../../tiktok_downloader") - def download_videos(settings, tag): """ Runs the tiktok-scraper command to download videos for a given hashtag. Note that all the videos are downloaded that are returned by the tiktok api and as a result, its a time and data consuming process. @@ -90,7 +86,7 @@ def download_videos(settings, tag): os.chdir("../../../tiktok_downloader") return downloaded_list else: - warnings.warn(f"No video files were downloaded for the hashtag {tag}.") + logger.warn(f"No video files were downloaded for the hashtag {tag}.") os.chdir("../../../tiktok_downloader") shutil.rmtree(settings['videos_delete']) @@ -134,7 +130,7 @@ def log_writer(log_data): now_str = now.strftime("%d-%m-%Y %H:%M:%S") data = { now_str : scraped_summary_dict } - logger.debug(f"Logged post data: {data}") + logger.warn(f"Logged post data: {data}") logger.info(f"Successfully scraped {total} total entries") diff --git a/tiktok_downloader/run_downloader.py b/tiktok_downloader/run_downloader.py index 529b336..5b59afd 100644 --- a/tiktok_downloader/run_downloader.py +++ b/tiktok_downloader/run_downloader.py @@ -1,13 +1,16 @@ import os import time import argparse -import logging +import logging, logging.config import global_data import file_methods import data_methods -logger = logging.getLogger() + +logging.config.fileConfig("../logging.config") +logger = logging.getLogger("Logger") + def get_hashtag_list(file_name): if not file_methods.check_existence(file_name, 'file'):