bellingcat/tiktok-hashtag-analysis
1
0
Fork 0
mirror of https://github.com/bellingcat/tiktok-hashtag-analysis.git synced 2026-06-08 03:18:31 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update file_methods.py

Browse Source
This commit is contained in:
johannawild
2022-05-04 00:22:58 +02:00
committed by GitHub
parent 7bb3dbaa0b
commit 8e6630f362
1 changed files with 32 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
tiktok_downloader/file_methods.py
View File

@@ -4,6 +4,15 @@ import global_data
import shutil
# setting up the logging
import logging
from logging.config import fileConfig
fileConfig('../logging.ini')
logger = logging.getLogger()
"""
The file contains the functions that operate on files, such as writing or reading from files etc.
"""
@@ -18,7 +27,7 @@ def create_file(name, file_type):
elif (file_type == "file"):
with open(name, "w"): pass
else:
raise OSError(f"{file_type} has to be a 'dir' or a 'file'!!!")
logger.exception(f"{file_type} has to be a 'dir' or a 'file'!!!")
return
@@ -31,7 +40,7 @@ def check_existence(file_path, file_type):
elif (file_type == "dir"):
return os.path.isdir(file_path)
else:
raise OSError(f"{file_type} has to be a 'dir' or a 'file'!!!")
logger.exception(f"{file_type} has to be a 'dir' or a 'file'!!!")
def check_file(file_path, file_type):
@@ -62,12 +71,12 @@ def download_posts(settings, tag):
os.chdir("../../../tiktok_downloader")
return new_file
else:
print(f"WARNING: Something's wrong with what is returned by tiktok-scraper for the hashtag {tag} - *{new_file}* is not a json file!!!!")
logger.warn(f"WARNING: Something's wrong with what is returned by tiktok-scraper for the hashtag {tag} - *{new_file}* is not a json file!!!!")
os.chdir("../../../tiktok_downloader")
return
else:
os.chdir("../../../tiktok_downloader")
print(f"WARNING: No file was downloaded by the tiktok-scraper for the {tag} !!!!")
logger.warn(f"WARNING: No file was downloaded by the tiktok-scraper for the {tag} !!!!")
return
except: raise
@@ -96,13 +105,13 @@ def download_videos(settings, tag):
os.chdir("../../../tiktok_downloader")
return downloaded_list
else:
print(f"WARNING: No video files were downloaded for the hashtag {tag}.")
logger.warn(f"WARNING: No video files were downloaded for the hashtag {tag}.")
os.chdir("../../../tiktok_downloader")
shutil.rmtree(settings['videos_delete'])
#subprocess.call(f"rm -rf {settings['videos_delete']}", shell=True)
else:
os.chdir("../../../tiktok_downloader")
print(f"WARNING: Something went wrong with the tiktok-scraper video download for the {tag} !!!!")
logger.warn(f"WARNING: Something went wrong with the tiktok-scraper video download for the {tag} !!!!")
return
except: raise
@@ -145,20 +154,15 @@ def log_writer(log_data):
log_dict[ele[0]] = { ele[1][0] : ele[1][1] }
total += ele[1][1]
logger = global_data.FILES["logger"]
now = datetime.now()
now_str = now.strftime("%d-%m-%Y %H:%M:%S")
status = check_existence(logger, "file")
if status:
data = get_data(logger)
data[now_str] = log_dict
dump_data(logger, data)
else:
data = { now_str : log_dict }
dump_data(logger, data)
print(f"Successfully logged {total} entries!!!!")
data = { now_str : log_dict }
logger.warn(data)
logger.info(f"Successfully logged {total} entries!!!!")
return
except: raise
except:
logger.exception()
def id_writer(file_path, new_data, tag, status):
@@ -181,10 +185,11 @@ def id_writer(file_path, new_data, tag, status):
else:
data = { tag : new_data }
dump_data(file_path, data)
print(f"SUCCESS - {total} entries added to {file_path}!!!")
logger.info(f"SUCCESS - {total} entries added to {file_path}!!!")
log_data = (tag, total)
return log_data
except: raise
except:
logger.exception()
def post_writer(file_path, new_data, status):
@@ -204,9 +209,10 @@ def post_writer(file_path, new_data, status):
else:
data = new_data
dump_data(file_path, data)
print(f"SUCCESS - {total} entries added to {file_path}!!!")
logger.info(f"SUCCESS - {total} entries added to {file_path}!!!")
return
except: raise
except:
logger.exception()
def delete_file(file_path, file_type):
@@ -214,17 +220,17 @@ def delete_file(file_path, file_type):
Deletes the directory or the file.
"""
if not check_existence(file_path, file_type):
raise Exception(f"ERROR: Attempt to delete failed. {file_path} does not exist!!!")
logger.exception(f"ERROR: Attempt to delete failed. {file_path} does not exist!!!")
elif (file_type == "file"):
os.remove(file_path)
print(f"Successfully deleted {file_path}!!!")
logger.info(f"Successfully deleted {file_path}!!!")
return
elif (file_type == "dir"):
os.rmdir(file_path)
print(f"Successfully deleted {file_path}!!!")
logger.info(f"Successfully deleted {file_path}!!!")
return
else:
raise OSError(f"ERROR: {file_type} needs to be either 'file' or 'dir' !!!")
logger.exception(f"OSError: {file_type} needs to be either 'file' or 'dir' !!!")
def clean_video_files(settings, tag, new_data=None):
@@ -237,10 +243,8 @@ def clean_video_files(settings, tag, new_data=None):
for file in new_data:
settings["videos_from"] = settings['data'] + f"/{tag}/videos/#{tag}/{file}.mp4"
shutil.move(settings['videos_from'], settings['videos_to'])
#subprocess.call(f"mv {settings['videos_from']} {settings['videos_to']}", shell=True)
shutil.rmtree(settings['videos_delete'])
#subprocess.call(f"rm -rf {settings['videos_delete']}", shell=True)
print(f"Successfully deleted the folder {settings['videos_delete']} folder of videos.")
logger.info(f"Successfully deleted the folder {settings['videos_delete']} folder of videos.")
except:
raise