bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 11:58:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Correctly serialise nested NamedTuples

Browse Source
This commit is contained in:
JustAnotherArchivist
2020-09-01 03:16:25 +00:00
parent ba250aabf2
commit d9572ec450
1 changed files with 16 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
snscrape/cli.py
View File

@@ -229,6 +229,20 @@ def json_serialise_datetime(obj):
raise TypeError(f'Object of type {type(obj)} is not JSON serializable')
def namedtuple_to_dict_recursive(obj):
# Convert a NamedTuple to a dict; also converts NamedTuples in its values to dicts
if (isinstance(obj, tuple) and hasattr(obj, '_asdict')) or isinstance(obj, dict):
if isinstance(obj, tuple):
obj = obj._asdict()
for key, value in obj.items():
obj[key] = namedtuple_to_dict_recursive(value)
return obj
elif isinstance(obj, (tuple, list)):
return type(obj)(namedtuple_to_dict_recursive(value) for value in obj)
else:
return obj
def main():
setup_logging()
args = parse_args()
@@ -241,7 +255,7 @@ def main():
entity = scraper.entity
if entity:
if args.jsonl:
print(json.dumps(entity._asdict(), default = json_serialise_datetime))
print(json.dumps(namedtuple_to_dict_recursive(entity), default = json_serialise_datetime))
else:
print(entity)
for i, item in enumerate(scraper.get_items(), start = 1):
@@ -249,7 +263,7 @@ def main():
logger.info(f'Exiting due to reaching older results than {args.since}')
break
if args.jsonl:
print(json.dumps(item._asdict(), default = json_serialise_datetime))
print(json.dumps(namedtuple_to_dict_recursive(item), default = json_serialise_datetime))
elif args.format is not None:
print(args.format.format(**item._asdict()))
else: