diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ab376a3..67a3cd9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ - - - -Fixes # - -Changes proposed in this pull request: - -- - -## Before submitting - - -- [ ] I've read and followed all steps in the [Making a pull request](https://github.com/bellingcat/vk-url-scraper/blob/main/CONTRIBUTING.md#making-a-pull-request) - section of the `CONTRIBUTING` docs. -- [ ] I've updated or added any relevant docstrings following the syntax described in the - [Writing docstrings](https://github.com/bellingcat/vk-url-scraper/blob/main/CONTRIBUTING.md#writing-docstrings) section of the `CONTRIBUTING` docs. -- [ ] If this PR fixes a bug, I've added a test that will fail without my fix. -- [ ] If this PR adds a new feature, I've added tests that sufficiently cover my new functionality. + + + +Fixes # + +Changes proposed in this pull request: + +- + +## Before submitting + + +- [ ] I've read and followed all steps in the [Making a pull request](https://github.com/bellingcat/vk-url-scraper/blob/main/CONTRIBUTING.md#making-a-pull-request) + section of the `CONTRIBUTING` docs. +- [ ] I've updated or added any relevant docstrings following the syntax described in the + [Writing docstrings](https://github.com/bellingcat/vk-url-scraper/blob/main/CONTRIBUTING.md#writing-docstrings) section of the `CONTRIBUTING` docs. +- [ ] If this PR fixes a bug, I've added a test that will fail without my fix. +- [ ] If this PR adds a new feature, I've added tests that sufficiently cover my new functionality. diff --git a/vk_url_scraper/__main__.py b/vk_url_scraper/__main__.py index c0c957d..ce47a24 100644 --- a/vk_url_scraper/__main__.py +++ b/vk_url_scraper/__main__.py @@ -1,63 +1,63 @@ -import argparse -import json - -from .scraper import VkScraper -from .utils import DateTimeEncoder - - -def get_argument_parser(): - """ - Creates the CMD line arguments. 'python vk_url_scraper.py --help' - """ - parser = argparse.ArgumentParser( - description="Authenticate and scrape information from vk.com based on a URL or set of URLs." - ) - - parser.add_argument( - "-u", - "--username", - action="store", - dest="username", - required=True, - help="username for a valid vk.com account", - ) - parser.add_argument( - "-p", - "--password", - action="store", - dest="password", - required=True, - help="password for the valid vk.com account", - ) - parser.add_argument( - "-d", - "--download", - action=argparse.BooleanOptionalAction, - dest="download", - help="if set then all photos and videos will be downloaded to folder output/", - ) - parser.add_argument( - "--urls", - action="store", - dest="urls", - nargs=argparse.REMAINDER, - required=True, - help="must be the last argument: any text with one or more urls to scrape", - ) - return parser - - -def main(): - parser = get_argument_parser() - args = parser.parse_args() - vks = VkScraper(args.username, args.password) - text = " ".join(args.urls) - res = vks.scrape(text) - res_json = json.dumps(res, ensure_ascii=False, indent=4, cls=DateTimeEncoder) - print(res_json) - if args.download: - vks.download_media(res) - - -if __name__ == "__main__": - main() +import argparse +import json + +from .scraper import VkScraper +from .utils import DateTimeEncoder + + +def get_argument_parser(): + """ + Creates the CMD line arguments. 'python vk_url_scraper.py --help' + """ + parser = argparse.ArgumentParser( + description="Authenticate and scrape information from vk.com based on a URL or set of URLs." + ) + + parser.add_argument( + "-u", + "--username", + action="store", + dest="username", + required=True, + help="username for a valid vk.com account", + ) + parser.add_argument( + "-p", + "--password", + action="store", + dest="password", + required=True, + help="password for the valid vk.com account", + ) + parser.add_argument( + "-d", + "--download", + action=argparse.BooleanOptionalAction, + dest="download", + help="if set then all photos and videos will be downloaded to folder output/", + ) + parser.add_argument( + "--urls", + action="store", + dest="urls", + nargs=argparse.REMAINDER, + required=True, + help="must be the last argument: any text with one or more urls to scrape", + ) + return parser + + +def main(): + parser = get_argument_parser() + args = parser.parse_args() + vks = VkScraper(args.username, args.password) + text = " ".join(args.urls) + res = vks.scrape(text) + res_json = json.dumps(res, ensure_ascii=False, indent=4, cls=DateTimeEncoder) + print(res_json) + if args.download: + vks.download_media(res) + + +if __name__ == "__main__": + main() diff --git a/vk_url_scraper/utils.py b/vk_url_scraper/utils.py index 742ab8b..b2d5147 100644 --- a/vk_url_scraper/utils.py +++ b/vk_url_scraper/utils.py @@ -1,16 +1,16 @@ -import json -import os -from datetime import datetime - - -class DateTimeEncoder(json.JSONEncoder): - # to allow json.dump with datetimes do json.dumps(obj, cls=DateTimeEncoder) - def default(self, o): - if isinstance(o, datetime): - return str(o) # with timezone - return json.JSONEncoder.default(self, o) - - -def mkdir_if_not_exists(folder): - if not os.path.exists(folder): - os.makedirs(folder) +import json +import os +from datetime import datetime + + +class DateTimeEncoder(json.JSONEncoder): + # to allow json.dump with datetimes do json.dumps(obj, cls=DateTimeEncoder) + def default(self, o): + if isinstance(o, datetime): + return str(o) # with timezone + return json.JSONEncoder.default(self, o) + + +def mkdir_if_not_exists(folder): + if not os.path.exists(folder): + os.makedirs(folder) diff --git a/vk_url_scraper/version.py b/vk_url_scraper/version.py index b30be3e..6cf0ee6 100644 --- a/vk_url_scraper/version.py +++ b/vk_url_scraper/version.py @@ -1,8 +1,8 @@ _MAJOR = "0" -_MINOR = "1" +_MINOR = "2" # On main and in a nightly release the patch should be one ahead of the last # released build. -_PATCH = "5" +_PATCH = "0" # This is mainly for nightly builds which have the suffix ".dev$DATE". See # https://semver.org/#is-v123-a-semantic-version for the semantics. _SUFFIX = ""