mirror of
https://github.com/bellingcat/vk-url-scraper.git
synced 2026-06-10 20:38:36 +03:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d155c1364a | ||
|
|
8882a87048 | ||
|
|
a95c675e9c | ||
|
|
8864e7c87d | ||
|
|
db9b613ae4 | ||
|
|
37828b4be4 | ||
|
|
1a3a7dc0f3 | ||
|
|
f67707a740 |
11
.github/dependabot.yml
vendored
11
.github/dependabot.yml
vendored
@@ -1,11 +0,0 @@
|
||||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: "pip"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
open-pull-requests-limit: 10
|
||||
- package-ecosystem: "github-actions"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "daily"
|
||||
28
Pipfile
28
Pipfile
@@ -4,8 +4,32 @@ verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
vk-api = "*"
|
||||
yt-dlp = "*"
|
||||
vk-api = ">=11.9.9"
|
||||
yt-dlp = ">=2023.2.17"
|
||||
flake8 = "*"
|
||||
mypy = ">=0.961"
|
||||
black = ">=22.3.0"
|
||||
isort = ">=5.10.1"
|
||||
pytest = "*"
|
||||
pytest-sphinx = "*"
|
||||
pytest-cov = "*"
|
||||
twine = ">=1.11.0"
|
||||
sphinx = "<5.1.0,>=4.3.0"
|
||||
furo = ">=2022.6.4.1"
|
||||
myst-parser = "<0.19.0,>=0.15.2"
|
||||
sphinx-copybutton = ">=0.5.0"
|
||||
sphinx-autobuild = ">=2021.3.14"
|
||||
sphinx-autodoc-typehints = "*"
|
||||
python-dotenv = ">=0.21.1"
|
||||
brotli = ">=1.0.9"
|
||||
certifi = ">=2022.12.7"
|
||||
charset-normalizer = ">=3.0.1"
|
||||
idna = ">=3.4"
|
||||
mutagen = ">=1.46.0"
|
||||
pycryptodomex = ">=3.17"
|
||||
requests = ">=2.28.2"
|
||||
urllib3 = ">=1.26.14"
|
||||
websockets = ">=10.4"
|
||||
|
||||
[dev-packages]
|
||||
sphinx-copybutton = "==0.5.0"
|
||||
|
||||
1973
Pipfile.lock
generated
1973
Pipfile.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -2,11 +2,11 @@
|
||||
flake8
|
||||
|
||||
# Static type checking
|
||||
mypy==0.961
|
||||
mypy>=0.961
|
||||
|
||||
# Automatic code formatting
|
||||
black==22.3.0
|
||||
isort==5.10.1
|
||||
black>=22.3.0
|
||||
isort>=5.10.1
|
||||
|
||||
# Running tests
|
||||
pytest
|
||||
@@ -24,19 +24,20 @@ wheel
|
||||
Sphinx>=4.3.0,<5.1.0
|
||||
|
||||
# Sphinx theme: https://sphinx-themes.org/sample-sites/furo/
|
||||
furo==2022.6.4.1
|
||||
furo>=2022.6.4.1
|
||||
|
||||
# Lets Sphinx parse markdown files in addition to rst.
|
||||
myst-parser>=0.15.2,<0.19.0
|
||||
|
||||
# Adds a copy button to code examples in the docs.
|
||||
sphinx-copybutton==0.5.0
|
||||
sphinx-copybutton>=0.5.0
|
||||
|
||||
# Live rebuilding and reloading of docs for developing locally.
|
||||
sphinx-autobuild==2021.3.14
|
||||
sphinx-autobuild>=2021.3.14
|
||||
|
||||
# Automatically adds types to docs
|
||||
sphinx-autodoc-typehints
|
||||
|
||||
# For parsing and comparing version numbers.
|
||||
packaging
|
||||
python-dotenv>=0.21.1
|
||||
100
requirements.txt
100
requirements.txt
@@ -2,94 +2,18 @@
|
||||
# These requirements were autogenerated by pipenv
|
||||
# To regenerate from the project's Pipfile, run:
|
||||
#
|
||||
# pipenv lock --requirements --dev
|
||||
# pipenv lock --requirements
|
||||
#
|
||||
|
||||
# Note: in pipenv 2020.x, "--dev" changed to emit both default and development
|
||||
# requirements. To emit only development requirements, pass "--dev-only".
|
||||
|
||||
# -i https://pypi.org/simple
|
||||
alabaster==0.7.13; python_version >= '3.6'
|
||||
attrs==22.2.0; python_version >= '3.6'
|
||||
babel==2.11.0; python_version >= '3.6'
|
||||
beautifulsoup4==4.11.2; python_version >= '3.6'
|
||||
black==22.3.0
|
||||
bleach==6.0.0; python_version >= '3.7'
|
||||
brotli==1.0.9; platform_python_implementation == 'CPython'
|
||||
certifi==2022.12.7; python_version >= '3.6'
|
||||
cffi==1.15.1
|
||||
charset-normalizer==3.0.1; python_version >= '3.6'
|
||||
click==8.1.3; python_version >= '3.7'
|
||||
colorama==0.4.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'
|
||||
coverage[toml]==7.2.0; python_version >= '3.7'
|
||||
cryptography==39.0.1; python_version >= '3.6'
|
||||
docutils==0.18.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
exceptiongroup==1.1.0; python_version < '3.11'
|
||||
flake8==6.0.0
|
||||
furo==2022.6.21
|
||||
idna==3.4; python_version >= '3.5'
|
||||
imagesize==1.4.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
importlib-metadata==6.0.0; python_version >= '3.7'
|
||||
iniconfig==2.0.0; python_version >= '3.7'
|
||||
isort==5.10.1
|
||||
jaraco.classes==3.2.3; python_version >= '3.7'
|
||||
jeepney==0.8.0; sys_platform == 'linux'
|
||||
jinja2==3.1.2; python_version >= '3.7'
|
||||
keyring==23.13.1; python_version >= '3.7'
|
||||
livereload==2.6.3
|
||||
markdown-it-py==2.2.0; python_version >= '3.7'
|
||||
markupsafe==2.1.2; python_version >= '3.7'
|
||||
mccabe==0.7.0; python_version >= '3.6'
|
||||
mdit-py-plugins==0.3.4; python_version >= '3.7'
|
||||
mdurl==0.1.2; python_version >= '3.7'
|
||||
more-itertools==9.0.0; python_version >= '3.7'
|
||||
mutagen==1.46.0; python_version >= '3.7'
|
||||
mypy-extensions==1.0.0; python_version >= '3.5'
|
||||
mypy==0.961
|
||||
myst-parser==0.18.1
|
||||
packaging==23.0; python_version >= '3.7'
|
||||
pathspec==0.11.0; python_version >= '3.7'
|
||||
pkginfo==1.9.6; python_version >= '3.6'
|
||||
platformdirs==3.0.0; python_version >= '3.7'
|
||||
pluggy==1.0.0; python_version >= '3.6'
|
||||
pycodestyle==2.10.0; python_version >= '3.6'
|
||||
pycparser==2.21
|
||||
pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pyflakes==3.0.1; python_version >= '3.6'
|
||||
pygments==2.14.0; python_version >= '3.6'
|
||||
pytest-cov==4.0.0
|
||||
pytest-sphinx==0.5.0
|
||||
pytest==7.2.1
|
||||
python-dotenv==0.21.1
|
||||
pytz==2022.7.1
|
||||
pyyaml==6.0; python_version >= '3.6'
|
||||
readme-renderer==37.3; python_version >= '3.7'
|
||||
requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
requests==2.28.2; python_version >= '3.7' and python_version < '4'
|
||||
rfc3986==2.0.0; python_version >= '3.7'
|
||||
rich==13.3.1; python_version >= '3.7'
|
||||
secretstorage==3.3.3; sys_platform == 'linux'
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
snowballstemmer==2.2.0
|
||||
soupsieve==2.4; python_version >= '3.7'
|
||||
sphinx-autobuild==2021.3.14
|
||||
sphinx-autodoc-typehints==1.19.1
|
||||
sphinx-basic-ng==1.0.0b1; python_version >= '3.7'
|
||||
sphinx-copybutton==0.5.0
|
||||
sphinx==5.0.2
|
||||
sphinxcontrib-applehelp==1.0.4; python_version >= '3.8'
|
||||
sphinxcontrib-devhelp==1.0.2; python_version >= '3.5'
|
||||
sphinxcontrib-htmlhelp==2.0.1; python_version >= '3.8'
|
||||
sphinxcontrib-jsmath==1.0.1; python_version >= '3.5'
|
||||
sphinxcontrib-qthelp==1.0.3; python_version >= '3.5'
|
||||
sphinxcontrib-serializinghtml==1.1.5; python_version >= '3.5'
|
||||
tomli==2.0.1; python_version < '3.11'
|
||||
tornado==6.2; python_version > '2.7'
|
||||
twine==4.0.2
|
||||
typing-extensions==4.5.0; python_version >= '3.7'
|
||||
urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
vk-api==11.9.9
|
||||
webencodings==0.5.1
|
||||
websockets==10.4; python_version >= '3.7'
|
||||
yt-dlp==2023.2.17
|
||||
zipp==3.14.0; python_version >= '3.7'
|
||||
brotli>=1.0.9; platform_python_implementation >= 'CPython'
|
||||
certifi>=2022.12.7; python_version >= '3.6'
|
||||
charset-normalizer>=3.0.1; python_version >= '3.6'
|
||||
idna>=3.4; python_version >= '3.5'
|
||||
mutagen>=1.46.0; python_version >= '3.7'
|
||||
pycryptodomex>=3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
requests>=2.28.2; python_version >= '3.7' and python_version < '4'
|
||||
urllib3>=1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
vk-api>=11.9.9
|
||||
websockets>=10.4; python_version >= '3.7'
|
||||
yt-dlp>=2023.2.17
|
||||
@@ -138,7 +138,7 @@ def test_scrape_video_only():
|
||||
assert len(res) == 1
|
||||
assert res[0]["id"] == "video38556806_456251917"
|
||||
assert str(res[0]["datetime"]) == str(datetime.datetime(2022, 3, 24, 5, 42, 38))
|
||||
assert len(res[0]["payload"]) == 31
|
||||
assert len(res[0]["payload"]) == 34
|
||||
assert len(res[0]["attachments"].keys()) == 1
|
||||
assert list(res[0]["attachments"].keys()) == ["video"]
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ import re
|
||||
import shutil
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import requests
|
||||
@@ -43,7 +43,7 @@ class VkScraper:
|
||||
self,
|
||||
username: str,
|
||||
password: str,
|
||||
token: str = None,
|
||||
token: Optional[str] = None,
|
||||
session_file="vk_config.v2.json",
|
||||
captcha_handler=captcha_handler,
|
||||
) -> None:
|
||||
|
||||
@@ -2,7 +2,7 @@ _MAJOR = "0"
|
||||
_MINOR = "3"
|
||||
# On main and in a nightly release the patch should be one ahead of the last
|
||||
# released build.
|
||||
_PATCH = "13"
|
||||
_PATCH = "21"
|
||||
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
|
||||
# https://semver.org/#is-v123-a-semantic-version for the semantics.
|
||||
_SUFFIX = ""
|
||||
|
||||
Reference in New Issue
Block a user