mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-08 03:18:28 +03:00
Ruff format with defaults.
This commit is contained in:
erinhmclark
@@ -1 +1 @@
|
||||
from scripts import generate_module_docs
|
||||
from scripts import generate_module_docs
|
||||
|
||||
@@ -10,12 +10,12 @@ MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_arch
|
||||
SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
|
||||
|
||||
type_color = {
|
||||
'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
|
||||
'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
|
||||
'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
|
||||
'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
|
||||
'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
|
||||
'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
|
||||
"feeder": "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
|
||||
"extractor": "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
|
||||
"enricher": "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
|
||||
"database": "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
|
||||
"storage": "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
|
||||
"formatter": "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
|
||||
}
|
||||
|
||||
TABLE_HEADER = ("Option", "Description", "Default", "Type")
|
||||
@@ -34,6 +34,7 @@ steps:
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def generate_module_docs():
|
||||
yaml = YAML()
|
||||
SAVE_FOLDER.mkdir(exist_ok=True)
|
||||
@@ -48,49 +49,49 @@ def generate_module_docs():
|
||||
# generate the markdown file from the __manifest__.py file.
|
||||
|
||||
manifest = module.manifest
|
||||
for type in manifest['type']:
|
||||
for type in manifest["type"]:
|
||||
modules_by_type.setdefault(type, []).append(module)
|
||||
|
||||
description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
|
||||
types = ", ".join(type_color[t] for t in manifest['type'])
|
||||
description = "\n".join(l.lstrip() for l in manifest["description"].split("\n"))
|
||||
types = ", ".join(type_color[t] for t in manifest["type"])
|
||||
readme_str = f"""
|
||||
# {manifest['name']}
|
||||
# {manifest["name"]}
|
||||
```{{admonition}} Module type
|
||||
|
||||
{types}
|
||||
```
|
||||
{description}
|
||||
"""
|
||||
steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest['type'])
|
||||
"""
|
||||
steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest["type"])
|
||||
|
||||
if not manifest['configs']:
|
||||
if not manifest["configs"]:
|
||||
config_string = f"# No configuration options for {module.name}.*\n"
|
||||
else:
|
||||
|
||||
config_table = header_row
|
||||
config_yaml = {}
|
||||
|
||||
global_yaml[module.name] = CommentedMap()
|
||||
global_yaml.yaml_set_comment_before_after_key(module.name, f"\n\n{module.display_name} configuration options")
|
||||
global_yaml.yaml_set_comment_before_after_key(
|
||||
module.name, f"\n\n{module.display_name} configuration options"
|
||||
)
|
||||
|
||||
|
||||
for key, value in manifest['configs'].items():
|
||||
type = value.get('type', 'string')
|
||||
if type == 'json_loader':
|
||||
value['type'] = 'json'
|
||||
elif type == 'str':
|
||||
for key, value in manifest["configs"].items():
|
||||
type = value.get("type", "string")
|
||||
if type == "json_loader":
|
||||
value["type"] = "json"
|
||||
elif type == "str":
|
||||
type = "string"
|
||||
|
||||
default = value.get('default', '')
|
||||
|
||||
default = value.get("default", "")
|
||||
config_yaml[key] = default
|
||||
|
||||
global_yaml[module.name][key] = default
|
||||
|
||||
if value.get('help', ''):
|
||||
global_yaml[module.name].yaml_add_eol_comment(value.get('help', ''), key)
|
||||
if value.get("help", ""):
|
||||
global_yaml[module.name].yaml_add_eol_comment(value.get("help", ""), key)
|
||||
|
||||
help = "**Required**. " if value.get('required', False) else "Optional. "
|
||||
help += value.get('help', '')
|
||||
help = "**Required**. " if value.get("required", False) else "Optional. "
|
||||
help += value.get("help", "")
|
||||
config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
|
||||
global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
|
||||
readme_str += "\n## Configuration Options\n"
|
||||
@@ -98,18 +99,18 @@ def generate_module_docs():
|
||||
|
||||
config_string = io.BytesIO()
|
||||
yaml.dump({module.name: config_yaml}, config_string)
|
||||
config_string = config_string.getvalue().decode('utf-8')
|
||||
config_string = config_string.getvalue().decode("utf-8")
|
||||
yaml_string = EXAMPLE_YAML.format(steps_str=steps_str, config_string=config_string)
|
||||
readme_str += f"```{{code}} yaml\n{yaml_string}\n```\n"
|
||||
|
||||
if manifest['configs']:
|
||||
if manifest["configs"]:
|
||||
readme_str += "\n### Command Line:\n"
|
||||
readme_str += config_table
|
||||
|
||||
# add a link to the autodoc refs
|
||||
readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
|
||||
# create the module.type folder, use the first type just for where to store the file
|
||||
for type in manifest['type']:
|
||||
for type in manifest["type"]:
|
||||
type_folder = SAVE_FOLDER / type
|
||||
type_folder.mkdir(exist_ok=True)
|
||||
with open(type_folder / f"{module.name}.md", "w") as f:
|
||||
@@ -117,10 +118,10 @@ def generate_module_docs():
|
||||
f.write(readme_str)
|
||||
generate_index(modules_by_type)
|
||||
|
||||
del global_yaml['placeholder']
|
||||
del global_yaml["placeholder"]
|
||||
global_string = io.BytesIO()
|
||||
global_yaml = yaml.dump(global_yaml, global_string)
|
||||
global_string = global_string.getvalue().decode('utf-8')
|
||||
global_string = global_string.getvalue().decode("utf-8")
|
||||
global_yaml = f"```yaml\n{global_string}\n```"
|
||||
with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
|
||||
f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table)
|
||||
@@ -144,4 +145,4 @@ def generate_index(modules_by_type):
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
generate_module_docs()
|
||||
generate_module_docs()
|
||||
|
||||
@@ -5,7 +5,7 @@ import os
|
||||
from importlib.metadata import metadata
|
||||
from datetime import datetime
|
||||
|
||||
sys.path.append(os.path.abspath('../scripts'))
|
||||
sys.path.append(os.path.abspath("../scripts"))
|
||||
from scripts import generate_module_docs
|
||||
from auto_archiver.version import __version__
|
||||
|
||||
@@ -20,33 +20,35 @@ project = package_metadata["name"]
|
||||
copyright = str(datetime.now().year)
|
||||
author = "Bellingcat"
|
||||
release = package_metadata["version"]
|
||||
language = 'en'
|
||||
language = "en"
|
||||
|
||||
# -- General configuration ---------------------------------------------------
|
||||
extensions = [
|
||||
"myst_parser", # Markdown support
|
||||
"autoapi.extension", # Generate API documentation from docstrings
|
||||
"sphinxcontrib.mermaid", # Mermaid diagrams
|
||||
"sphinx.ext.viewcode", # Source code links
|
||||
"myst_parser", # Markdown support
|
||||
"autoapi.extension", # Generate API documentation from docstrings
|
||||
"sphinxcontrib.mermaid", # Mermaid diagrams
|
||||
"sphinx.ext.viewcode", # Source code links
|
||||
"sphinx_copybutton",
|
||||
"sphinx.ext.napoleon", # Google-style and NumPy-style docstrings
|
||||
"sphinx.ext.napoleon", # Google-style and NumPy-style docstrings
|
||||
"sphinx.ext.autosectionlabel",
|
||||
# 'sphinx.ext.autosummary', # Summarize module/class/function docs
|
||||
]
|
||||
|
||||
templates_path = ['_templates']
|
||||
templates_path = ["_templates"]
|
||||
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""]
|
||||
|
||||
|
||||
# -- AutoAPI Configuration ---------------------------------------------------
|
||||
autoapi_type = 'python'
|
||||
autoapi_type = "python"
|
||||
autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
|
||||
# get all the modules and add them to the autoapi_dirs
|
||||
autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
|
||||
autodoc_typehints = "signature" # Include type hints in the signature
|
||||
autoapi_ignore = ["*/version.py", ] # Ignore specific modules
|
||||
autoapi_keep_files = True # Option to retain intermediate JSON files for debugging
|
||||
autoapi_add_toctree_entry = True # Include API docs in the TOC
|
||||
autodoc_typehints = "signature" # Include type hints in the signature
|
||||
autoapi_ignore = [
|
||||
"*/version.py",
|
||||
] # Ignore specific modules
|
||||
autoapi_keep_files = True # Option to retain intermediate JSON files for debugging
|
||||
autoapi_add_toctree_entry = True # Include API docs in the TOC
|
||||
autoapi_python_use_implicit_namespaces = True
|
||||
autoapi_template_dir = "../_templates/autoapi"
|
||||
autoapi_options = [
|
||||
@@ -59,13 +61,13 @@ autoapi_options = [
|
||||
|
||||
# -- Markdown Support --------------------------------------------------------
|
||||
myst_enable_extensions = [
|
||||
"deflist", # Definition lists
|
||||
"html_admonition", # HTML-style admonitions
|
||||
"html_image", # Inline HTML images
|
||||
"replacements", # Substitutions like (C)
|
||||
"smartquotes", # Smart quotes
|
||||
"linkify", # Auto-detect links
|
||||
"substitution", # Text substitutions
|
||||
"deflist", # Definition lists
|
||||
"html_admonition", # HTML-style admonitions
|
||||
"html_image", # Inline HTML images
|
||||
"replacements", # Substitutions like (C)
|
||||
"smartquotes", # Smart quotes
|
||||
"linkify", # Auto-detect links
|
||||
"substitution", # Text substitutions
|
||||
]
|
||||
myst_heading_anchors = 2
|
||||
myst_fence_as_directive = ["mermaid"]
|
||||
@@ -76,7 +78,7 @@ source_suffix = {
|
||||
}
|
||||
|
||||
# -- Options for HTML output -------------------------------------------------
|
||||
html_theme = 'sphinx_book_theme'
|
||||
html_theme = "sphinx_book_theme"
|
||||
html_static_path = ["../_static"]
|
||||
html_css_files = ["custom.css"]
|
||||
html_title = f"Auto Archiver v{__version__}"
|
||||
@@ -87,7 +89,6 @@ html_theme_options = {
|
||||
}
|
||||
|
||||
|
||||
|
||||
copybutton_prompt_text = r">>> |\.\.\."
|
||||
copybutton_prompt_is_regexp = True
|
||||
copybutton_only_copy_prompt_lines = False
|
||||
copybutton_only_copy_prompt_lines = False
|
||||
|
||||
Reference in New Issue
Block a user