mirror of
https://github.com/bellingcat/sugartrail.git
synced 2026-06-08 03:28:31 +03:00
implemented zip download including csvs and json formats
This commit is contained in:
seangreaves
@@ -19,7 +19,9 @@
|
||||
"import pandas as pd\n",
|
||||
"import ipywidgets as widgets\n",
|
||||
"from IPython.display import display, HTML\n",
|
||||
"import requests"
|
||||
"import requests\n",
|
||||
"import shutil\n",
|
||||
"from pathlib import Path"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -43,14 +45,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 3,
|
||||
"id": "0632780b",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "4df7b16d214240ddbc7106f2e0120942",
|
||||
"model_id": "1161d0ba01644c1aa3c0a45f82658a7b",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -64,12 +66,12 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "8f0467c4899241579255adfcd5d270db",
|
||||
"model_id": "ccd5abbda2f140888ee19075be763d79",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(description='Authenticate', style=ButtonStyle(button_color='#F5F5F5'))"
|
||||
"Button(description='Authenticate', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -78,7 +80,7 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f755529c26c44e8aa0a6c701f9270e0c",
|
||||
"model_id": "88984a8fbde64f7491a6ce08316758f4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -158,12 +160,12 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "09a265b50120419c83fbee18954959b8",
|
||||
"model_id": "1c410b4c2eeb4345b7bcdfcf1622c502",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Text(value='', disabled=True, placeholder='Insert Company ID')"
|
||||
"Text(value='', placeholder='Insert Company ID')"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -172,12 +174,12 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "33909f0e627f48199089e9a66cb40e46",
|
||||
"model_id": "e44b1eae883d4d30abc7a21b5f65b207",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(button_style='success', description='Initialise', disabled=True, style=ButtonStyle())"
|
||||
"Button(description='Initialise', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
@@ -186,7 +188,7 @@
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "6cfca501106049db984df7a5edc72140",
|
||||
"model_id": "028a5edf0df54912a206ceea49c025a6",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -214,6 +216,9 @@
|
||||
"generate_network_button = widgets.Button(description='Build Network', disabled=True)\n",
|
||||
"generate_network_button.on_click(lambda bt: generate_network()) \n",
|
||||
"\n",
|
||||
"navigation_button = widgets.Button(description='Update', disabled=True)\n",
|
||||
"navigation_button.on_click(lambda bt: config_network()) \n",
|
||||
"\n",
|
||||
"def init_network():\n",
|
||||
" init_button.disabled=True\n",
|
||||
" response = api.get_company(str(company_text.value))\n",
|
||||
@@ -244,14 +249,11 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 5,
|
||||
"id": "6eb9065a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"navigation_button = widgets.Button(description='Update', disabled=True)\n",
|
||||
"navigation_button.on_click(lambda bt: config_network()) \n",
|
||||
"\n",
|
||||
"officers_for_company = widgets.Checkbox(\n",
|
||||
" value=True,\n",
|
||||
" description='Get officers for each company',\n",
|
||||
@@ -354,7 +356,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 6,
|
||||
"id": "7da39d70",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -380,7 +382,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 7,
|
||||
"id": "bec983fc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -391,14 +393,14 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 8,
|
||||
"id": "1c68e6b4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "515ecf4ccec248a286bc98390cd9b90d",
|
||||
"model_id": "9c4b347bd3ac4fc894bab6e419d1515f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
@@ -406,7 +408,7 @@
|
||||
"Accordion(children=(VBox(children=(Checkbox(value=True, description='Get officers for each company', indent=Fa…"
|
||||
]
|
||||
},
|
||||
"execution_count": 9,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
@@ -425,10 +427,67 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 9,
|
||||
"id": "ea0e8392",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e1b9d44b6b5c4885865e137ce860e1c4",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"BoundedIntText(value=1, max=20, min=1)"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "bdf08191a94448b9b9e714b2e70d1f40",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(description='Build Network', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e31d7367788d4e0d9f1e396c44223425",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HTML(value='')"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "93193405b6b044a98c36b21a761fd65f",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output()"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"build_status = widgets.HTML(\n",
|
||||
" value=\"\",\n",
|
||||
@@ -461,10 +520,39 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 10,
|
||||
"id": "6449cd96",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2e3ae2038efe4a8486ec3bcc35d6a3b9",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Button(description='Build Map', style=ButtonStyle())"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "3a2ab7ede61e48f6bcc95150e10cd500",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"HTML(value='')"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"map_container = widgets.HTML(\n",
|
||||
" value=\"\",\n",
|
||||
@@ -499,6 +587,21 @@
|
||||
"</html>\n",
|
||||
"'''\n",
|
||||
"\n",
|
||||
"def generate_download_zip(company_text, network):\n",
|
||||
" Path(str(company_text.value)).mkdir(parents=True, exist_ok=True)\n",
|
||||
" df_company_ids = pd.DataFrame(network.company_ids)\n",
|
||||
" df_companies = pd.DataFrame(network.companies)\n",
|
||||
" df_addresses = pd.DataFrame(network.addresses)\n",
|
||||
" df_officer_ids = pd.DataFrame(network.officer_ids)\n",
|
||||
" files = {'companies': df_company_ids, 'addresses': df_addresses, 'officers': df_officer_ids, 'company_details': df_companies}\n",
|
||||
" for key in files:\n",
|
||||
" files[key].to_csv(str(company_text.value) + '/' + key + '.csv')\n",
|
||||
" file = str(company_text.value) + '.json'\n",
|
||||
" network.save(file, './' + str(company_text.value) + \"/\")\n",
|
||||
" zip_filename = str(company_text.value) + '.zip'\n",
|
||||
" shutil.make_archive(str(company_text.value), 'zip', str(company_text.value))\n",
|
||||
" return zip_filename\n",
|
||||
"\n",
|
||||
"def generate_map():\n",
|
||||
" map_data,path_table = mapview.build_map(network, clear_widget=False) \n",
|
||||
" hbox_map = widgets.HBox([path_table])\n",
|
||||
@@ -515,9 +618,8 @@
|
||||
" display(pd.DataFrame(network.officer_ids))\n",
|
||||
" with tab.children[3]:\n",
|
||||
" display(pd.DataFrame(network.companies))\n",
|
||||
" file = str(company_text.value) + '.json'\n",
|
||||
" network.save(file, \"./\")\n",
|
||||
" html_button = html_buttons.format(filename=file)\n",
|
||||
" zip_filename = generate_download_zip(company_text, network)\n",
|
||||
" html_button = html_buttons.format(filename=zip_filename)\n",
|
||||
" with download_link:\n",
|
||||
" display(HTML(html_button))\n",
|
||||
"\n",
|
||||
@@ -527,30 +629,78 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 11,
|
||||
"id": "684a116e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "9b914a00be10491f82169cf06e9fe013",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Accordion(children=(VBox(children=(Map(center=[50, 0], controls=(ZoomControl(options=['position', 'zoom_in_tex…"
|
||||
]
|
||||
},
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"accordion_map"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 12,
|
||||
"id": "1e328d41",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "1952860a8548480dad48dbd10b764791",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Accordion(children=(Tab(children=(Output(outputs=({'output_type': 'display_data', 'data': {'text/plain': ' co…"
|
||||
]
|
||||
},
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"accordion_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"execution_count": 13,
|
||||
"id": "9224dce5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "e7dd9b6842f440b4bbb7d080bafefab0",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Output(outputs=({'output_type': 'display_data', 'data': {'text/plain': '<IPython.core.display.HTML object>', '…"
|
||||
]
|
||||
},
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"download_link"
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user