From c4a3a45bf7e542086b59c5767a3c67993e0654d6 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 17 Mar 2025 15:35:09 +0000 Subject: [PATCH 01/27] Script to auto-generate a service account --- .gitignore | 1 + scripts/generate_google_services.sh | 115 ++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 scripts/generate_google_services.sh diff --git a/.gitignore b/.gitignore index 72d8531..704947b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ temp/ .DS_Store expmt/ service_account.json +service_account-*.json __pycache__/ ._* anu.html diff --git a/scripts/generate_google_services.sh b/scripts/generate_google_services.sh new file mode 100644 index 0000000..2668fd3 --- /dev/null +++ b/scripts/generate_google_services.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash + +set -e # Exit on error + + +UUID=$(LC_ALL=C tr -dc a-z0-9 /dev/null; then + echo "✅ Google Cloud SDK is already installed" + return 0 + fi + + echo "📦 Installing Google Cloud SDK..." + + # Detect OS + case "$(uname -s)" in + Darwin*) + if command -v brew &> /dev/null; then + echo "🍺 Installing via Homebrew..." + brew install google-cloud-sdk --cask + else + echo "📥 Downloading Google Cloud SDK for macOS..." + curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-darwin-x86_64.tar.gz + tar -xf google-cloud-cli-latest-darwin-x86_64.tar.gz + ./google-cloud-sdk/install.sh --quiet + rm google-cloud-cli-latest-darwin-x86_64.tar.gz + echo "🔄 Please restart your terminal and run this script again" + exit 0 + fi + ;; + Linux*) + echo "📥 Downloading Google Cloud SDK for Linux..." + curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-linux-x86_64.tar.gz + tar -xf google-cloud-cli-latest-linux-x86_64.tar.gz + ./google-cloud-sdk/install.sh --quiet + rm google-cloud-cli-latest-linux-x86_64.tar.gz + echo "🔄 Please restart your terminal and run this script again" + exit 0 + ;; + CYGWIN*|MINGW*|MSYS*) + echo "⚠️ Windows detected. Please follow manual installation instructions at:" + echo "https://cloud.google.com/sdk/docs/install-sdk" + exit 1 + ;; + *) + echo "⚠️ Unknown operating system. Please follow manual installation instructions at:" + echo "https://cloud.google.com/sdk/docs/install-sdk" + exit 1 + ;; + esac + + echo "✅ Google Cloud SDK installed" +} + +# Install Google Cloud SDK if needed +install_gcloud_sdk + +# Login to Google Cloud +if gcloud auth list --filter=status:ACTIVE --format="value(account)" | grep -q "@"; then + echo "✅ Already authenticated with Google Cloud" +else + echo "🔑 Authenticating with Google Cloud..." + gcloud auth login +fi + +# Create project +echo "🌟 Creating Google Cloud project: $PROJECT_NAME" +gcloud projects create $PROJECT_NAME + +# Create service account +echo "👤 Creating service account: $ACCOUNT_NAME" +gcloud iam service-accounts create $ACCOUNT_NAME --project $PROJECT_NAME + +# Get the service account email +echo "📧 Retrieving service account email..." +ACCOUNT_EMAIL=$(gcloud iam service-accounts list --project $PROJECT_NAME --format="value(email)") + +# Create and download key +echo "🔑 Generating service account key file: $KEY_FILE" +gcloud iam service-accounts keys create $KEY_FILE --iam-account=$ACCOUNT_EMAIL + +# Enable required APIs (uncomment and add APIs as needed) +echo "⬆️ Enabling required Google APIs..." +gcloud services enable sheets.googleapis.com --project $PROJECT_NAME +gcloud services enable drive.googleapis.com --project $PROJECT_NAME + +echo "=====================================================" +echo "✅ SETUP COMPLETE!" +echo "=====================================================" +echo "📝 Important Information:" +echo " • Project Name: $PROJECT_NAME" +echo " • Service Account: $ACCOUNT_EMAIL" +echo " • Key File: $KEY_FILE" +echo "" +echo "📋 Next Steps:" +echo " 1. Share any Google Sheets with this email address:" +echo " $ACCOUNT_EMAIL" +echo " 2. Move $KEY_FILE to your auto-archiver secrets directory" +echo " 3. Update your auto-archiver config to use this key file (if needed)" +echo "=====================================================" \ No newline at end of file From 29db537fabe15f4941372d04495c0c0f295e598d Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 17 Mar 2025 18:11:18 +0000 Subject: [PATCH 02/27] Docs on using the script to auto-generate service accounts --- docs/source/how_to/gsheets_setup.md | 49 ++++++++++++------ docs/source/how_to/share_sheet.png | Bin 0 -> 61725 bytes .../modules/gsheet_feeder_db/__manifest__.py | 11 ++-- 3 files changed, 40 insertions(+), 20 deletions(-) create mode 100644 docs/source/how_to/share_sheet.png diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index af17274..eaad809 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -6,12 +6,31 @@ This guide explains how to set up Google Sheets to process URLs automatically an 2. Setting up a service account so Auto Archiver can access the sheet 3. Setting the Auto Archiver settings -### 1. Setting up your Google Sheet -Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. -Your sheet can have many other columns that the Auto Archiver can use, and you can also include any additional columns for your own personal use. The order of the columns does not matter, the naming just needs to be correctly assigned to its corresponding value in the configuration file. +## 1. Setting up a Google Service Account -We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches the default column names. +Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it. + +To do this, you can either: +* a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder +* b) run the `bash scripts/generate_google_services.sh` script to automatically generate the file. This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created. + +Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4). + +Also make sure to **note down** the email address for this service account. You'll need that for step 3. + +```{note} +The email address created in this step can be found either by opening the `service_account.json` file, or if you used b) the `generate_google_services.sh` script, then the script will have printed it out for you. + +The email address will look something like `user@project-name.iam.gserviceaccount.com` +``` + + +## 2. Setting up your Google Sheet + +We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches all the columns required. + +But if you like, you can also create your own custom sheet. The only column that's required is the 'link' column. This is the column with the URLs that you want the Auto Archiver to archive. Here's an overview of all the columns, and what a complete sheet would look like. @@ -46,21 +65,18 @@ In this example the Ghseet Feeder and Gsheet DB are being used, and the archive ![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png) -We'll change the name of the 'Destination Folder' column in step 3. +We'll change the name of the 'Destination Folder' column in the Step 4a. -## 2. Setting up your Service Account +## 3. Share your Google Sheet with your Service Account email address -Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it. +Remember that email address you copied in Step 1? Now that you've set up your Google sheet, click 'Share' in the top +right hand corner and enter the email address. Make sure to give the account **Editor** access. Here's how that looks: -To do this, follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and shared the Google Sheet with the log 'client_email' email address in this file. +![Share sheet](share_sheet.png) -Once you've downloaded the file, save it to `secrets/service_account.json` +## 4. Setting up the configuration file -## 3. Setting up the configuration file - -Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration. - -First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look: +The final step is to set your configuration. First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also put `gsheet_feeder_db` setting in the `steps.databases` section. Here's how this might look: ```{code} yaml steps: @@ -75,12 +91,15 @@ steps: Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'. +If you saved your `service_account.json` file to anywhere other than the default location (`secrets/service_account.json`), then also make sure to change that now: + Here's how this might look: ```{code} yaml ... gsheet_feeder_db: sheet: 'My Awesome Sheet' + service_account: secrets/service_account-XXXXX.json # or leave as secrets/service_account.json ... ``` @@ -90,7 +109,7 @@ You can also pass these settings directly on the command line without having to Here, the sheet name has been overridden/specified in the command line invocation. -### 3a. (Optional) Changing the column names +### 4a. (Optional) Changing the column names In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like: diff --git a/docs/source/how_to/share_sheet.png b/docs/source/how_to/share_sheet.png new file mode 100644 index 0000000000000000000000000000000000000000..f45a56d1fa2ab0a5c4f43008704152cd56af130d GIT binary patch literal 61725 zcmeFYWm6nc*Dj1Z!QI{6-Q5ETF2Nxx50aTEzR1SR#svDI!64W|v;@A}wr81SXIG@HhP!-kUS1ClQ zn)lmicuw!PdGUJYY;Xt$!|#!yEIq_c)1ZUo1$OlxLeMH?B4=S3qBU8Amod!qS66>X zM;%Bzz&dW`07}J0&Jc5fy>?~j_E!%?p?Y{Krc{Vgdvm0(f8Kb=-a2>p84>pDB0h*| z<6{>CiT2NwZg)B9K1SY$(y~!wn>#{j8nK=>>Y7`ZQPY}OCHzL(HOk=W3-bnwGNdg{ zk03ojf5Ie@&SC!%jdan3H;$RwbsUk;+P_y)Y=oVhaUQIK?o;*;f)?)C5zPL!^nEp#kgMeNjddd1E;#*jZ=S!0g;xVAtX z-aqIcs+=2%E%DKEWtOiiegj} z0TVhhz}igK+%y5s#6k-C9)}t$5hMQk;sh_)Z#0l>fLZ90vb;k$=+CQosWQJeiYq%3 ztPt2_ww#ve4<)o{Fz}v&{PGiCAEBGshub&bfS{Kt7fPJTDO%S;hp2Q5DUqKT$I}|W6CV1j z3(8o|6MSVnDu6t$u$w`heLjM1967)68dd{(Q}sWg&%PSMNr)B(h{WK)m!cKlYdJ*3 z|E!yLA=A;?e;j$P@KYn15b&zAIecEFXy6EQT-H=^!19C=iOdDDo2luEnSC0mNB9me z@(mQqa;~%yLz2N(Hl6QK!2_7|3v3uu9m}=P5zg(Yc)XIGze>seDxxoJOg}X`Vd1(- zAOqS~f{Ulon^K>ufnw~H__nl?eb{q>GT`tqHTgycb+Oak-L25$UN{21eDKZ`IkMNC z-{Ia)3wHDQQ?PVe?Y7HaIMfCypzY~TFc&8xKPS1(CPG&b1rl5#B{nFSmJ{nsKJrpK znkqa`zo;gha-UlrLQ1Hb8A}CBXMcbuMpmDa3%(JIZXdb}K~3H}Fbp8nYm+C0k~S=Y z6Vo3AS1)^tWfK;6PZ_Kzh#g8xRjG(TErmO(YAOCcyMWp;|zVBow1LE7VIl+xCh;1P-#rUBmh zQ+EfMC`(B8s$R^M*c#Zm@jYZU^0Dw@d|sbBoi(?On~V85wqQ`ytbG!)=b{%*f z3LYz&g!3^!EcQ9mx$bzJxh2qfh=+;jiCet&&g!ZL*H6%wuIF)%@J`NVI}7iW@=GZy z@~eBLdrjVYpqPoLTk2!2fHIsBM2}yB%TFie$GsB4!cBbeH_FeB&V?` zHlwMW6)x76Hy&2Ckl10}Sr{Gxil`CNi_@DYFVK&uy_y|02!1F1NOZzmweoX@7o_v4 zW2kB2Ph)1YewTjXFY?BxeI54}pN%G?)_}$my#_WOwk3nSMn7jO z$5h+qnQ!aI?ag_7(;a;s!;g9Blfz5aF>0lg9)?A1EBq^-p=Wp^Kc^Ph=W7Pe8^r7{ ztT|dX%r^K#I0r1kEM@ni4Bs8UBaQTnB#w-XR~>N~kskRnl1{@WPoH4Kb+tO`tX`uo zq`s1K9zREpC;ZJtXQyIm%5w_l0OjDM*seHKewt{F^D#BHJ9gBnc-&Dp;zVyLyk@e- zbZvFb(y7@g_Zy+;Fnj8)zFhPi1jhsSkP4rsy&i8e1Z!`6*dH&>q z@Pw;~Yh_sUmj8O+*ZI=v?$blbqrkf-yoSLbyZhLpnl@Lv7&lk*LDL!oMT65Akdx zdoOgIYu?T17nTqH95f#s8zc{32_GLI87Z=m8L1)m5ub>t! zjI9QYkEw|YAm(AVww`F$lRqt4&aJ=Le%?tNT8nFq)&&VL%;3Do{)V&8fIwTL>Odjj zw))*QJqaxdq=f#XWH)U0?1$J6>_{!q#@Fqv_bG1fAva-mIP9vt85um6hqRBhuO+aR zVLw4Xb1O~nzdkDmXb^Ka`Wv(cvFNdQQ~klI!S{^ml4FsZp6j01Et8lAPwA!3W*%kh zXZ@nvj~O4TZIEu@x9XmbpIHC8mYtl@$dI0so#m@n#P%aCC(l9D=l3puWp|U}Z}B1b zu`H&w2F_oeLe%+`&ZG0e5|3;sBtsf8kq_bzcn=h1^r~W2hk5mlP+u2sya@vdE;s>5TGe%Hfo+9sPv z>c`Yl!)U|nR`Ip8y0cY|Q`2v)Bn}+EUOgf$6?{LaE*#n49UtdLh@>wR@Wo_J88da( zoW5IL(>&4aw8C)L_+e6~TYs0}U-{&8j!E6G<2ubCTF`>d2@!Y zgzgwAgCQXBg^yf#+uKqh*-m>%Ypr-ZUMqo3aM8DG(|a2ED7q$@h=D>t!@kGYezo$X zyjY_(4_UnsoEobjYGib zvTbgezwB)J!$uWDExeJ4Z?4zNTwD~1qri^ydQYlb)k_Qb_F6>9%j6~PZRMcT{Q~P8 zZ)2&;^sx^twRh3ddtvAGS>lm%^V4|ovVs={DtWIrb>LO0MZfm%`8gZ<>2LWW63)kH z$6Z$=_Y%S;il%<&%#Ye9(>ENBCgjiK7qw~Uu4`nqmbG6@Ied!WtVayfTPMY|#4y(P zdY+zp?n7TAza^X|9Ej5ee7W^GCYVcasBJMN=?(YqT*_WqVc8qM_wR)v)FgMDgZA!* zg)+c~Li2ViiPz1=ctR->p1-96=hq#h{nA~9stAAkGiwaK*vE(RZG|?2zd2%plAVQ$ zx`sMDoDKEZWL@$)*TJV2C2`0D^RWq_KW7&SC%V#w$ zf>wG;)@o`{%#dp&DCkH#C^*O!G~^_PoRB1>0OtSPf#WWK{Xf?*=6@eDy71~kK}kX> z$x7*XL!ag%w&BVxMxcW5F!A8Ee#onuO)*+yU1#aKdZ70TRd>Ybdo3BeFMi-U=sM1l zlf~7N- zM=P>aN`W!|J!lZ**^C$^4E28>ntX7ymO7KA@8#wGdr(QpvqSQKicmoegU$VI|2_tzBoV;=h&*BClku zvMi_bdRK%4-)HNr59SD|4F4Fu!WBkp(l6kpE3-OZH|7H{G1z;nj}sk)*6GkwqQ%t3 z%ub>H$8|ntynve)22Zk2jKawPTK_%1Y{52C+rgk33w!8TB9aai^qpUu9lWKIK4x$X z&1N2kKy@1$vkW|zi-(8j{_n?LROPzS+$^I}$qLCOl`rfD z%rN2*H4#UjjFtQXQJgvV+4opryw>V?&DMUgDJ;9qjkNN zHJ#sl+g(zPM_oeT8OFK-=hSyx#}6il3nDl z)4UiX>5uI)9r^!pKm1tq^MM`Ke%_<-3qwVdAu1`xK&q3gg#eE3(j7n3F9-c_Ai}J9 zz;i4(g%|{s?5Z0#Xmzk#H2BOmU#x&J-~3s3zQ^BpzQf~0eu!aJgXf=GrG!%uApUug zmB;ZnFHgX;H;?D39*^||A=Wr;YWWNjMyu08(xv^C>nf#k78DcTCEf) z<04$@@tR;F`$B7`(%q)A*kyx0fnM^RoCT_(N=CDE4wX2?O^gNw^ZrKQUGRu68{+IRXum>OH2d;~P`(T63 zXZ-?#(3bd0N%M&4E({uiqPX4+D6IkKKYu*$aRhR$b$C>*vXtcfbD%Id7g~34^P%q+2q@1+d4!gtGis~(iCCTfP@UTpbm*}2iPY<~-g;J!K<&{l zCGVg1M}(wVUhbqpCV|Vdk4iknQ;hqIF*EVg^}6kKpbIr~5AUC^nIc|!cE8-P<~zME%&)6n{^?_x6QMi;j$dt|dgvuT=6+0+ZNVYnUR9)1 zJtDABNNf(gp6EEKApz25%a)}^{BsUgSdxeZ=LrZw_(AlEMOd^cYoKtn^jb;Beh~{3 z89yMo*q62sS^o?SKPiI?&M)ksmA;q@8bzZJ&7xhcRB{qmf~b~s{t;*FlO;9pyt;Yz zBlBErh!@aSul&j(Z=G)j--qzXW3ms%5VZuVc6y$z+N=FjSbK=DfxJ*vNN`y}^N2~J z3zp$%It1A&B3zSM0)HaweK6DuJWdvs_6_a+X#qNf!4Vx2dFcEWpwYVDP{dN3as8?= zX%&fBlka6RG;}f|peT@cb(Q`dd>DfxHuKQ!q2Bys?Gau({PuD1xZl59ovF2a^{%hX z)cBUY76J(9GuNMM=+KQ-UC1DtT$5=S0%XXtqoeVC7fl?`BW_%CJ<|RW zMgHHP{%6Jh-}3&CZv6jDkzmpimZ=9ny=W1R2 zAje&GqsKp1jo<3cG~i)K^nNq6e`AcL*5RG_z1e)D&2)<6pW+52zq>WBGSl9GPi3C# zJ~ljNeL>(>>;LP4v(g}#pZflZ8sW#P-}PkKhpLbJyU7~t)C)mn z8ME~+pNQc&K+7m~Y>DA_D}6|IH44M9@~tAOzeR<$KJ1srl5ykN$K(_1$t)`q?=AT2 zpV}IkOQwNqvMeyT)JqW1+c?hNNodgg8-jb2Nsli}oVss5UQY^aWTdefB2=e0@sHFC zx$kF?ICss0rM+67&O5XB(oNT2el{T^qmhd)cfH)Me0m2Cdz{aI#eH2=4S*x}-V9D5 zE7Qn#LO>_B^j!WFnX>n&MHrkQ{x~Z7u$#P}X_LnV*^5%CFfh85f9F{X59G}L^{oPz8w3{ds(%VF0y9EPh zF!Lp^)pE*g?M7tza2~<$mF{jMJCzqCWnt@9?EppKXwn}5C>3w~yBH}Wa z1$$i%5g?wo>%{0}s47%_N$Y-lecB&DfDSvjD)IHtw!Rtt)D<5h!A%=3#2E0K7~f(cChxNN_J zvJ)E&_y=*EPO4kq5wo|hd9FBQFD`U=ZTK%eU+2H6+V#c45K092Y1ZtEEEFpw93Fpr zR3j%~Gv~83Yph>g=OrNt-CHb*J2kg5EVHS?c^PS>_k1M`Y%cmf&6N9jXZkH`S zt8d2SeXf2h$ntt;NV}@*^X{8gtLR2d~Fb= zLadIN^>{QU4UQ!L!ssU;{`P$BLnklFy3RB83XYV>czf!7bC=+QkQ25rFo0lC@>$*B zJJj{&c3HHB`#}r|xRnNsr=*ApIO#+d9-AGhoj`V1@s-z`mKwQien)IE8}_75O(S5t z`{igw+%f_-7qN3{@7v48{_E2Pgt>UQ1NSs>f}@vLc(ab5Mxdw>1|C8SPOt;3&HZB!(xAnQU8y_@W@Pv2X+Ax6Z2T`@Mz_(W^ zQZ$Lw(lmpy?>&#l{dYFucpo`U{ck>iU;2H4ZA4R;Kbb7uHL66ii{C$|xCK`oeurpl zuot9ysHw1PjaIyueaEGrd7FJ~_d}}w1PlCaD?DV{+n=kt$=rj@F=Jx|G!*Z0dN-w{(bxBnKhCjuj+?m&Ip!tmAYlp`{)RY*+e@WSX9Au8Oj z={#44py)9TsISP*0yc16gDct)P^=jMV;cC)sZ3JQ)oW$VB2IAGw*! zVhvlSgilIJvXd0aUNx7J2nmd-<(X*(R=*p5nUO+-cRI+FMQf31A394h%kPofp}jxh zHO+*JKqJELarm(nFLIutYsSz!lpk>HN0>W{fzv9hKac6Xm#T}F$2Cup ztVSesA?ywvmLv!gBgNj!0&mY&NV=+{9OYZVs?-Pv{lYDu3~xV~+VEN?@sCu<$s`Uv z!A*ZKAEwgT3=d&+9Yhx>K@vF*;}PFM*y)j#mKVgqJfr+c$>2QP}#gWdA`b9{SF2$`x^c3 zAWzKqF_wGEx#0&r!q!@=pM}u=%(a_QA_2nY4+?#s-hRgny zTNLqEq`%T2#cTGt@Xt<5bV!O|*^a{N}1rkaVa%XL0#i#hSeBiCWc8D~OPN2@4&FS<2Q4!GWNQvCo&u5&!{k? z_Epu^5H2k3eFd;lV#UR1HTGz~ug>vLdKq}W-eN?uf7eRWmXc`h5wc;S;+?9lp(k^t zDv7$HnqbK&lFhGv9h)}0B+j@?Ht*7!*s*%i^dn8J{^mv zI`vC7jCgc_s=@HE4sUJny^B!Sv1s;kW1fJMR0eeVGr)-_x;vI zLL|FYnr?#9;$9?XrQG3yBM}c(crG$MvjgHCzsc3`ts^#A)-$gyOlH2-p*6RpAFZGj z)d4{Fp%8tngwVwiy?&F(P+hE*dhL;-S?|7a;`PUJ^au`>vUM2$5lBTgK?9JCS?fr@ z6Z*$3`>CnR7uzVjIu(nJZ?{gt!NbHqTHO!Ex7y@f6SveL<3zxS!Re&34)ABFoe(?Tk3%8>nRZq zM-~JVO30+Qz55bA@ir1&e4AGnVXOU$qd47d)20rKfHry`zoWmaFS3`S9r9ZD>ng-n zZj-~tNOxUhif}63$!khc;5Yn0^*C=h*8qz=UDjfjDm*EQURxD4xfS=S0oG^Cyd5|u zwJTzqF`RLqmUhJ4OOs2(r`0Qmc{0yss+HrWlU4s;MI5xm;f5hgZ zmZX-jtM;ru{9eTO%?Mh=uC&A@4%LtzYtSlPQm9MS?kw7(Z5$_BNV{-C1C}p^Qq=$P zkCF(mqSvv*huKVL^>QHUvrd?|zA<1{PG1gfPUdCEA~fB!42yA_d}$+|fO%wuE|Gwb z2JWfz@|KKHtd!0KVJsh2XnXD*rMp(Ial7^#pp-gRUi-;QZ*nXX$ z`{_m=MHcW`b#>={rGM#;Jo;(f%}&!BK#ef&-Fu%W=Oof)J|(yVlnk4sH*Y&b4pYiW z34Gq6C{&0uO#f^!$~Tote^N~Tb$9Y&o{S+*x@D_pGjH3c#4Jk*MCsUr1xY?_rlfoG z6WqFX#$D0ga@-~d6Yrq5#bChwh!i=W391R8u@p0)&}{?KKjn?sdSMf1sEghF#Kj7g z7aj|Jen9t3!CFr9_oGqkEsVEtJGJMZTA&z8kk@$=8y4EJk{>iD#Tu+PGbC109FHda zUc7oH;Na?Tr8!NXfOc_Qj>Dy=hN{}bCT`MDZ|(kJ!h-I7Ssar1PVXN~Zm{U4R9r_K zeJUNfeTo7coDFI0CsR9rS@gJfy0TQ!`1&yYTW)zjY;#4=+jicYbm%lOaeAHo=5wKA zaAZcsh085u)-GfacK=g)(Gsn0GnGTM`=k9==M5>xuV?%y(oj*%6shHFz`Yb4~QZ8ZZM{Cq8#3 ztYEM+XpwG_N>W%?E1dw2;rv6$623Wrsi^avi3Z_$tIpI5j7kfjIAo9F?S2HFpJ5tF zBoqN%SzmzwwTo0xqRX=F%}AH5m5Q8qHF8$6*U7)bu4hTmnD{8IKhvxCqy>thivnye zWyD@ebqucYh^Wx$PwfJ$0r45z<5yJzK9{d7<)`hCnZ{e(AK=>LU37FLfv8FyWeWgs zjIbibPp}vB6|-?l%Z~}OCSsUk5?0Zredi&luh0K&mWmh?^*_B7kM5E1c3Q1hN@2&rEjcX1Tf^r%yc*}g2-6^t2*wagodOsK7iE@k z?uTJ^1kF%nbP36%?2uwQVYTPZ;vs3^Z#b@3$MTeZj_`eL*8q=6dPiCGBnGz;-55~D zoz!%)%Yim?2YplSJb0(O2!trwglPCt*5bCK?81#KymE}RWzh@@G4XP!IJYVX8$tfT zhBX&k+bt~CF_=f^s3_UqTlYCWPOeX-7(ADTIkQ&r($-Cq7mGY>BO*PxBLz=5PBT+Q zcU5oafR){O=A+wLNTH{I7mm^JX`6qe5P;_z$osA*GDYge4fUFJa6HMGb69-#j@-^2 z2;tdymm@5aBLO35{1xE9*Ur4})T)#fPY%!8eTEp*?~iE%d@vLA;K+MS0cy8m<}WXq zcOG5~P(otsH zv4pJUR#+c0UY}l7!h#BV7~ofg2j!NVPwRm(s9Xd^)CebxggBRZ;{E}c^31vHL0BtR zJi5Wf*s+nwc!DVVC2HZoohNx+0#kS!i7TKkFfiVU!r)`&&wHA|8D+iiC!H$2GnJ-V z`>#p|Te9nysxI<$V;@LY8{9N1H)r=o->$DEhZamnM5U9Rjog{BYTi!@i&|o?Rk+C2 z(%iv}iLQOty8xaiqhi1+{SEDNFdK$zTQ4Z4J%r4 z?iGXR+p;K6*Wg^7{XXKXh2h57Wjx0Mm!Rp}%R#3k&CJuus$W?fn>fy7Y!FgPBvc-B zfTVlgJV4%ughTb!R#K5cv~Gd|MMN8ZMFIihI~MHyoL{c6;~4$2pCd%xR`1!V^h#Jp z>VY09rgKDAl3gQqmf*PIr7!%*MAx=4i9<9&B-}5%l`e8|C?wm^Y?-*$jI7#B z)+bfr+I5eyvtuVG*A$LA;K zOGA`DbfARwiap$#+UmF+8tf3@znra!Qs9^XkVK{`%&f58SNTh`1~Q)diBf?TNc)Qq zsaDJpnKjufJ;Y^AqmSf#-Xoe)>bjWV>BtkcSo&fY!9fx@=L%Xj;sjdlJ0t^>mzWj~ z^}TEg1FOr1s2#ens4Y~?rjMWp7n(9^c|jT5o4=XH8ZOsj;O(5Fj0(<$;6}{R#W*AV ztIuo}-WmG=#rWh?2sZpTfN=oW%mvT&wwM#IJe%<1y~HxxMPi$|n)uRku8O#}!5h+5 z_PMciEae>mD`7v1>xz%So;&ccCCVU*(7Vx+`wAzHB=x9=5n|k=BcmYcaY)&fIXlRd zzn_F>Uh#vUicjrjy1J~VjZ&+u&C&Yg_Z>pg@yvY0*6#*o-g{aU;Bj~F1Z<0O45#S_ z%H_?4>~Zj7RE$W8+^-4e&4LuU!d86?NqoDS>g3TR$NP^vX?FkoBP1S5jBv)|lq(zOZ8}M^`;C&9V1_Yl$kqFD7%2%~FV*Iza9x0ZY-* z1GJ>RuBw$#mZ$d1?b>U*O$DVwHz)UA++rZ}r=AB2?m!=^Ya@bQYG7Srr3l7!gv6I{ zlA^Z6ojLTN`U7{O?NC$KeX_nD*SItq8&Mo@`d#iCEIiA%hp8rw)jQR)m*Vl_b=>wE zzM}e#7g%We$M^!PQ5(g(SieWw`4^DDTfi;Eo_0)j4g(&Z%giDbgEqHg9Ri4>hykdD zc!^#ZB`#n;{AZH9FW+%_Mu%gJQPz+66Qe~>hw2~Y4{ zg)NoDia4Mj9QXR13GA_L19xuG^Ee)G*iiI7DH&0PqOgXjlg55DZ$O4Rnv3O*T6v)} zHW*^PuI0mX?_CheV!xz&7EcA1pLxARJDDE!Vu`=>wdqiI+8~Q&pZhJ3B*X?Yq>w0x zuH8Pc#(H*+?Z$1*0oOyrQPf;z(c+`;_kFQ3r~AEj01sola`y%Bxf$pH9`I1N>Jug^ zZT!sWD`*WoH0^MasXB6e8KLHnB^_*Jrds>;fs1A zkM@l#9vQzA*dXcX!!(5F%G$BM`nIEA!uR!wa$|sp+{v{UtLxyUPys8+U+Z_S#h=H0 zU&rnL;A}7PzikjY1ki6{hq|C^G1LxfY>p|Sdh+Ln9K-Xn*ZODCwwnchIwx$`3cup< z@6POX<-M%)et;EQ1GGtlt#7L72q+zYdiUkh z)PEL8B3f)C>{?`YiO-c{p|@qowx%7r`Yp8q!has#KGP$}*3?irNk~-C7DY>pMXu1V z@E%hi>UuYvrSA$yu>3I{ojpfBir*vHuLSX@GAMO)?RBym9GxuHn)WEctvqdGu${hY zX2a7729{S~ZZ&G8vlqp@sqRdEttk~+>v?T3`r5wMqIEY0F#fgLl39j?^B^h`05zg& z*RkjAc)w;5<3Ftleb*VWDYf0|(!SGCc|j(%O5S@>zcL({P=`kTKzMLCQbVQuCFe8w z9q~bOy2Hx0P;}y8Yh<4t8eSY{%G49GE25NG{s9t4te}Z7djimU+pxoTu$R9S(#O6- z`08W-m;3HP+T@FgI_*M7a#9tg(?3GGk3%gLQT8ZVEfQ|fe>HkPJFMBABg7SIX7YgR zx%PU)UCOCmMHmQJE9fP3(uFfGwi+XnVzJX2U}K4YKyj)$o5wS8KigUCjG@LuS{?NM zq^Y=nkIQcteiQ27@v^OY-R|s{N-R~d93!QTL^K=YR)Zf%IOiCNQC!7}#f!gmY0vvc zoD_JTE6v(rioKNdv`T>-W+4lkDu1mk_l_$P1I-K&pEtp zWZbLMc3h$6CDv!DL?v6Y`;~J(3UBsgI>3AH{=8gNz*|&~?F+`^WoXfg_2U_0sQWrR z9NMJigVBzv9d2U|q>9%^5v+;)ed2Eaq_+26@}94{msrQX<#8IelY@=2tLgnxna+Pa z#X*S=ghoB4+Dxx{@seBRDIlRYzRRJar=dQ0WPcpFy55|tY+;IHhfwgU$}JD;_gqti z9`C-2#n)FmzqZEifewmt5-VBidY!AMRYphM!W*NBaDj7p?2WX#=~Wf?GcjWn#_ldh zxq`0dqamAFqEW6e(^e$$jH-2gLbmqo>C$UJn zbhrJ3fnrv<*tCC|fh^Ea)c6^T5tD1%h@@8q?}+N-9*rf)A3<^DK$lTu?#Ia04^#1@ z;Wc~>W;l(d)UP$rz>he5Aw7>Up%lwMH~7EQ;y#g6Ij+;q`p!vMYxDfMmX^Z4kcber z2C)P(I)&Mr2N^*q)mL`AgJkjY|D@Kyrn)9gh%PcC-BSv^3#zxVy7m&W2+^V`lEqR?=Sc>{-bSQHxk6~E8{et6deJbU^FI}NlMJWEO zGkJ42gxTK11d~MdJnUzi0m^M#cU9YmXl8t-uvasbJxH-aGu-|!XaMRd3J`Csu5 zY!w@)U;r3(()HvmFpS6jtR-|W{y@WLHQplOhC34f@pt2yq1_24o*x!0$F&1Z6_qTw zT7DwxOF$!JU8z3I6Ad7(hXT{&MWulqNa+69+;dp-4Y)ABKxmI(g|%vJ&zgFQCNLHi z`Q8FWJlDD1RZ>Z9r#o9Q<`3B1N=kBpHR+bl=c{pTln2IaT_e|UsyajZooCmR;qa+k znKDOIWA!~)UF)Upund^?Z35gc-`}4q=#upoX`XToCs0tpeT{qPrSH&DieFNMC(fJ_ z4)OuB6y>->>a8)t9`Z@i0kW}5iGR2~92x12r`%7M>!q-xqHoMaFMuVywlf8rDMtu^ zMSYv_YGYium(8!gH)n-tn* z&YmrZLPV?|2HU#!gD-JQM3hLi7`^}{vNRxVpyxjhSaE*yCuc~-ihrH^Y&=oW zD;z9jOu<%kLqKimBE-M$z2k!P(SYC*IBH`VOv}#X)Oxq;n+%GoBRsGn~2gT;7!;-BGqlh0#qTY2 z(l^A(Y!)lJ=;`N}GYFZ%5`12|J>`pY-I)qjy@wtuI=JB3%>8}z-r1-HI=7pS^X62cxa`e&~>Oc~I^R^K#WuVfJzlmB#-;~G&XvuNk@|3&G;q;@?Ca z?o0q?DjE$p2-q<=Jdv6xkUT{M?!wE%e~#l@98E6b0!&`A{Z5}S)&1tv5q;18366Dw z&b+N_B2TQdz%1YCO>lmae}xrubQzyO90atF{()W1sQoaybQ6~VoyVbL#uXL^A>N&X zF=PR7L|vBK?-oeN)_DXqL*7=w3F;MFq}%axI}QCkfHjwhP0a~^Nz@=58l~h#J9E(P zXHkiCV^96MYbT_9=vtPVY9tbB9rT>X#CtAE~h}K6ir@ zW+K+4DhOsBL8;+_Hu4iisOR+~vL0?FV`AAx`_r>OCu0+vMDq zifx9F?wxCVuI0=^+LuD=N$X$Dm=ygc1upnPzZ zZ+3lP8{&O@@qx5K3=VjSk64VFZDftfl>SdXWplLe-c7RR`4`ReiUmW%B7|l3bp~M} zTt6&Eh}dLMvCRSUP#yLucm|z5)Mv2A2ZXZ2z{P#!?bv*YM1=RCGlzYQ!^~8VcZ6{0 zkixWLEU(fE5xN1${<~LlqShu^qSAz!>Z1X_K{6I^&TNO*(^+e0etj8ZLjbSG<0xOa zT%TZ=C`>|pxe@WUHDi9X&>GM(G-ZV-nuqCVz0enVd)-NH^- zkHfYT(bEclp?WTm$6+(|dGJMDV5kbVmc$0&Ttvqf-ye5}H2o7&AS6a&am}k0v?FLh zA1L6kc9EDL+w1zcT@vlJ0%=CeX<%A6Rr>@fXF?!-eC}HSEqknlU&D&LHpCRhXe6zE zaa7G9Ns2KbbA|q0!CcqxwgeKbHNVS1sDOTfA0m7F9qS##h+g79$-n_aG)Y(_#qt(p zaR?4O0;89?G;DK!IvuoIOd*XVML&M7IU^Iv8C_LmxIzk0x_hH250rg)z?CNxby4{aEIijKmZLKluE~8rrlhuGbW9uQY1LPSv?MwG% zjbFmhO0ZM%i1=*K9zg(XnNS!)N-r0jx24L#*OMjQILto65K*+6_A0$=^|%H^Gs*&h z2%gju?La4+0A0*g0iSn=c`1{E)1mK#8HY$hW2$146CF2HPr!=c2d!Y1GDBS*9VG@{ zt1*?;z49D4142`W;MP=U&7+ zfl=FDy2`B>l<*%Pi3oBi7ou%a3d-aB+Y zN&kB%l>WR3dV|ecmH+Mo3PWFFVE`J(nBvn#Z*TuNf9@q0u68J(0jXv zxCB445v`dOuRCW>CES^JxQWKGz25Q9zp|FQ&N=V?XmPBrg{b{5p8SO1>-|n>4Sj{h zjTvstOF=+IwJdwt9@#?TSkLFLHlv_VgAwrX<|v8Z@6P`AV){0Dd-<6M+*kWH9dkKg zyAl;eu<)mTTx~OA(ayPSC_dVKB1z|CzMevQRmulLx_Y>(BjIZ{qR3t`V*#^f`a;0; z#@K<0OG5av3}u*bm$#ih|Ghv=Aq1!EHrB@oC|M}_UJ0=iy9MiZ_vvxn$$g_|Wn5!z zGnG0O(JvLGkaAIXz{ilYN}Z>ttsh~1p^V;Z^)TcA7c~Eb}xUky532hQW82QFm zOujo3QamnN&s`y+X?-~!lsU#8k9-pg`Erq(5H}cst(PjvjS>2{6*`;&lI$g11#l^u z3ThQXD@z4w2kI#@&(;j|!>Y(WIE3RfR|ITzFcD&x*!(%1{e((Z?HfhZ#c1olD;H)1 zd%heX3UAa~0R{S?5i2BM)Zr@Nj>2v%tJzpdIzKQG89X3VjCH>hYT-ua)V3AT&@9Vv z6t|V3_3PGn>#}8Y=$zz^>=I{{kGg^9AP&{Xi=;SI9BiZ@%YQI8@}d+Tg<@zl=nFjy zqNqoh9VLjb;FX2_LVY;1gcUr5OweY6^BEC38rR zf#%`0Mn1sKyaR--hcrVcWG|`<(d*|=fN_~oz_f4M)-zF88>6N|C!u^R%gDw5VB0Y2 zFjNNo%j%fh&=DE6zm@q+I7~055v^#?xc4bLM6bS^eZl>lCjf;~znm$?r5Vk^yI&XN z{Vm?+;HeuD>G@Kv>T#q|B~}pvSeIOnMda>qrsDS|t}qcK*lQBH`2-slcjvTZ`^f4V zG6cqM$6|fu(`wQY8Zy`Q$TMuAF*M9O;H^P|<%BUvAun`{FPz9IEW5+Q8G<8FP1^qG zUmMb6q35_@iV3Z@|1b95GAzor>-&}vP#RP~8U&PXq`O2~>5iehTN(uE?h;VCyQHK` zItHYK0R|X)V0aI%^SQ3;yr290`hI)9xXtEl9CPfw_O;gkw^R-Ag@%nWp8Gztye^TR zBT05M__?9cUS zAf}ZoGW}MX#hy)c<#XWz6Nm5wc8w1!8%^9 zv|6X1R?@wA5>)2kN;EQP)FT}0=mBIphPKgIQsrF7PxdCmED)!uU8wPkg(I1#e2;B% zu-BYaCzwvRyU+`+rAmvH>!32^tHoJeBmc6Q@k}Rx+{W>Mim*4ERvq>9{c3>K2bXO( zsV6~Q@Ew^i#w6y_;Po9J3pixo&j1L2-Qm3cee3mluPmm`GHZkNjU1t2kjBRwOZDq= z)WQO>?I;oZ#ppx(ra;AS?5Tyik-0=V6C{49_FJ@>)JE$yVnQK$v@gvR7{G23%$}id z3j;(cTTNK9seWV9v)(E4t#Ti~{rS8-;oOt{*^TPDq;zT=p|TLihh=?=G?4o65bdh+ z@%}N&<~6*{>*-TheOfiQ8X7b!=U8~k2Z%srq5th;9nD!s=}qhiMRbDT2BUtsxjY%5r;4+vPz;V&^%)szDwnze$qPf==cHiEN5vvr_t)6Ts?@aV89Mgwxga9HCO3GgU)IO*HyP-7 z?~G?#-1{qANh|zOc~|@cW0LV5Ugks2nf+}&;x~mfyNndB-qecL&*v%U?1J@c>em}& z7-}J$9jC)g==zF|(02uo1%m2<;dILO&&Oxoq`@8(`e`dZsQE=eSZ;e% zNp!e3P9IL>xD!~6D8a+URu$0RatQM-820wCz zzM7LOu8i$-r=h1?b@jQE9b*+uhyPIwzH8iL>hJR9z;St2e;m3Kq5C5A5P%s(&4`Q$g zv9E-HkZey=?)o3Zr%3o1Pvm}Ls9MVWuQvhY@PMQa5Y?YC72W=$7BK|KO$~X?P5V!8 zr?Llp`Qhfd8sa~_@b?cA$SfcrF9soJ0s_<|#+*RT9nUR@@gLu?)DIaDXmE~Y^6`yj z@L(1nMPFx!4AK7Ma6SA+9y6*ApdF;2LfHu1Xy-icOA{oDjRq46hY|eq^WP&^=r%Rx z5MwTO$;4u{b8N+{rq3(1{DamO(D+azxv{*V)}v$jr&h-?1IoLh(+}4FAiSeOrh7$2 zT^Qn+T>p>5R10+2s}c2Y|Jc<@aex$v;K#@5e|+!|FcspNmA?YJ%0CbeBmf%0I}A;E zz$fy*g^I5f03l(mP;$mUd>0}11Q`F<|F5w?r~bc@{m<0;f7AHiYenq;$2I$~Kc-ur z0$M4|nM^=oy}>u5Q&(be-M4bSqPesAWNhq!zry^mOZO!lhuCbux3yH*4q^89-=t9yMzstfg?+0D zn6c|!p@MabHnA$~Ph~0d7`XRJs^7c+qU^k zj7qU3`IKdQMU5nfID+_axrvW~BA{UL8(67WeK02Sqvg9~hp$Y?UtfuZ=Inmj4crad z4c-mel}^e0)S9vddS)yBK055^)fd6?+UbIwW>I(U<;tccKYTvc`TxmR_&|g7W3l6{ zb8E}al>E-7a=_C%IsVjB`)LlYh{%oBM6#_A={x_0C8cjK_h1$MTH1LlpXsTP%U7^nWlndCpHJuVk*1otd|_!3Ep;0s8qc z)Qi49N)qM|H?_E&M-%81Pj+%L&vdQd0EbgwJ1%i{ADkUbz&2rpO6z^$G0#dG73OzF8gWfGIw&Z|8on*VVa2U^qN zC0TSFO}*J(4N)4#t^wX1kerISBOmcQO(v5k`}QEdEZk4&fj&@DgR@-{`-hjd@oj*!hQ0%G`V*ojiWdE z-nu>Ac2FXR)9w4U{eUa17f{$$LnCH7S|xgG~!S~1T94AZ;%la z0!M?F(Dut1FT}M)@IW3@W$W}o%U}+#|9}jK`V{D|rnNjgQo0mpUi7@HB&hpi*3AvX z&d7buO7rnSc`sb7#e{GoU`Mhbfq1CZ3Jq- zTr;Gv}|U8ouJ&~K1YKV(l6Dih|$h2J}I!MMp0 z8@+$ud!C7Z8lWE5OsaeBpn8<>m+kya39xB54y-}SCMJ{FsLIg@^fL!yh7QoU62sp> z9E!?$z$6ssk2-y^%{pxOS9=40V3gX(8?RV{*kl;Iwr8n(t|zlYVr>&H#w}<^+j85@ zMG*VKTu%FMx$fXrpo@vRAjx{2q6Tg2$ctk2aC=ZPt4&4@lY^0WMYDfGxTem3C!`7o z>T3(twbsL!K=JD`KloRk4EcUNa&sx*%8~&LLuqkYvZDcmV$L%hVE3CSGBJ673u^be zIY%E|kDo}h0xOQCL2`FSxo`b=wiRFlZB?kt4Gzmawk>Nh#o%IihLVEESaX%efA*;e zF&=`vcIlR=M|oIoRmR$7Cya~}!22jF0{x?I7)URKNK_UkAfSyTP3q)t@l1 z$f3AV>9GhsJ~>wKKn=L_hT7d+3J)+hc*28bW@7%<9xK$>9*P-nsfjZhrcaA+AAVPi z2TvtB+Pe82-OtU*`LGOSQTQmcRiP!ceWp^HawFXsH@z3p>7q&_!%c`mQzXCiw^>$7 zhn&>)I(+D_iFwLE6&rn<{_&2&WoMLwt^L^7k>X+DFxw9Dvkg>wAKywn#u8<76>d50 zgxN)Z!5w`9&w^A<|Lt7*z=AYvnAH1!jFoG`I9VCQ=y(6vo+>Qfy7IId>)4NqRXje= z-9&cReBTdAVTqc0S~xQQn{)uQ=hxFflM&;FAL6^z4_B+}hkF)V4I`MsA(|Q{`)0A- zBe!52&A8|B9y4QpnxG4FBngMH71!TZ&cBvWsVcBAtj~Uc1S4N2A=*u3yD3bJ|$<5Zg zw~ff31YZ6xvXPh{K5*riorlBNI@#4rl(1>cs+LR}g2uC`H5~j8YJ)Bmof%HW49sZa zqPnA@7HRY09Uu_mOubUEp)=;mEucjQZI$AKZm*o&-sg8J-_0nCauk9|>~_X7HNjN- zX9u#eZ;qB+z;R*!f=o!|388XZ%X_`m>{AuFTEC;B6rR_rTfbd7KX40+m&#cBiQ+XS zM}KV?=}@f5V;CSj`#$@$;o53Wx!4bWQQSLCKI%*^m@ci`G%ninbj@_1Pkq!loYK0q zsL^U>V8si*3W}G0@ok-q$wpCno$|hU5-Y&Q)A1@%`L^sqzn{s-Bw$a7=Xq8mJKC-5 z@Fb=kz?-EfDhahban6_IeI7V?4^L~kPRSf?T();m64)qeuM3QD+MdZpMuzje;EgHl)wRdee!+3LX-@y8h20e*xM9XjK1Rf?~Xc-`Ri0*FF@;wni;!b zPfm_7SB^!SGRgNPU)$(YKo#_7567xz7#45r{iPGEpD4FPcuaJ`c-`BfGj!3sM1PdA z>`bmrv4Mr`L4s-$2L*6S8BUv>qn-U78jzr(b|!h1J#rOG$Z|Z z))NL}>q-+3Hh79?1lyy#j7|+^ykPkbe_qpbLz*^wz}{;rdc6P`6lfqoXZ|e>ZoHrL zZHI9=XxA2S?#IV%7$tcvm1}d=?dJGoqEj}5nQH*^1$yB(l7&KE`R&O;v#BW8#b)dn zo9G0G55lRccswS?PzG6e{=n*D(S{PX-iwPS>jz{jMIRbI^F`S-+iZnY@hFSJx*XM~ z6C&_`|K%e~iWYO5S9=%s^@4!Hf$6(ell^M274bq84uY|aGk`qAE!l`NK z^7Xpk+l<*FG-~y7VaC>CwYYGV32?gF&@1PdQTT5e!)`Gr9tWt~&L<+FjPu&P(C3CB zG5RWwq*ROoA!cL$8XD{hTTc9+QlEwgkJ^qXB787NYj0?(*IFUVl@=b4DfDJp*XQPw z6?SX^0jFdTv~N>ch~iZQdtmDCjqrQJITR^rtTpN)y1#aZ*TP7T;mpFDpWi;D^9B35 z>Q$PGUwiG=;e(bu)G?Quk_6)!PUk*^DG03cK5YOWs5BR;mg2*y6R}B7KewEnDj?&V z(h9yCbDLkYbU$i>A=9A2`5e|(T=#!1W;xzQ?$oQ_gN~PoVdKPr;Pied$#-BoOFbVg z%64~adMwqelSYPPSLJ}mZ8z=Lv-t0mKIOzXtoKgS5p`b_CUOTl3~&8v$6Ru^Xic7F zIEdo@>x*2_%C*T%Xh7#gK`vj6;hzq90jIU+h!f{AmqQpn$LFDdnoIkd`O37rs71bc z9?RkLh!+_6<9Ix^ekj_zJ|@);tQ);wEB?BTiJhPVE;^y!?{dbU$tzHGz0=JxhoQjP z{BhEMdm*y@T*v7yE8Keq?X(Tr=@WE_sFqWJL8Dj=jBE4CMaZVqTZnk zTPpZyP8~Fs(=qA~KgykRc_eP~JESVe?6P6G-l{IG<#MfU3<#83>NHr}9kswalO+9Z z+T!&BHr;w^S>U$j2=CK3fm-pks$rCVjSJJRdn>bK0l#Z@zKXMD95ybmm6JTdgY52B z_r`KZ>1=FPwn4db7fSB|w>qDGt1C}gMQg?$xT{OM_D~IzEVA3WN^6d|{YIzhDAT@m z>N5j0jUOPcF|l?HGk4uS-SGGPu&#d!%+sYx_D#TXP&dX&E|7M#n!*Q2{*xC?qC_?t z5u-`p)NPwG0z#sc1P`$w>9u-S%VR3k-rLDNXsN&1>@+j^=Q^?uQw;sR9`HSxIW{2N zIC$h?tHf>mgxzg?Bq75>O1r2QTJ~5NsXbZqH@L*`>g9^LOpNcyXgaJ;(3f2pdJ8-7 zXTjpIjk)bARGZtr2~;?%o)1UMv@@={tQ3U4Y?hC$?PUqLSS;x|Rmv7zmbjtNHWlr@ zt{J4{F|`xQRzCCxFTv{$3G5Utd|2BKYcm&JXV$FheuHt*DLmu$0OU>!kWV!>n1U&k zV&7G-?ba|mO2PXmC#vgo;MiE4uQ>Lgrr9E;2FXIRDGT>*#JGoJhTQrOweVY7=PSk|KyO=L<)Hoyhv za=~LzP^0vffsz9J;D`O?>LoiBS2?L^DDthzgM!F zuk7H5jJoKh^uD1b%J#c>1aA`;ItrNV0j)O7W6z1+{*>GSl)l(In%XVl{EU*(A-*~1 z$K5cC&n+I99hZ6(c$5^iyH@~r0CoOukTL)(r{(B^e9(hIx2etK!fmiB<&6X?MYK|$ zBgISAKxF*hip`Z*CLczgvpR?<3m$Sq4h8sMlRo4`ZS;uV9!t6a2|lE)Nk>3N)rRwf zOJPeEFcSUSzq>=Ap!c|^YxYBQm;&LkUEWoBcemL93lxM#<{PhV8tV4GETv)$SM<&( zEP%al<#;kED5%S@^19gzeiBY;*T0`g#V>?2y%-Jc#B9nhAmeHMljT2PwF9JUC~HfH z#&$Ky#&Txb*M1{$9MHDXbqC^*F0iHp8j?4c9Y~2?Cyrw`c6T|J$<2Q))1rtE+V$e4+;-euymq3%hQD3gK{I^PZsH6;i(&Yy1&+=aE2R?@FWP^* z5XVzxWbSl=>ich-BkhVVKh^t1|NeIP7FG~|hNODu2KKwp-hcv$r(Bt~^^pLZMX+UpShrLogEskyNN%7HSO$lm{oBlEHN-1?tbe9Xy@d+%`9+3%L zG%faih&j4K0vhJ6&i*uwrEwIFfill=poMfPa(*cboeAIoCT)(L7o=t!?C=%EMjFiz z^<;Kn^#~i{RWv;2EI(5)-BF9}tas0mYp4)})vmaIrLBL(hs@_k$jQ!c1y8 zu|1V;+-Oc-^(@SBwL1s%cO(~M)qVRP3CYW__hAv6QN*^p63-?HJjYus!k=U5by@KY zp!5>?82H@rK>D4miY3kT{de^}LTJ$THvNJ$-|Z@82jBgK3(qB5{Fw}!(-EyHL0lHK zY>qEW;-jGH&A8bzK_xI*p4kcvH4fau69P!bAM5s`k(p;U%;^YT9U|6?(GcP>s}$xp(;6| z(H8hsm73u=PTDxd{Rf!|D$fyOqYLiKhrL<7y9RJ_u|7v}aqY;6jgVMJ)3Faif$GY_ zv*k4dFKh0VgiiXTCL+Zk*6W^!~hv-!XyLg&^OPu1R;ti`@M-Jwf$x(JRHsjl!f zM2CDkMvS;CIwf1_SeKejdB@_j!}1UTI6K$jrvXP^7U-;Jw+23kZljBh=9xxG#1#f5 zS?=E}9wG2NIoZ8`@-EwwmMdI3xCxe%(ngg7RJxtZ2)W~F1va!c6}i=g#yof!yuqeH zN-lr;Hu#*CngT63&3=w>OF2M*ec^GBks@SYS>bk>R`zkhm)w-hk{|vzRwX5_g2$LX z?1jq%04wRxUEAp0W3$WEd&{=Ma{IP9X4{lZ%8^jy!m?(yN21q7>T0d=Lf5|tcSIW% zo<4Tcx!!vzrJ2!;wu?^DF1nCq;|&~n=!KcppST@buVr9;iOQ%D{1V=VWFH2Vb$b7a zj!RU?l`K(~X{-qAfvIw8q`HzYzZ|+lAkl$vO(DzKk#(Ep@nxhPkI2c^X73$97fcIk zlMaa-&0Xw?X%*kdXm_T_Vpt<4wx4l`eQADkvFe1FYuujttUIoNF+OkL(NVF*?JD^I z=eCo!6myKN?!{#5G`gs*XiAU1o2Vhn9uZ-g($sfr5nu1!xxIB+r*=hL6NN3SOp!Y7 z?Csp)uHiP4SciTqwYhJYHzdU1w*90v0NXV|I7Y)JL$7TA;Z{;Wjwag-AO_OoSaH_3p82SJ1o^6AfGqhF`xE0FhCp|dEPLeM z1*iDxz-#?5YVNW_U<~U zB@g}x?Yyj9HVb@%B*oWI@w=1ypuBy0c%#4~$4?+DcAR;s)y+|)ZTaenu=g2;iD`X( zFRJ>9C!pu;9cx!6eiApy%uT844R}&f+TU9^M?(~z97QRY6w5^IHjZ|U!MlR|bDF(ifDSga3%K);fUf=E;|0q(bv1K!qi#?7GA$%YAMsZ{hm{lg zKwkj0LEyWpq(r?Wy~C}M8LhMR$T5GR1qL`=k4*}wWB{(? z%P@m{w|*ktNg|S#S0}Ia z6&C3d(;3&AeZKSCTMAosN}DUOwe#g#04j@gQ98Sm$#JER&o2z7I6M5TXQ-f4)9fgV#*jJ=bi!-AA^N*^se@-{=~no#MzmXH}u| z*m+?}3MD2_fH;APc(-ED1CAFvP`gn)kn^=Uj;MiQE2kVaKY&S;DE+u0((JRDEKU?I z(ax#EGQD)m&ER6*sPZbY)9zSJt9&*naQk<${MxWFxu81^W4Th2=k$QA=l)*F=+nl> zzJQ|&TO_RBy4dIKw-5aAjd04*H&%?L&3%HjN{{@ePu~9=#8BERciCmN=PecM1h|Dg zP&+%iNe4hcFmSoyuAr?<2-fTF(yrrd>V_}FVIzxAL+hZ2*L}~4S42d(Qd-_s=t7GD z>@G$4(}P{!yw&eztXit>%WZo}*Q+5b4l*pB2Q@Od*iF6(yB+MaZ83?q=0l6qR>-}z zF`u6Ygc2ts0P-C7W@VSxrNY3M_UBGF8#G5k=Kup2paa*@?!V^(Xs{Ul=xlG=M(>k- zax8*G8O(~7J3`58`kTC9vMEOC0Tnu0)SCqXh*dCmmWK&CCdT)Sq__8>-c>A7}4KC5PQuP9_Ff1c%UwQX(HEH#l@&52UCIeDwsSBw}ID zY~=^D#4=lVJj3L;*wq)GH%-U*fD-t^ZmC|LJNMZKedFYJ{VOe=X56oDBPrw}x25xv z?X2Ll8hI8I>E@x)x6;91Q!EV$s5jdvU+RQyeyHWlX7^%B&E{V}Fbeq$x6^xk1kJ0p zVj7nUE}o?Fe=hs=2kYnRBN2oi zgsoC}FFuOy96J``OsTH8?&LV5ABZcx)>-Z+oe3f|cr>MoTKjeAHP<+v0jHOCeFFuD zKOO(R-THKdQdb@Aud5vy=Hpc9-4d@HEdPGTq3ax`sl%x2G5)Wl{DWT?S6=WY)T2{+ zQ(>r$_~~2muKE~YFvIh|h$f6=S*TSW{;sSt6KNR_Ji_k$Im1e$wX+1Sr~=OMYpEhf ziGc`whsRJ$sTQ`*E1|d8I!5!l7Q!CyYKk9*Wu+~2%_kaXJK6ymn$!#|A*Wc?y=8rm zyM{M314wX6qwQHQ-aoq?;&b?1#CI}IoQ!wxQs@JGnV?4!gooXaf6-Ul z>RRi6B-C1Ys^N)zYkB{|kU4-lxRTOP@tV^KO@Xz>HIDuW5~{=B_Vsw3+)*r`Zltm; zgD)Hx7nzvTmKq6*BMhZuJqR}HG)O^-y6L)GpT<`NN@dl5e|gj*iWkM4kjsZQXo4Yb zyZ2SSpFAwoZLUJsgOBumSLT#feyq1y^IMeAXm{8h?w37hDk2-2?R9N`=Gn>`=})+V zCUUJrl*!x-ISDHT9Q})>5em^^u6uCpgPM1-#|-7KrHn+!Jw|Xe>DeU%|I7t?yoysh zdPQ9EX-YE-h-n+vtpX7aoDtV*1M!OBjZG!d%@0kj2Q&R0yWwz4AP;6dJscpDS1&7$ zXLH)B?D=w<3F?hj_cgl;#9M11{dg7m@dh3%b8qx+?fVgEFz?PLp8(rYx(|vfL<; zDw^SoUabkw`sYyqz$ZHx6Y)Ncga*&Zok?*euO;fsUfD%hBX7!Gqq_)@aLq^ z8*1d|a)5{Gu|blW*|&-cSH)_lr=1PYtCSk^QmZPa%@rR4q+4ye!xvGv#_>tuTdf)w z$$rOy*e@)yn6^o#NM2E<{&cfdR|AWDt=J}bJ)KvZy`d^%qgXc}7*g}*xS*{l;;{F# z({81Huf`T@;|RJquYz&qCyF}^zY}vw8>GQ#Z|TuA@RH&|^q;qThRW$=W;0*L6O?lz zoz}5ppCV3cl(Jv0g7;oXm+emslmc zL%^~E&`y^Nr?0mJNcBQVNHFrBM~QH?T*OvZ{kqH@VW^?PU$^Ioz9iBA*x?Mteyi26H`y zv_jGXEe3#qMDDWG%u9yjNlDyL8X16cKW5){2nx<|u*Sm!;KC+oIAoTrEn5VHcI-o8 zUp1=?cG_X%m_nR*MAcvT))r#e=lIeX#7en_vt`Mgq^L}&)YiSG2E7Z<4;H!b)bQ9J zQ1F6kGjc9Jo-m-Uz3}%%P8|oZzo0d*UGkXOm8ocq50_8oA&vv%&L^9t`pWHt_~-GF+TAc|F|-$$GflYgg10$U6bCs z1RoMiubWo`$9ksQZW{wFg{={aBZ1G>XdN1A+mu^KcP4AZS2omk({KSgv@l5H1|@pn z_HJ(Em#JYxG9L;e)`Qesby-`>8kHT{Rq1brD~(GshrySml?wIjM-Hj1Ye*21@!3Vr z5!PK0aR1Z?X-lrlJzXc3Y%V6fa5-iQ{=tneAh4)K?_|4oL5}tN3#q&7&oW8QrzFw- z>*x}%{DPhe6jsNLn{IIkG7cBQHthlWoWRZ&;r+h*x6(a}Tz`Tci*xQ1>4G0x%}(i{ z+uiVZlQjyM`*10PKku|YQ-B{eEP0$^oS(%eoZDSuV4XTRST(i)ewXr9SCyyze&yP> z=K96ls<+Yc`%X~!X*O?#_(2hvFVF(kV3Kog0$nvZ>M%d_nZN8Kv1(gl4~ln)7IXa( zIT#M@!90x6T5a>->py^$7}~)ih?VlFpb^A-s85?l?I6YWUJcnE9XlR>Ci8cInm;4n zHxHE!0P?oq4?B^1+Bj_DNtL-mX1P=>hH^qp+u4trJgz8)Oga`EClfW!;Y9ra=gFi zO0g>=QUEbE3=l(Syy_q(daFW#nRTpEo<@yNi~HweyA_Oaysb6|9qDGo*q{BUkUL+8 zh90=yVVOEtU#;d|Gek-J~yEw|S2` z2af`<<_RJaRpou*3pn4dZebIlQH_h^b8UerLv#orM>35qx$Z?t@2N z)>Nu(>%L%TykjC8m@L1qt8ngA%`bKkq+L0l=i+1>EQl)p{LxTbbE242H0Lv$SbQ#} z-J=?agPfnp?#`a7>VX$NyBm+PQOi+AnH~GY5auf%Fv?}RB(7ke@9EGWW;w0aZ?21% ziWw`!MQ(v#IGIabMa(i^wDN`Fa#eMRaAWL~2`7_xsCvo7`Y&Tq+&T}_V`SipT!}T0 z;x-Qdu=czMF7=j`fE%qxIY*Hp%)V3m=*W~6J3wc&&fuU!ceEqCImRN7YfW~_VT4dw z#25@F4Dxsu=xUYT|9Nqq+~FpAPaJZ#C1+&b>DIgE+3S0I!tiDlIiL~wmUJ$7r!aP) znAu}5+56>l4j?k5u7lsZjWQKvmqiU)aT>HcDk=Ub@n|B_HsFDHI-I`#MSR_Ncs2q< zsqdKY2N!Zp{xcaB<=-|&gA_itpCdLXadD#ncq4k$5#YC9C*7#>fMKENzM%;lAv8gg zkje*elq$}%O@=$5rF}u7?YEdn^Ndp4^(xEG9pq!ZY^$o1M_d;7miGR)oGz0}vY9zd zyl6U^>~2{kSAEGlQS#XK-PiO=Ilq^j8>I<6w4XLE0L9cyh+kF#jNzAIH@?sg4{RDq z`sI}Y702X7An5mHX5#Ibt!4O0VbqU>SP@PjIzfE&E=r^A>&wj#^}64)`M1_3$9U%T z$!&Yt(_|@4gWoRZBc;}|f^*T2{7grGIlR>Ca;UA4C_$Xpvv)Lek=u?Sg$<%VZ!#C; zj^Kk-HFW}Xk+9Df-OKh5KC4>ehaKgFA=bZEV+8GBV$`Z; z!YEjH1hJ6L?GVAo#L#*aZ%}mE$n1B0lzt9Mv`tu51i2?3+;R2}-X zD2~kBqNv5eIz`ULf+VEfESn?i_s{O~D^@j+I-nH%eP{U&=0~Nre1Q>Niuc}j;)N^( zIezvL!VMr)Z0`yxEs1l>P_3dALN6e`O=3$?O;uBr?Lt~ZoEyy zM)P#2{F#&D{r9KmI@#RN(Q`gM2wBNT8;wj#M@@NXZ>;MLG2|zZB(3D(>}44mD{@gS zbiWeSsfXb$?go@}c>9E{p^`|ZORGq#_2j4!Uar6F`ihR_<=gh{V%_Zd3(@e5sH=f5 z;Iqdlxt{w@C9@yuUa**@?^RWZx666p5^jJ++p2C>+U7TQz-Q@*#H;{8Wcp{Pip=11 zt%+5w3E_^o^tVnyFo$?Fhy!~CoA<>5-I6!7B-j7UEmr+AP;`fJy?{BzoF`83&l+{D z$ArD?HN}ihjdAeGoO7>CbeuFM5&w)bACaysdHV7@v%XRb^PDI4mi*U6iA4^%?NrfR zQ@!_=kA5_MpPaS)RKV`Vb|A#7IUoq_*hCE+=Lpb!9Z9_|TJKuQG8dADkR~*?-kbg! z1rt8KAH8>*$PvB+w_fP9eOb6QU!r-X#=l=$xCMmATu#0Az|MyP1bPyRm^me{)?}AC zgQ;ZX$tn_b83Lmd>e%LZY<{znC&(v0S&W)3uVYCkA{ z7~NWHdrpvkC41DL=nzKJIcdjeJ0*Baq{prjCUP6+5KqEY93^7=%^`?;Cr zdO+8#@8`zO(U$$pM~h-u^~jQiqKxM}xyN!-r^T3X^$%245%yK=v)>;xy}B%PkWf>P z(A`pPsybOco%7c`_FX@5xu!JYq+8txA9*BUn~|A0n;z`yRo`DMbMr%QS1|qW;{AHA zQ4}x3FSDc4AfursKy^EFM7x)gQQ^lC4d>x+C9--=M^$KCoj}DJUzn9m45YG)+VJR$ zFIQN4o}VbR6Gq`Rs+YFkUxt=pD>e{-zpBL3#4m9x8g-^4D*;67#fuTLS#1g4dj8xV z_2(OgjOmpkKrxv%wg_WUL3nwcV!*Wd&e3nS@gkKG>!!+{V9KBKeELld9?)Jz_v4jH zj+gJJHYdxq>r8OX^n6d~PQ&*qjb(4r%yoYM>3y+^zRV(CQo%7;;8oX6>EXWiLuh$j z(Ril;W|^==`u>p;ii|gf5w!^S&wuVWU0aGC(7k@*z8?@{rY4$%Z>hArC|O$np^m8*XmNecl{ntDbhQZC7LQl z2rr`U*`C|O1Rn<`ct`X;&(=<+Lvl&DFr7DpX7D!e)jn?C>IHW-#Lz@rBk{=+RoL9O zLIF!ENnIQk{fv3W;h2lOp5Jr|r#crbS02wW?hjA)&#_zxB~kDIn0tir?z~YLV(jj; zj4exEz&ZwJXsyvsP!My%i7}5hsvU6ykzNw$cv2+Y7AM7sJ;pvS@J_AhkyFujuU~Pb zM1y77H{i}1o;j9YNn5&503KS;lV)+DWyuNGVXW4 z=JCO2HP79Y|6=kdQA6mrt4u@DF%?2KH0QN%vh1rW)g!2n@)|V850{J2!@{#7BU$j-?4*<+>TrU{yE(#kN_nw z2t$$GZIU*G)VoXw8=|EjVve)h?K1t*hTbkyDuXjiJ6JLfpn&%+`TdMqUrGl3n__xJ zq27A$=EFw8EK$&nDKlSn-Gsc^ceu{iPpV=u5RdW|-x(Gp3h${RqE;Yq6JlvZpHdJ| znoGhlO!Bzc`#k#7cW#H!Mq(17D@5s-tDvk9tUNJ5l91*mRiFX#wgLngKTdluOhma; zhw|E>$&kv9HLJryxLp*hPv3^@FrYSOX_QX78x?^_f6h@r(9wOpy4RkA-G5^UPd8rk zMq7!Wg{h%97UuYgurvI0@G4xta%h{cI%kE~fplF%NbX-*?^hB<-Hq!10?Hb38pcV$ z7VGaX3kuAmQk>(ggg+!DV2Ci}IM&YiDRvQ-WM$3MjRc4StlT%sqC9%suZ`QJ_jX&O z5zE}`Su2l9l$dmR@BLyt?=O{`mR~5!Ew#XqE-Eb7qP5A|^!dg+C|2)jCt`e+ z5T-c*C-)Ekz0+_~Mx z;pF?}?;N^nH|O>0xerTo94?gD6 zH+@^H812W*!!i}%YNfey16nslh0t|lSxnuDB{!Q<@=7XI(^Er;9XqFP3{ur_$6P$oW{X+SruoF!+@ZHYN`&RNbUUBC_0= z`#Lzc^Zf~*viQvAqAc7F(#5niU7Ar40>7DFa(h!6eJP_^I@6psL86I=`b5fR8 zgE00UA-!TmBpQ9uX3v*jx?9P*JJ-^*RA0JeqBFGz&JJP?Ik;HbEnsaMyY(xxO<+VP zI-gOkJ!^-twZ^t8#v>3{K==_Ns>t^vW~Uu@t@uZ!WS7rUe}(C@C*~umnD>{B)sR?o zk*a{rHE-D?hw(v5FXognS3B*FKX-o$W8bT^Te5JTwaTFU@Sc%%G13EyDBw5A%aUuG zBO1{S&0@*;r%_jP9b09VHa^-(FWE^Ex>*A%i(S}uH?eUm$m&!Hdk4?}(f+>vem!0P zG0eV+9#0s4P?zm(2$ZiQ0m>jJlA4}wbwfxk!rtC07frnn^4oNTMYYSHTx7pF#kpOZtSrusmT+QKaGE+tyS4R1Ye4?X+$~q?02>MM@lxV9Op7*%xC;k&TO{M zGd}yXOUK1EgQ~NC^L<*%bK_m_jgQ|nX5NKOVC#R0M!Y7{qYKyD*No#f>_IU zs;St|`zFuBY}`uWT54a_2Z8_G?5eoo6V9Lt3m|WgaX{+8n>ivb)2!l^w}N?Wz`)*z z-@l1tHATg67D%^!W-(}AAlZAhBH$YG9NWaQW+XI|Z>=O80!oM$kML`M~WQ7 z5UEzn(ntbNdNUMjB0p7b z{F}Fj?GIcq++8VC#o7b+De~8kVu{$Ih(1%LPE^5zS{>V!c2(}{67uJAY(KrzL*kyH z;)y*96hpEz&o^%!l6>*)*_l_7`Jn|bhA|jYw{u_PQ5V3Jmsvu8^{b%btkwN+zg=nT z9@T@3IwJn3>`b;@iFBA?%oBBm*dyUBro<$ti>yWZgiJvRX>QVhJCe(xurV96>F6g@ z%oOLG!UAUm`h0Y?pKz}Et~#le2ow&9yjHR2)j=m2L71hg&2_ zY^e22z0e)O%573Ef;bUwS@Ew=pPM(xKl&CZs_XiT?B+@rI|mp>h3$tg1zB&{>w;UyzLq$W0Qj|K}L%Py4 z4%&5AaKmoP+-s>l4LtdtHSI>&AB2IkszD`|V73M7yQoj@=95x=v7#rZfGh5RYhn&#lMyv zh9AC)+vspwsmS-JS_8@|hzeG47TcJ7zY~<{`0`SV7%`4_eK6=zSXs!9Y3)IvWAaRf z(J?0s`{fWK|d0e?#j^=cD5BbvvvOY#a{j4k&5bh?N=WG@r`Fhvqhvxr3IYoHK3FYOpMXN5L25hx0mDYeCv6|yU^=`={bi{(}Ud)8n{ zFeP}@-1|)#<+%ky-wL)00 ze@C%pslK;W!nzao1F(z&w8_9VrC+*YDgAzwYr-{%sy7^$hj;Rn z&Db4#l$?i(EKdVYZ-(yaW+&fO+uG$pE@XQQ_FHS}=N%Lo-{Uej_H7Fd$yi`S%yE_a zI0=c$0z5mdZkBWNot|?Ib0kYoH`_Zk2ZaRWM+IdbV^Dp1gbe)iX#{yw)Yq_BDU*gy znEBg$ms6to>mikJ!*Hf~-)A=Sl{}L3eC|4+U!|(e@v{=MyP}FN{#L1m*kTqLFY0=F zumr2zCAE$GC#nF+4i>rLGD-Y*Kr((|nnS2UrQOMe)s^vy!LY#A`F;~VGSeH*A0P$E z1of}t?o-SZLmW$&1UBn?*^(yw{dGHYFIvi%>x<>w=Dr)qdjEgyy>(QUO&2#R2nqrc z3W!olsdNhn+<<^|Ntcp0-6@TPf^A|c(~-OVj|pTXzR7xg{stn=5mzO~+e zSip77Tr+$2-m_=NZ|qj6nu(p#Sw2eKu6Sr`?kra`zZPnH3~R2V2N!GFXGX~gsFXlg*oCd> zS+7{Eye;PpTtQ3!=IMd(5VPEI{E01>TxU2LKgeqZpaJjcNfIsrc^7@YT6%K9&`D7TufHl z+MP*I^9@Ca#Ps*MfXL1`&Nkan>E%*(jaz4AVthbTfqe=5Fb6`Xxw1ubR+Cm&wGYGp zF%D8rpjLY-C?;gDcJc@xn$^`APgDv5dri4ygzU!~KiZSpBAltG)#dNoMVwFeP66QB zi@k0c8Aa!rS){d?hWX@CZ3KaKQw>eD@4FwP`oT6>HE!qey;JgK6yE2P$v zA3hB+HK-Ag&NY;$()XskqF(3({0a6R%b$jS^!#JU?w?WDCu>xBj`r(ndqh~Mu1$#C z&~H`)B!%z+;@!Y`?3J)aG6L!)xKg5)yI-Q=(FJJ-tWXodwv=wL4(t9qZAc;%u}@zz zGQakipUrf9n&x9lXwYym)VuvXpx!ea(jaV+jN7j1uEt=iA}4X(Eo|2;fK0laNF412 zPR$nxycW^Z>yxGVl1KKBC|p5ILdK)lAiw1rOIp0tZmS3Fsjk@Pt*u)Qg8egk7!r!k zC;cdE))kV8Hu@Shm(vyDqvN&wSl(~PJ{91IZ`sW^(P1@~I;L-n@^g6}X4S8AxVGI6 zMf-A%q4%*uSV|y|&`s|7?qr3pr$6|}3yU;J1|_wZ$L}0_b2Kg6iFfAp(W_a^S~npf zA&ixmOlIY52pB2`VTZ+U&V+xgTcR~8bj^>0(v5L0KdP3Fd_h(&Eu+<|r%KT*6Y4?m z&v4DaaP8-!txH!)=4zdfIbA=1l2UTHbct|QqrnzT-t+0FkE6O`SX!;4B?t4W68hU) zf*w;EK-1nCkl&D~EOsa2|Mz1k(9fM4Tt&KT5wt}vb3)2hj^Q~<75ou2%Ip68TDai$ zX|n54F+ALrFjBWyGCy|Fr?*LiX6~CZDdZwx)U+iAdimkC<|wD2K3pFH+s{#V?;V?Vsb z8}m8hU#0o4D*ExA;0swfoRRyz1z3DvG9ZB#)(n?8|0hfM(tv%b2s7cq&$<6QVmAmO zDprecif2RmUtfagfbN2wzu*4Z$q(7T_%Zw34I)HHgwFOKvHoY+2uKgmKzHk3GJf{| zYv5SIK+XIsE}`J5HHhifQdnAAKAT!&{=4@OO;Ro{)o)>s$R0g< z)LA^C?Yt$FB+vn4Z3nlS8kvDH(wU`fuj{c)ne~}jpQZE2VPdqEjdbx$K*$V+0EC^|}8_^l_ zDWQ`V-~%Pp3yb;J0@cgAtFRhz9ONoh^bR;?qPy$dE||J?QX|sjY;BYPR0)L$_vJBQ zk#;8wqAi^PoIPTOICd1$AkBIgsdX47oKU+^U zZS3Ci!8@y6hI!)v4%lsWdUv@I$}@WN)z*xbCX9PJoz)dg>rtzPw%m7%vcK7f|d_*R#h;-Y0?6S8WUJ&pGjL>`0?=i`U=e{<;$9T3dP zK&-Lf2=;^MJb(vyZfN|@GT;fM`a;HFWQ~WvG}+HZa+3nA7U+Th^Y35u#M-)$zB80U z=f56J-vVkgWg7W1tNr+9W7QRRaP;LjCi&~pccACL|8F|~_pMG)ba?o}D%<8*d6%N5 zq-4PIYv=!>_;=BC^!3G}m~_c0D84$Bz0TR1cv#Y6E)dN5m!?WoLXz$4>r+<`E~_ud zWh$W%k_;$o8D3lE{D}TbE&WtbvvQ=gv>hdl4~>W&Xqie_uhBBi?%5a0|IL$0SJcIx zHl`U~7T_Ox&Rh|2cvXBqkf}df5d&qqjTM=7Ss(uRaJ@PZq)kWJdw;e-v4035{UV^2 zb#Vs&eP(t*w|-g^G5O;gD5TR#SKQGtEi&)^*#c=C7syc_Rq5YC=>uf%f0OzDy)tBX z$8QHapCl^>>0VkKUrdB0Sk969kK+iA1TaMI2P&n#8{1oC;qy03H)(m@MSW>8am4G7 zxYz8g=%!(&9vf&eepe9F#C|lg%l76#unPG85;>O%@tM8nFNO+wh&Xyzz_sf*a(b%j zJYnZzseP#c4|zSkyL4c1n_+4EXl%Z*O8?uC96MGwO3)0(FV2Eg-+*2baIIV2Qaa;a z)AVrNI(BU`0^=c#N0Oa0(4KWr4cHox)i~MZIlX`Z7cg)(h*HMpl zY3#Eir0rXSlC)*5J2jQfeGx^pfE^)92r4>n(pupSV*-Iej-^Grt`ID;I#jv;&}deQ zv~f$}%sIQx>cZH9BwrYuv%$wxAISvC2LE#`LO{Y%hqw-{ZmC`vra{jUt!dxRLJb;^ zLs9p{6+_dQ?34*cYExuymlmM>njx@{=@U~zp;}TMZ~5#(W?}m8QAn{awu*-stoq(O z5C0MwX(!O}BOar{H@`^y(?(+iGV6;S^z|>9O}yA)ZYQuB{bhl<-vOf8h|)6q-@XG5 z!d=mq-1I0<08>p@M>fzdRCMt=SaM)SK8?!XDVM_RMgOP~b0hjR|<-f0OzDxiX{!Un(;>{p({ZtQLTyIY^{VG6F#2cty3vw1VSRZ;GlN z0mDnT64QZiG2znGfAShJI5=*ju&I^aZTxh2y!MXU?Nzz(RjNAi_<1B5A#_*om)>48 z1iv#8>3yVS71?$l5ml|*cGUjTRUgL^$N2vg57OWZ;os}&-6Y?Uoi$LM{rYC((jn(! z^?l6%``+H$K{b89bAYdjw}wYJQ66aK+%PtK;)c(Nnxj%Gdbl}3eXu$Z%B1Uy+w`XO z^KYCE2?9b)iZ%HFi>AQk%<&>1mu9)}&9apu^0}j<<2H@t^lXnXNw^g6*^3Bz%}pm4 za5s2cj9G1znFi%2>TQr1+6w&W6a)kWyTFh7NtNTbjUO_8ajdQq2pPXuVl+fIQuDgJ zI5hmR{NopXUM6E@9D9l%rTJv#{NQ0k9G0p61k6n%-X$e073)Q9h&MyZYCnPGiG@%p z9@G!D0G`+lR=p$ia->L?$zt|drfPW}J9VPMWnJb=jWAl%JTUY8{C&m2plTXe#?Cq5 z^nYaT16v=Ni-CjW_s1Z2S#KLN{(D{7S~QPyCB3i3s6j?6`;-$I2tI@YxRS9>kQDeE zmE6P-zq-C2OT);BZp60-usQQ#a+1MpCZl1)ogko*unQ8gIo5_YPzOxr!R6f+mElmd z5OCiY*HoEgd~8g8sngM}<&-l_1WY~F!Ei9=f_i^wIN-)3C1c}_2;hULs-oXzc|Pe3 zQx_2td7G5Jj6YHBP<#sfU6RO^+)1LPh6D2e-UHdA% zCkcjm{uuYz-b58cLs_GiLny@5vh$%vfe97tCZpuk3OPZsv9S-#!YH-n1&DILUNd+n z`<9Ydy}~kW&sXovR=sfDGnec;$DlJ>ULfo;Ri#fa_Cpt|K`0W-bm9>?4*;D|E;o&X zGR%Q1iu}F_RlR{`2PkSf%P+P^%~gFLoRI~nchB_0g4^5M-6~m0n&9U!Ui}iZlLm*S zII4kcMY~CG4~CazBew99{L-}+5CWU7CgySa1YGD}DxTK?OqW;#PA%^UP(-6qa#NKB zCInyw=HF50nID#G*11@Aax4u8?s4abIL0#CBB&K512|V7RveBRrGeT+?Cd_BtS)X?EOZjkP`v@8FfPOQuE3h;Vf#~l$CzU~0U9ty6Vzh=#T0nvj_{8Z^8 zj5U?Tl9=4L-;*YWKeaDSZ#a;(5Hkd~s;cGz@q(eF<%>vAJRC$Q!I|$Jd>tWw3KUu| zoq@O*zU6+HTa@SIIJm}45=#K4Ki?9(3$FX&$i?b^q3Q*dUH}0m+e$wq+Et`SB{u$e ztN>Xor#+grjeuW4Rf)y0zr}zxMcuumO4PiE)JMf=WiU79>G5)kWVS!3vJ&yWN%T05 z%g)y0!9~SZv3f(>80;U(Q7)&A_*-^%5i=?3P>uNv@Via#jXLvHc|nJK0X#{yG&u;k zxGsQu4=omJmXrNr@lb8~Bq*~mYYKP|`sH+YX1`iq#J_TJtcf?uk^=QQqB+#Xt)s7T z84uw&lVmPGr{t`|+jz(5a%#UaQB`ghz1L6&=Kv*Pu`lbv4JL647>Fb2%ZhbYv zk;Q^lz0ev*biJ(QXH*Go-zYWm#>04eyZsJ%cn3 zJsMr_at^R=KoVi!N;Evhc%-OPux>GDn?AaE7DPQPDM7`o@;M3aacP`Vo5HLjDFyl{{r85?=msRxgA=G`?>AU9w0nC&N_ zu=}5gPbTxIMxBcfwtt2Oe++tuA}GsFdxGZt3#(xf#Mw+3Vs6FVAFhVzSgxUpu?)TM zh>(TfayJo0F@jcg@dOG#gTZQU9`J|i8{*^CpbTxIQOK|;3g>Io#5P8b9`jeJ3@5*Z zl&0|m?usMCwRMzNfR!SUGMenbRRX0(?a0Q?8y=@PR*&bfO1Ivc zoQN7A_VEC`m$L_D>NRgB9d$-2?+77*93pQ3~J}^A=(x}8wc?&n}3*@oJ(ts zgh>?tf_||y+ygIWUJh1Mz`^MO-%oM&Y!G^4s zCjQ|aD6Z{U!+}sSW`oTztZ*|X*GK<1CgyeT-`}@ZEEiRMIjlh72xFrXR@GUzu_1a9 zQH^-FzWcuTL9)s%hgImE=xSM2q~SeLn1+vOsi2kNv(>@Gy^-pB!IhfB$JLI5bac_z z`8^p%9|JhPJS)0Y6U9Jih|{Raftl{B3}oIhhn%=;Z*xG|qQ+W@H>w6@|Ldyio76^; z<+VwR<$deNUx==rRK)T27soZ53Vd|1R{1a5x&hHKS2|t%XY^p)Fc_Ps5YjN#M2PZ< zE>Xa1mEog`HBtcBltjF4;O^!MR~Zf`les*5%=y5F2)og>;(43>PL#&JQ64%iTg8$O ziOVvQ0TVN`U}%Jp>J3q?tXwK%MKmrgU$&iIIw}=~e2onxPZrV_-2yNHzjHF3wa3eM zmQQW(Z4{JncGL}bon;)=-7LjCPdKRyu$ektoBb`2&=?79-|w8n#_zGapAtmhkyU*) z?hKo_FFXtF{}{|-IzGG;&1qx!jJ*ajS*Y#99C)+hZr*KSI(#n2w>L33b-&%#f1)rE z&)oHtIej@bfO2+9i^84+>}Z=O~U~2XQ%$CW4*@sXcQ~mQ8x0;eP|RHG!oCYqwia1V}G^7jRRnt(7Way}F)V6w3!L)auy+PmyKtaNl%_SlrHa3^9p!v$gJ*cbGeLWzeEQfv z@UocgfKVbCPry}GZ1V0>BgMj8d=H-8nt}VOm|W`&8EFjwCxT|KIWQ7fCr=WL){uzQ z1?cN-N^mfgpo+L-w4W7vs~xOPWG}XPu>Y3J`DlE2Ex(SHZQF=R6enn<->4!KK==p% z)Or{X7c?k_+h$f=Y&4!o(d@I}gn|J*N^*|_wm(Sr$jpcjDb#M}M7XZ%Zo72nseM8I zS8oip;jU6He~jBT$E(@9!`gvIyTQv(#Ukl!gi?CwSX5Z9za#P`k0k6155$}5LkfG{*jkd%~0GwXp zUhy0tj%swTu}Bb(X39~)zs3zkW6q|AO5)UXg9BJX8ZbsX_qn#J;|v%R zZmugxZ<4erBtI${vjT^ZZ@`{{}}SyW^H^ zw33N)0F%bf1CJ9uk4Vy{!2>zD0i70HSsJy!IWgw20Vgp@OA>hkK(J^DSPjGWx~~M= z4k(vEgUuAw&ppuSwren5u-kZ$hIFTzE6vJ?e~Yt`x*&lFA#bw>Eh8@l70(`lqLTS4~uUSs%&X?;5E(9xMyltacystW4H~IdIvBhrzAg9(O7%0OY+$y z@*c1AIs^|2lsvZ2!1s zoVUgtc;q7bI3e^OSqVOteeSMC_j+Xe0$p7p^sS?^brlQDr|dxmFey?r!0D|NWvg8ewp5ucxK`XE2?ee>B-3z5QJ{Z-vORDc0&Pr{t zrSKtjwDwe3>G#Y;e#R|_-5+<^gb0Nv1-sN+aIG;8jqlq1UoW796ny7sv3tm38c@F+5bb zF7U~wvC?d2RhRSfIr4TsWX4n;e~Yb<4JkzB<#55P&hvw8N?s0nFVeYEQzelx1wg?X1+uY2jG3aZ0AT{I3YaVd?y4>O-k_( z)BfbMn(wbM0>ytu^~3Fa$?K7bbrI8apI@YjzS2V}FyQDJYF1g1=ds#rl4MVxNEQm6 zZO_S}xv)QDLAf8(%la!@itTqVi|Q43a*!e+(|)~FxO{WBZC>PS^Cm^}nDw(4+s?WdNk=>FQRfoc&MjT-Z!* zK8TsbgA%`oD|JZA*Lp^+u*_siAVFdQCM;jvQa~U^c1Ay$lhb^c7jE6g5UutnA0{fT55KWu*Vz zQgf@{IqFpuMr)=^cg~I!rxp6m zq-7jS@+rBw6oSbx+k-4Kt)VElA6M9ct-dp!^*3KHFnqRWQxXQ->_XX};J1pD7^PV% zvRpZE=c!ep&0LV1_uXWjZxk@(|CpbnC|}dY@L_zO3Mxr4AB}9i>tuEF|`={Enf|lc6;+pDbsAt zLXqA<=bK4PHXb&Fj$Fpw4&B|(a-)UOzPl|eHHOU!_r>s4fVq{F`2qX#t|(eupB&CL z-=G-TBGtzhu6ON_`#I#Q3hZf>bnS-{zH4z)js8xujL@5tm9&si`EtQb%*)L~B&R#f zhMvP!Gj26LuOwTWylLPj)`iWj_;B(DFv?Xm4KLei?nwBb*3!Oqw&0@H-ZOS#M1jh7 zp9s^3MEMozE>lE(EokbkxS^sG);ZUy@!L8fAR4i4Mlk!FN?@g2q2lSJPZTIytw>qI z`s#e4Ut(m*F7pier~2`<5zE1|;`y$+GV<^71xFMViAw?T0*-Vt|1DVv;(^w#R3brV z4-Hsmh`EiVh23rSyo-W6eoImyZ9v}NJkky93S9!FC9v*fET;(D~PN|OV1YWyfg_!VJArP&55>q1riGt3*y9GOd$Y33J4$)(1N0*GH-1|cDmoR#qW zn6?l?XUAs}d64l`t|fWdnt1*aLE_{0iFu5Fx7LBU|Z?`s3EB8oECEwF% zt~HZ8AAP`81mncrm{6^xTXp?V%W7NyBstWmAdWf=LS?jS))dSOo za|vp(uA2jK4jM;qUO~yUaQA#113)(Koj8b4gV0ybW;^(aWV(3VV^9+k#+t3zR<^$G zy9Ph@qrb|-TWOZy25T_u?dwgIoH$Db?%m#vM9);ae2s?UZQ`H&=d0@OyjXeiRZ6iX zJ0U;V#>j4Qtn;0LZkpBF&d?RQ>b8WUr#` zr?uycMg&}@IclyOHkS1W*X=7*Kz-)F1Jc5~*fhuY*Ab&Qxlg}-EMxhwe?rMIQx>(` zG9a<=A&*r{gC3XM$1R?>l}V-5i)tuztkaON>_F!I#sag_XlV>N_AqA)z0b06Gda?#wp(6^WBp2X&N27Mdk~VQLm1V=vB(pEf3d2*G*o} zS2i!UvM#ZN8A%#Wt9s+7DZXqI-;28Mqyns=gp_Rx$;*VP=2sj zNU_pwt3wEs*`kS@s~QSxcf=KtZ!3`{-!5-W3qfD#om)M!om=Qq_!gQ^Z{uNSJL?;! zYk5R~G2bM2k26$~vB7`Fj3(5XBaNEm-NExleQw3Tr$G>*RPD&<@C8#HS^9S-t*x!t z&iP$or--4hafD317FzIg%hg92qBV{V@aGGfocm$Zu>?trDj*f18s2aMN)Sn|JDra2 zb&0Jhd>lh6Qli_ON2pw<8z}Nrg~vD)Zf>K$ z)gH`x&sJn(bs=Tc+lJD-n3k*AAV4GY_IrMuPUil7!Sn&iWc&_E3APRd0h35st-E|+8-5gSicDB8wY6@5(8%iS$% z^RJ$nvdsEu^MZZK{q`Q~IJ6dX<{hWb3u>yEc9=<(fpfSq1pu`(4(&}8k78P!`h0t7 zgqZAS-H1&81&D%!;c$T)h0#zJMIgGf(p~mWqsMp*o?>P!5X+8%q33nAXR)RfG3-kR z%+9l!RJ!eA^i?I4S-&<+E&Mx%n@`3pH!WXrPA=k zr*|si=V3$$p`b21u|#>nh;I?sAvi^pzT&cX?MT&vMy*g_)4`#zI7@&_KA2hxN~&F_ z^avxsLxG=8FU+Z9oZ*AqR({7v7LQ*lx&S4t* zT9OA!v*o>TLM8RQCz-V+CSjEF)G@fx$xDY*V&$z2eGeY1NycE8mxkS-TrL~5cCL6m z&(E>*k<)T-1;e@GzGRhIMEfZh{dd_+%PjO|Md414qP$x)B{7L9_|E%#oCMr1@{Q9z zalF{^p)+a+qa46tQVtJOrS8SMQzq|S*4V=6N)GCTeC#ZZh+3t)%BP&E@3GBKDi@qs zbtS%jim5ehvr-(O#(=E`|-qoN;^IV3a z-PT?VqU-IO`j6d{UQJE!}c(7eTRKtSKgMQ)@(y45zGTT!lxp zwF{2yj=!f(GbZpLFf6f^`_6UR;JtIKaTRAzocKeTL~rce@pibx!l6F?w>+XFRVf

>aCH`7k9d?)RlhMW2P^#<367acz&@z)zBys8hc5o|6$d{YkG%w{jJ#PFe1oOS_E=XN| za(rl19x{lrM_2a1M6>u9cVUqE-djp8V{N+Q)3%^OKadkjSTZ?rVFOX{yg;+ejJHWOX^K$XCApK+nKNGLrFL=PTU~4WvhAAU+qRJMAY~0 znNMGT`VqjzSQ`lJ3jIpSFr%puMG3=4kJ~`gf}){ITyba?e814oVaA-~_iNx^R0wobgt$+7ET^iE<4CwZmO|U! zKcjE8%yV=g#DPD2ojLbvVh@i7<>T2W?LahplGDZIT4xU`+!g24ae`)|*E!;+7RmXw zz@$0W*!z#Hh`1l6kNMC>rPsOUs8i_CO;YkowAHlqF779AkMJ4B)E<_p*;!t#m@lm8 z#L9DwJXz>c{C0+~HyU^~9`-@)ietuiT&p4FEgqvDw8CH#y2!#gMb0g%4=+C}^zo;> zTQSvYdwGlW=xfRM?&P*QYsGsjgOqj-acs^rY!%xLBSo~-<#70?e8ru$iF}bB!^$nvIq-$Hr!z&RmoF?g(scK1`!@UYa>X(14!HmQt9`%Z^8r zHZEE*Lhkiq*_-Z7G0#qu2X+|H2XYcTQC*a|kx~+onV%OkyYVoh$zer4OHY$`uL%+t9MCuM_AxrX-*C)8SOC znFUxWibcr%*#dFLkwjc}qF-;|o>JkCtxX?{5>kCFL&*76SB*lJc$RI_YLOK9j5XB1 zK(jo$^UL79(;PDH;i5)*lvf!e`ZWw|MyogIDqB15z6d#3v!rK=%paRUqOz{tf{!nY zRVz)gc=1*mEmOkui8O>Oqp_8k){x8i0r1auL7Ef{J#%|&!kDXrhh&^CAGv8Cd2C%L zKJ@vtAIV^wBClBxaa>j_s!zYQ_gV?9D+_4`En5l2mj$Vg$Q74d3J=uUqmGihOcSgg zev2wSbd<+&J&s0EY6N8zL6oHf(}Ewbc;^$}3RQX;GB*fSmn<0&j2m6hE!B*~o^o3J zx|ANH@Ugc>Kt5B62R3<9a-^y&FU?L})qXd$Mc&GUB}bi)GYUdVaRkoHVcm>*GR5gS zF^$n>H0sBJCB{9ywX}mvf_`|&Ir-@X++LSh5@Hn!Z-`J%Z}UPU>q3LqjOwTu0N6XRZf7MDZOr1% zvzQE#*M0oYeJ}?%<;8~98Cvzq#T>s@$CNL~?$zF^^@2O^*nO}nc@A3Zjdu)C=(bwQC_OWu+ zZu!&ac=s@Rp^|vcgclRWky=XTNr{cs_oj6@%&j)Ugga~0J>m>y34O5_oE`Ix@s$eh zQGQC|5SOhNk1#6(j>HZoxlpl_2z|=ZV~V0H{K3voPfEu4CgxqIT(`2}b5gwF+9yQR zwBl|lPul#WnCI8b>QI$*12>U{Ij`bP|0wb%FnS=)rKY>E|AM1H9oJyuQ;6|sr6OL`%hvr0Rp$xb z&jTbI`&kAKymg8>^vB4lH;oM1>O&N4qwxyeu8sh7HnoDhvmTYJ)X!9N?^G zqI*r@x^b<6NvP@q;g4KRBlMf-VH9so%tBsPs!!hS9dt6YRK+!LvsfLc-5ndJ>BA)# zyxX5cI?+*~T#UuR`M@NS@>{8H){(|d*Yat+=bZ3e26ZK4^_}ya3&KDq>r#W4=Hw!g zZ*uKR6L8u3=V+Xm6;a8-r%!xZ$OK3(QjRLC%en_>8OSom;q_M$NZ5?KA&$hhO7$$0 zznn4@7e3w557>O?Y$=RT2iMP){K!liM_S=drb6r8y>Q>rT1XS9LHq7;N0>-OAfAy{ z>#W!O$sRo;KA&^m*%yb6L)TTE`H5HYr4KD5F^SUUt%$xyo*72$)#NI($~)S9BkE{i z55|85Jc;KwZcU5MTdvhNF-?E0l$`X%rj*9c3-{WbOW51%%x|xBP%jxDhEq~*>wDh0 z%ktEuTYO}bT_?Nvrq!wob5U_<`GZlLyvYyv>tio>?V)bzEMZX9=X7W zG1?p0kz^VxGX^oqb9vh<-lkU)1V>@LarKJQZ*rSHn zz=s}Z@h!Ic6}dB7<^HU6U)NNGSGrAWhc3PHiz>z4Z+#YjVfQlYFGKG zwJq1N*^(DK65Fqn1U~T`ZORtYs|sFya68zW(u_YDpF5B3Xn~-{NUXB;I(DA5N!-v4 z?atn92ZqnIBnFOKSN9O136dI-yuPpS@(ff240bz|zU4FT)%vN6L*%N8-#iW%Cl0*V zUUv)5I40kaKOpD`V zx6@r>UYxl|*xb(xwU50OLyA5qPVa*pZPa3CA>7=bARX6}_SX3@IXBn5deDkBQKx19 z_v(80xS`x7p+&7q&y{aX+7YM>A7ZcJb91UT@qVg%amJpMoTm@Y`vj-ow3e*m^5y)h z(PW7%7LpjnV$WjOw2Pna4)R(qalF6ECgqb^ZTj6w|BVm+l=}U^iEcH?Cm(b+CDy*$ z$`HA!@tf7gmGBKF)Qzna$qbcj5;%y7RkRYWVd$aA!!2L>7+N~Ok84Hx)QCOo<_4?P zXt^b_6J_A|!wUjcb?B9(+tvFf;@BDhDx|fzT-1&^C&yNfz($q2;(J4#ps>Uay#QT@ zb>|NCv0wG^z+-~rUL1JftxcN?lBX=4iESQ4f8=5Rd;tmr;vcs zCY?afx>OU*=bhf|5I%f1RHY%;QN{3l%EdA#XV_Y)Y_B*3bwdn`Jp0od<2Omy|Xn0RgV40k|T^gB_Y;z-=(&r z=M{s`M89kvTLWPS0n_G}a3*cOGQz>kcPx2wGT?yz;Fg3DP(Euel)Nn#+`E|AYuN?2 zYoI4)JU-7Rb-7`h$JS}%F1DsYiNa%#cRRKvch9!&>ADj{zYxA-V6v-E*kAcBzchH! z>u3wMS#}t|&K{e+m{T#A+_c*xbx(I;m8dtuSfXzr61S$%@h>%)hD-ebEq1lFwUj5zQ|Kqv`CDR7?mAMxm8R z!6S#f8eIj}26IX2qpHdq1U?*cKNM(kDHDx2EuLr%XFNU7O~!=xX6Q>C?OwN|KRsO= zawf2|GvZjAtfP26M!aNtEIgGH9EiA=FZON)w!4o(n_4)654ZeOaTWNvu7dnWBA?cUtnRO655%ni(u9x$TlMU3p2mMX3$r0udO1$J2cRHH#~s=yVJdz4CP zU(huVzmFFP-Yw@inZ|8Zru{EjMDpu3DIhd;ZY56B6s%#<^oBhMN z%<9wwo}8i>L`OXe@B17`tMMl5s-6V94V&4dTENDb)Mmx&Ab)3FbUyzp_kZp#pisG= zNusTJazx0qA~Bc+D&1osh+Un#&@MQE4bQwk=TnC1fROe@rj{$UhHyF3=<{F_?UOvt zZ1<-&@>X1}#Y+nBd*>-`wL|-u_P^SwS_z-YGe2$&zOMIX+`Cx3Jo+_VTiD%2I-A;% z2ff9wy{ol1waR_9e21J4C|cie>fPx*UaS`J-tAnT+`@B-AZyf_xFO0AtOMo68CD&1tJVCynL_tN>{PY5~l>pot|C&09 z^5^~n(!J;tD@gbA@+q0bv!xYQKlmNf(Ms?`ZnZxY9| zBN+x?SPc4DI_4EesNp%I6k>{|m{D$ht{mkW zM%x8m1e2xmDWXM9){)UCQf2u@p{i@!`NG2DJzGY!%r`o+4)ioTD&Sqc8|TI3`)hIm znbZRGt>Vkh-S6wBJ z)s@WT2^P^x=o-57GdY%#%te9j>uE3DAK!LqRH!Q6N=XMI$Y0+ zhQ<;nKJN$T2zJQ;^c62OMSL|(t6!GP58Kv)Hc{P1wynxa_(`5!*n6?(2;A&nY^cOq zh<43WQ2(mm#o|R=kd?R@zR5bP?>UTvbc(b;-ftBqTX01b`5xmhrF>yDlfSKh>%%~3 z5<>VbbqwPob|w2?Vprw}CY1>^DUsyglV0wffnZ^(>RXcM2=u5rx34rd|1`fZ0%AdG zi25Y*TotyyH6r#1ovq14`~Wd>Ps{WdvjFGeb8~lh=flwb#|u9~iaH=#E7K(+I&%?H zoP5XeTl5nZi6v>a{@fN5yN@KuOPxeR)-kGrA0j$-u2cxWl?1O^o6`K2Zze+8 zU7{gavG%$B|1z+QyAPs`~NSoUvlg_jw0#ShMKG*hu#yu ziU{1z`)Vma3q*efur6{l!K)VPXqi)Pi{~PiPv%V7_s4s!{BbUD2!=5-5q>&%f6NP# zC@I9@aGo9!B9r2T1LlS$h3F``QjxZFhr$*_?hEh9Uj4bp|HDp@Y$1c)&l|ZYdgQG; z{yr1kq~7szj;1o>*Z|`rfUNW~V?#nvtf4B(O%oH*rj#OmMp}NM@a=P-YUBG zFsp!^Igzs$(bNz>^iHJ9ybMXaf>6V7_I)S7@_FPca(intfT-=l;1)*FvUj}(QYc<) z5B?`q41W8E0UVAF2X`+eevl7S+um1j=d_`2Da~LF>U%v9Q2IE6J{2qqKR@H|_$!Hs zjtyCXKU{z$`4Ks}Rql_x5Le{CwE{6&hz`9Q_PK_BL|wAu+c!%aw8M?IHRg!WP_ko< zgx~SQAWW#s$mO2jXUBvzbqa5@ukjGM;vsx~Z~Bbo0#NxE6Mh^t9@nou>- zM}E`5vg+lY+UWgtrLP=$CAAO9W20#Q#L2l~))ISJMY%x=!Nkp`Oj+Z6@~k_WCYZEk zB2Lew%4k%M;2RTCW_Zm+*S8&vWl{KK$a*o(D!yT0J$&p51?CUj9Ap2z5O?vpQikX__N1< zA2NtTf*9LH9{u~BpG{sOfo`zJk>vmA2FV*0g~^-ak>GEgBcb?1KsO#Si8uZk?$2)A z>KPFDni#0k{v0JX3JQ~u`K=f3zux@kmphcCEq4}EZ~r+;M9>Y2smJ3BT3l}O1qq4Z zOKjMK-~WpwL<+iTkdQ|Hb6OwVNVPwN%KkY@6wnRXC)BJzxZ-@}mzoNY!YCpWp) z@BG0^=KM%Ewb$!*L3RV915?xB;2>%RZ8YC+T!n>9%C_WbJX$$HTTg7$c>mP1)j|5KqZwHf8p^ye#H3D+x_ zzv2AG{qEZ2SoFvKg|4WG571gpf-%CNqdp?Q46Lk}6`>()mbN|TS|V*XM$fbR?K7E~ z@ThN}EVO zcI~wHMn3o#hGyujuRm?;6S3`OhizrtV?U3e(>y;z<6T4gbFHe8v~oza&#la2B|_i7 zFDWdHs?)L*P>;aXY8;5*BqO7H0<#x+YACC*6->=L;`^3jg+9S08o*%O5~Tf;=@{{S z8D0DjwA(eC&tFUFx9cK_87}pD`bQLnHH*&ngq+IDeyE-t_uqKG-%8~w8A!sbA$v>B z`wuETx#50itoaUa(=p>>LdO*qI;?|t8@jY&m`ur98{DZ219#LVjN+_-6W@^X(>`sm^A07xu>jc}CG7i+MUe|Am}Y@U!BsUQH`si>=KX9HnPU*!P7_ zy+T^(XSoS7-kbU+?q1TF9WSCQ4U&>eNtUh-Y|rXoqy zG}fa8{OoO>(J-Q=yt$+9hTW`%KE>odthrd!OG@_mu7jWsR*zfJ6p{ZK3iE>&-yu() z*Ae=Ljjm}*ahW~NImkmc2n15Ijr=y5@`{U7LoQtIc{SjYHx z$^D~&Oq-d0om|tF%k&p${r>L7ENQ?fi0pm5Th{q=he1)wMPIEKPQ4ZSZhl&~goRTp z?)AQWd?T?%VqsN=quZYiQfY5YckMG*c6gLC=e)F`xAJy7IrEeJw^Oz~m>vJ>`Lq7D zuj78RW(DuKcGYn==bsr3b)S;0&Izty`qg=#h3Pl<3-#a9nvM?kmizWIzrA;Mo5C_t z7b`*MubxY!?>?|S<(47n{X3>wW~p54+wac2M$e>gO6?I3Ntr$Cm&qI>{gsE#8kwxP znf!X!-su(3;X@a zMHDw)v%gh!*3x9Y!+M9;^0{}TPdGAV>sc)%n{qba zoqW+-sI|Z*;co^@U-T-@9oR&fR}CB^L)CD=4%lF>ZJ48m-Y`%GwnWeG4G=~zVC8`= lS~~|nVU#MJk?FDE5Br$@|NWhrPy85wz|+;wWt~$(695h}@|*ww literal 0 HcmV?d00001 diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py index 5143218..e443b9d 100644 --- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py @@ -70,10 +70,11 @@ - Skips redundant updates for empty or invalid data fields. ### Setup - - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`. - To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html). - - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive. - - Customize the column names in your Google sheet using the `columns` configuration. - - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. + 1. Requires a Google Service Account JSON file for authentication. + To set up a service account, follow the instructions in the [how to](https://auto-archiver.readthedocs.io/en/latest/how_to/gsheets_setup.html), + or use the script `bash scripts/generate_google_service_account.sh`. + 2. Create a Google sheet with the required column(s) and then define the `sheet` or `sheet_id` configuration to specify this sheet. + 3. Customize the column names in your Google sheet using the `columns` configuration. + 4. The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. """, } From 5c34ac12930ba091611e45a1b64ea9d954f3df64 Mon Sep 17 00:00:00 2001 From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:05:23 +0000 Subject: [PATCH 03/27] Update docs/source/how_to/gsheets_setup.md --- docs/source/how_to/gsheets_setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index eaad809..629b095 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -30,7 +30,7 @@ The email address will look something like `user@project-name.iam.gserviceaccoun We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches all the columns required. -But if you like, you can also create your own custom sheet. The only column that's required is the 'link' column. This is the column with the URLs that you want the Auto Archiver to archive. +But if you like, you can also create your own custom sheet. The only columns required are 'link', 'archive status', and 'archive location'. 'link' is the column with the URLs that you want the Auto Archiver to archive, the other two record the archival status and result. Here's an overview of all the columns, and what a complete sheet would look like. From f6863b8eb2055d736367dc6a4088ecf236d97555 Mon Sep 17 00:00:00 2001 From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:10:47 +0000 Subject: [PATCH 04/27] Update src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py --- src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py index e443b9d..f5aec93 100644 --- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py @@ -75,6 +75,6 @@ or use the script `bash scripts/generate_google_service_account.sh`. 2. Create a Google sheet with the required column(s) and then define the `sheet` or `sheet_id` configuration to specify this sheet. 3. Customize the column names in your Google sheet using the `columns` configuration. - 4. The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. + 4. The Google Sheet can be used solely as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. """, } From a57722846527efe61c9e2ebd28bdff7f3d4bc25d Mon Sep 17 00:00:00 2001 From: erinhmclark Date: Tue, 18 Mar 2025 21:10:06 +0000 Subject: [PATCH 05/27] Update generic_extractor.py for general/ youtube extraction. --- .../modules/generic_extractor/generic_extractor.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index e7b75d9..6a9e28f 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -203,7 +203,7 @@ class GenericExtractor(Extractor): if not result.get("url"): result.set_url(url) - if "description" in video_data and not result.get_content(): + if "description" in video_data and not result.get("content"): result.set_content(video_data["description"]) # extract comments if enabled if self.comments: @@ -220,10 +220,13 @@ class GenericExtractor(Extractor): ) # then add the common metadata - if timestamp := video_data.pop("timestamp", None) and not result.get("timestamp"): + timestamp = video_data.pop("timestamp", None) + if timestamp and not result.get("timestamp"): timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat() result.set_timestamp(timestamp) - if upload_date := video_data.pop("upload_date", None) and not result.get("upload_date"): + + upload_date = video_data.pop("upload_date", None) + if upload_date and not result.get("upload_date"): upload_date = get_datetime_from_str(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc) result.set("upload_date", upload_date) From 488675056bf2bac2d42721dda499f681ca27ac6c Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 19 Mar 2025 15:51:45 +0400 Subject: [PATCH 06/27] Download generate_google_services.sh script from GH - it's not packaged with the app --- docs/source/how_to/gsheets_setup.md | 6 +++++- src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py | 5 ++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index 629b095..4fb59ff 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -13,7 +13,11 @@ Once your Google Sheet is set up, you need to create what's called a 'service ac To do this, you can either: * a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder -* b) run the `bash scripts/generate_google_services.sh` script to automatically generate the file. This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created. +* b) run the following script to automatically generate the file: +```{code} bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh)" +``` +This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created. Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4). diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py index f5aec93..01ba8fa 100644 --- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py @@ -72,7 +72,10 @@ ### Setup 1. Requires a Google Service Account JSON file for authentication. To set up a service account, follow the instructions in the [how to](https://auto-archiver.readthedocs.io/en/latest/how_to/gsheets_setup.html), - or use the script `bash scripts/generate_google_service_account.sh`. + or use the script: + ``` + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh)" + ``` 2. Create a Google sheet with the required column(s) and then define the `sheet` or `sheet_id` configuration to specify this sheet. 3. Customize the column names in your Google sheet using the `columns` configuration. 4. The Google Sheet can be used solely as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. From 244341d22c3f0d96508a23e82ec5471948efc7f2 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 19 Mar 2025 18:08:04 +0400 Subject: [PATCH 07/27] Skip check for 'docker' bin dependency if already running in docker --- src/auto_archiver/core/module.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index 903a4ab..d086f6c 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -237,8 +237,13 @@ class LazyBaseModule: return find_spec(dep) + def check_bin_dep(dep): + if dep == "docker" and os.environ.get("RUNNING_IN_DOCKER"): + return True + return shutil.which(dep) + check_deps(self.dependencies.get("python", []), check_python_dep) - check_deps(self.dependencies.get("bin", []), lambda dep: shutil.which(dep)) + check_deps(self.dependencies.get("bin", []), check_bin_dep) logger.debug(f"Loading module '{self.display_name}'...") From e531906d7345ea23cd149e169aaaaecebbb015b2 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Wed, 19 Mar 2025 18:08:24 +0400 Subject: [PATCH 08/27] Create an independent profile file for each wacz_extractor_enricher instance --- .../wacz_extractor_enricher/wacz_extractor_enricher.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py index 975d49a..b66f03c 100644 --- a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py +++ b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py @@ -24,7 +24,8 @@ class WaczExtractorEnricher(Enricher, Extractor): self.use_docker = os.environ.get("WACZ_ENABLE_DOCKER") or not os.environ.get("RUNNING_IN_DOCKER") self.docker_in_docker = os.environ.get("WACZ_ENABLE_DOCKER") and os.environ.get("RUNNING_IN_DOCKER") - self.cwd_dind = f"/crawls/crawls{random_str(8)}" + self.crawl_id = random_str(8) + self.cwd_dind = f"/crawls/crawls{self.crawl_id}" self.browsertrix_home_host = os.environ.get("BROWSERTRIX_HOME_HOST") self.browsertrix_home_container = os.environ.get("BROWSERTRIX_HOME_CONTAINER") or self.browsertrix_home_host # create crawls folder if not exists, so it can be safely removed in cleanup @@ -50,7 +51,7 @@ class WaczExtractorEnricher(Enricher, Extractor): url = to_enrich.get_url() - collection = random_str(8) + collection = self.crawl_id browsertrix_home_host = self.browsertrix_home_host or os.path.abspath(self.tmp_dir) browsertrix_home_container = self.browsertrix_home_container or browsertrix_home_host @@ -102,10 +103,11 @@ class WaczExtractorEnricher(Enricher, Extractor): ] + cmd if self.profile: - profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz") + profile_file = f"profile-{self.crawl_id}.tar.gz" + profile_fn = os.path.join(browsertrix_home_container, profile_file) logger.debug(f"copying {self.profile} to {profile_fn}") shutil.copyfile(self.profile, profile_fn) - cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")]) + cmd.extend(["--profile", os.path.join("/crawls", profile_file)]) else: logger.debug(f"generating WACZ without Docker for {url=}") From 2921061fde756cf9f04a5e994546c6afdd386770 Mon Sep 17 00:00:00 2001 From: erinhmclark Date: Wed, 19 Mar 2025 19:19:28 +0000 Subject: [PATCH 09/27] Add flexible extractor_args to generic_extractor.py --- .../modules/generic_extractor/__manifest__.py | 5 ++++ .../generic_extractor/generic_extractor.py | 26 ++++++++++++++----- tests/test_modules.py | 2 +- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py index 128b006..9ef1cb3 100644 --- a/src/auto_archiver/modules/generic_extractor/__manifest__.py +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -74,6 +74,11 @@ If you are having issues with the extractor, you can review the version of `yt-d "default": "inf", "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.", }, + "extractor_args": { + "default": {}, + "help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.", + "type": "json_loader", + }, "ytdlp_update_interval": { "default": 5, "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.", diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index 6a9e28f..c2bf054 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -422,16 +422,20 @@ class GenericExtractor(Extractor): "--write-subs" if self.subtitles else "--no-write-subs", "--write-auto-subs" if self.subtitles else "--no-write-auto-subs", "--live-from-start" if self.live_from_start else "--no-live-from-start", - "--proxy", - self.proxy if self.proxy else "", - f"--max-downloads {self.max_downloads}" if self.max_downloads != "inf" else "", - f"--playlist-end {self.max_downloads}" if self.max_downloads != "inf" else "", ] + # proxy handling + if self.proxy: + ydl_options.extend(["--proxy", self.proxy]) + + # max_downloads handling + if self.max_downloads != "inf": + ydl_options.extend(["--max-downloads", str(self.max_downloads)]) + ydl_options.extend(["--playlist-end", str(self.max_downloads)]) + # set up auth auth = self.auth_for_site(url, extract_cookies=False) - - # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file + # order of importance: username/password -> api_key -> cookie -> cookies_from_browser -> cookies_file if auth: if "username" in auth and "password" in auth: logger.debug(f"Using provided auth username and password for {url}") @@ -447,6 +451,16 @@ class GenericExtractor(Extractor): logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}") ydl_options.extend(("--cookies", auth["cookies_file"])) + # Applying user-defined extractor_args + if self.extractor_args: + for key, args in self.extractor_args.items(): + logger.debug(f"Setting extractor_args: {key}") + if isinstance(args, dict): + arg_str = ";".join(f"{k}={v}" for k, v in args.items()) + else: + arg_str = str(args) + ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"]) + if self.ytdlp_args: logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}") ydl_options += self.ytdlp_args.split(" ") diff --git a/tests/test_modules.py b/tests/test_modules.py index f672ca6..248e16d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -82,7 +82,7 @@ def test_load_modules(module_name): default_config = module.configs assert loaded_module.name in loaded_module.config.keys() defaults = {k: v.get("default") for k, v in default_config.items()} - assert loaded_module.config[module_name] == defaults + assert defaults.keys() in [loaded_module.config[module_name].keys()] @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) From 799cef3a8c9b691b0c856ec1e36b87bb1742ca85 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 16:28:19 +0400 Subject: [PATCH 10/27] Cleanup docker-compose --- docker-compose.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 56c2ccb..07ceb00 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,4 +1,3 @@ -version: '3.8' services: auto-archiver: @@ -10,7 +9,4 @@ services: volumes: - ./secrets:/app/secrets - ./local_archive:/app/local_archive - environment: - - WACZ_ENABLE_DOCKER=true - - RUNNING_IN_DOCKER=true command: --config secrets/orchestration.yaml From f22af5e123378b0c66f1e4ebb0c5e11edb965cd5 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 16:28:47 +0400 Subject: [PATCH 11/27] Tweak WACZ enricher docs + add comment on WACZ_ENABLE_DOCKER --- src/auto_archiver/core/module.py | 22 +++++++++++++----- .../wacz_extractor_enricher/__manifest__.py | 23 +++++++++++++++---- tests/enrichers/test_wacz_enricher.py | 10 ++++++++ 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index d086f6c..9adb14a 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -5,6 +5,7 @@ by handling user configuration, validating the steps properties, and implementin """ from __future__ import annotations +import subprocess from dataclasses import dataclass from typing import List, TYPE_CHECKING, Type @@ -17,7 +18,7 @@ import os from os.path import join from loguru import logger import auto_archiver -from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE +from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError if TYPE_CHECKING: from .base_module import BaseModule @@ -216,9 +217,9 @@ class LazyBaseModule: if not check(dep): logger.error( f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \ - Have you installed the required dependencies for the '{self.name}' module? See the README for more information." + Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information." ) - exit(1) + raise SetupError() def check_python_dep(dep): # first check if it's a module: @@ -238,9 +239,18 @@ class LazyBaseModule: return find_spec(dep) def check_bin_dep(dep): - if dep == "docker" and os.environ.get("RUNNING_IN_DOCKER"): - return True - return shutil.which(dep) + dep_exists = shutil.which(dep) + + if dep == "docker": + if os.environ.get("RUNNING_IN_DOCKER"): + # this is only for the WACZ enricher, which requires docker + # if we're already running in docker then we don't need docker + return True + + # check if docker daemon is running + return dep_exists and subprocess.run(["docker", "ps", "-q"]).returncode == 0 + + return dep_exists check_deps(self.dependencies.get("python", []), check_python_dep) check_deps(self.dependencies.get("bin", []), check_bin_dep) diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py index 97e3bf6..c6454b0 100644 --- a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py @@ -11,7 +11,7 @@ "configs": { "profile": { "default": None, - "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles).", + "help": "browsertrix-profile (for profile generation see https://crawler.docs.browsertrix.com/user-guide/browser-profiles/).", }, "docker_commands": {"default": None, "help": "if a custom docker invocation is needed"}, "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds", "type": "int"}, @@ -40,14 +40,27 @@ Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving. [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format. - ### Features + ## Setup + + **Docker** + If you are using the Docker file to run Auto Archiver (recommended), then everything is set up and you can use WACZ out of the box! + Otherwise, if you are using a local install of Auto Archiver (e.g. pip or dev install), then you will need to install Docker and run + the docker daemon to be able to run the `browsertrix-crawler` tool. + + **Browsertrix Profiles** + A browsertrix profile is a custom browser profile (login information, browser extensions, etc.) that can be used to archive private or dynamic content. + You can run the WACZ Enricher without a profile, but for more resilient archiving, it is recommended to create a profile. See the [Browsertrix documentation](https://crawler.docs.browsertrix.com/user-guide/browser-profiles/) + for more information. + + ** Docker in Docker ** + If you are running Auto Archiver within a Docker container, you will need to enable Docker in Docker to run the `browsertrix-crawler` tool. + This can be done by setting the `WACZ_ENABLE_DOCKER` environment variable to `1`. + + ## Features - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`. - Supports custom profiles for archiving private or dynamic content. - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline. - Generates metadata from the archived page's content and structure (e.g., titles, text). - ### Notes - - Requires Docker for running `browsertrix-crawler` . - - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings. """, } diff --git a/tests/enrichers/test_wacz_enricher.py b/tests/enrichers/test_wacz_enricher.py index ceab83b..f4d1557 100644 --- a/tests/enrichers/test_wacz_enricher.py +++ b/tests/enrichers/test_wacz_enricher.py @@ -4,6 +4,7 @@ from zipfile import ZipFile import pytest from auto_archiver.core import Metadata, Media +from auto_archiver.core.consts import SetupError @pytest.fixture @@ -22,6 +23,15 @@ def wacz_enricher(setup_module, mock_binary_dependencies): return wacz +def test_raises_error_without_docker_installed(setup_module, mocker, caplog): + # pretend that docker isn't installed + mocker.patch("shutil.which").return_value = None + with pytest.raises(SetupError): + setup_module("wacz_extractor_enricher", {}) + + assert "requires external dependency 'docker' which is not available/setup" in caplog.text + + def test_setup_without_docker(wacz_enricher, mocker): mocker.patch.dict(os.environ, {"RUNNING_IN_DOCKER": "1"}, clear=True) wacz_enricher.setup() From 1e19ad77c628c3090e78c8093b48567ece65451a Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 18:08:19 +0400 Subject: [PATCH 12/27] Fix tests --- tests/test_modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index f672ca6..7067db3 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,6 +1,7 @@ import pytest from auto_archiver.core.module import ModuleFactory, LazyBaseModule from auto_archiver.core.base_module import BaseModule +from auto_archiver.core.consts import SetupError @pytest.fixture @@ -25,11 +26,9 @@ def test_python_dependency_check(example_module): # monkey patch the manifest to include a nonexistnet dependency example_module.manifest["dependencies"]["python"] = ["does_not_exist"] - with pytest.raises(SystemExit) as load_error: + with pytest.raises(SetupError): example_module.load({}) - assert load_error.value.code == 1 - def test_binary_dependency_check(example_module): # example_module requires ffmpeg, which is not installed From 5e5e1c43a179e678b8ae5614788763dc5eae0e83 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 18:09:26 +0400 Subject: [PATCH 13/27] When loading modules, check they have been added to the right 'step' in the config Fixes an issue seen on discord where a user accidentally set up metadata_enricher under 'extractors' --- src/auto_archiver/core/module.py | 6 +++++- src/auto_archiver/core/orchestrator.py | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index 903a4ab..c263d95 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -85,7 +85,11 @@ class ModuleFactory: if not available: message = f"Module '{module_name}' not found. Are you sure it's installed/exists?" if "archiver" in module_name: - message += f" Did you mean {module_name.replace('archiver', 'extractor')}?" + message += f" Did you mean '{module_name.replace('archiver', 'extractor')}'?" + elif "gsheet" in module_name: + message += " Did you mean 'gsheet_feeder_db'?" + elif "atlos" in module_name: + message += " Did you mean 'atlos_feeder_db_storage'?" raise IndexError(message) return available[0] diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index d06c287..cbd1af5 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -373,9 +373,17 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ if module in invalid_modules: continue + # check to make sure that we're trying to load it as the correct type - i.e. make sure the user hasn't put it under the wrong 'step' + lazy_module: LazyBaseModule = self.module_factory.get_module_lazy(module) + if module_type not in lazy_module.type: + types = ",".join(f"'{t}'" for t in lazy_module.type) + raise SetupError( + f"Configuration Error: Module '{module}' is not a {module_type}, but has the types: {types}. Please check you set this module up under the right step in your orchestration file." + ) + loaded_module = None try: - loaded_module: BaseModule = self.module_factory.get_module(module, self.config) + loaded_module: BaseModule = lazy_module.load(self.config) except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError): logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") From 6700250891dbefddad536e58caddd372ffec1166 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 18:18:53 +0400 Subject: [PATCH 14/27] Add a test for checking module type on setup --- .../test_modules/example_extractor/__manifest__.py | 11 +++++++++++ .../example_extractor/example_extractor.py | 6 ++++++ tests/test_orchestrator.py | 13 +++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/data/test_modules/example_extractor/__manifest__.py create mode 100644 tests/data/test_modules/example_extractor/example_extractor.py diff --git a/tests/data/test_modules/example_extractor/__manifest__.py b/tests/data/test_modules/example_extractor/__manifest__.py new file mode 100644 index 0000000..dc18dc7 --- /dev/null +++ b/tests/data/test_modules/example_extractor/__manifest__.py @@ -0,0 +1,11 @@ +{ + # Display Name of your module + "name": "Example Extractor", + # Optional version number, for your own versioning purposes + "version": 2.0, + # The type of the module, must be one (or more) of the built in module types + "type": ["extractor"], + # a boolean indicating whether or not a module requires additional user setup before it can be used + # for example: adding API keys, installing additional software etc. + "requires_setup": False, +} diff --git a/tests/data/test_modules/example_extractor/example_extractor.py b/tests/data/test_modules/example_extractor/example_extractor.py new file mode 100644 index 0000000..1c63383 --- /dev/null +++ b/tests/data/test_modules/example_extractor/example_extractor.py @@ -0,0 +1,6 @@ +from auto_archiver.core import Extractor + + +class ExampleExtractor(Extractor): + def download(self, item): + print("download") diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 326b93d..3367ce0 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -4,6 +4,7 @@ from auto_archiver.core.orchestrator import ArchivingOrchestrator from auto_archiver.version import __version__ from auto_archiver.core.config import read_yaml, store_yaml from auto_archiver.core import Metadata +from auto_archiver.core.consts import SetupError TEST_ORCHESTRATION = "tests/data/test_orchestration.yaml" TEST_MODULES = "tests/data/test_modules/" @@ -224,3 +225,15 @@ def test_multiple_orchestrator(test_args): output: Metadata = list(o2.feed()) assert len(output) == 1 assert output[0].get_url() == "https://example.com" + + +def test_wrong_step_type(test_args, caplog): + args = test_args + [ + "--feeders", + "example_extractor", # example_extractor is not a valid feeder! + ] + + orchestrator = ArchivingOrchestrator() + with pytest.raises(SetupError) as err: + orchestrator.setup(args) + assert "Module 'example_extractor' is not a feeder" in str(err.value) From 0a5ba3385e75f8ce1878638a407d49489e0635f8 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 18:55:22 +0400 Subject: [PATCH 15/27] Fix small bug in twitter dropin - previously the 'content' was being set to a json dump of the tweet, it should be set to full_text --- .../modules/generic_extractor/twitter.py | 12 ++++++++---- tests/extractors/test_generic_extractor.py | 3 ++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/auto_archiver/modules/generic_extractor/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py index e27a0c1..189a7e6 100644 --- a/src/auto_archiver/modules/generic_extractor/twitter.py +++ b/src/auto_archiver/modules/generic_extractor/twitter.py @@ -1,6 +1,5 @@ import re import mimetypes -import json from loguru import logger from slugify import slugify @@ -32,6 +31,9 @@ class Twitter(GenericDropin): twid = ie_instance._match_valid_url(url).group("id") return ie_instance._extract_status(twid=twid) + def keys_to_clean(self, video_data, info_extractor): + return ["user", "created_at", "entities", "favorited", "translator_type"] + def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() try: @@ -42,9 +44,11 @@ class Twitter(GenericDropin): logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}") return False - result.set_title(tweet.get("full_text", "")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp( - timestamp - ) + full_text = tweet.pop("full_text", "") + author = tweet["user"].get("name", "") + result.set("author", author).set_url(url) + + result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp) if not tweet.get("entities", {}).get("media"): logger.debug("No media found, archiving tweet text only") result.status = "twitter-ytdl" diff --git a/tests/extractors/test_generic_extractor.py b/tests/extractors/test_generic_extractor.py index 2089007..616183b 100644 --- a/tests/extractors/test_generic_extractor.py +++ b/tests/extractors/test_generic_extractor.py @@ -206,10 +206,11 @@ class TestGenericExtractor(TestExtractorBase): self.assertValidResponseMetadata( post, - "Onion rings are just vegetable donuts.", + "Cookie Monster - Onion rings are just vegetable donuts.", datetime.datetime(2023, 1, 24, 16, 25, 51, tzinfo=datetime.timezone.utc), "yt-dlp_Twitter: success", ) + assert post.get("content") == "Onion rings are just vegetable donuts." @pytest.mark.download def test_twitter_download_video(self, make_item): From 54f53886ef68fcb4a07e740de53241d47822ec44 Mon Sep 17 00:00:00 2001 From: erinhmclark Date: Thu, 20 Mar 2025 14:57:26 +0000 Subject: [PATCH 16/27] Update tests for default config values --- tests/test_modules.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_modules.py b/tests/test_modules.py index 248e16d..a9eec1b 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -81,8 +81,20 @@ def test_load_modules(module_name): # check that default settings are applied default_config = module.configs assert loaded_module.name in loaded_module.config.keys() + defaults = {k for k in default_config} + assert defaults in [loaded_module.config[module_name].keys()] + + +@pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) +def test_config_defaults(module_name): + # test the values of the default config values are set + # Note: some modules can alter values in the setup() method, this test checks cases that don't + module = ModuleFactory().get_module_lazy(module_name) + loaded_module = module.load({}) + # check that default config values are set + default_config = module.configs defaults = {k: v.get("default") for k, v in default_config.items()} - assert defaults.keys() in [loaded_module.config[module_name].keys()] + assert defaults == loaded_module.config[module_name] @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) From d6d5a08204db357b283c71b8af7feb6c291324bd Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Thu, 20 Mar 2025 20:45:28 +0400 Subject: [PATCH 17/27] Allow user to save downloaded keyfile to a different folder --- docs/source/how_to/gsheets_setup.md | 10 +++++++++- scripts/generate_google_services.sh | 28 ++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index 4fb59ff..eefcf9e 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -15,10 +15,18 @@ To do this, you can either: * a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder * b) run the following script to automatically generate the file: ```{code} bash -/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh)" +https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s -- ``` This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created. +```{note} +To save the generated file to a different folder, pass an argument as follows: +```{code} bash +https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s -- /path/to/secrets +``` + +---------- + Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4). Also make sure to **note down** the email address for this service account. You'll need that for step 3. diff --git a/scripts/generate_google_services.sh b/scripts/generate_google_services.sh index 2668fd3..fb11e37 100644 --- a/scripts/generate_google_services.sh +++ b/scripts/generate_google_services.sh @@ -7,6 +7,7 @@ UUID=$(LC_ALL=C tr -dc a-z0-9 Date: Fri, 21 Mar 2025 11:53:47 +0400 Subject: [PATCH 18/27] Unit tests for url utils --- src/auto_archiver/utils/url.py | 85 +++++++++----------- tests/utils/test_urls.py | 143 +++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+), 49 deletions(-) create mode 100644 tests/utils/test_urls.py diff --git a/src/auto_archiver/utils/url.py b/src/auto_archiver/utils/url.py index 169ed87..368d93c 100644 --- a/src/auto_archiver/utils/url.py +++ b/src/auto_archiver/utils/url.py @@ -4,8 +4,8 @@ from ipaddress import ip_address AUTHWALL_URLS = [ - re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)"), # telegram private channels - re.compile(r"https:\/\/www\.instagram\.com"), # instagram + re.compile(r"https?:\/\/t\.me(\/c)\/(.+)\/(\d+)"), # telegram private channels + re.compile(r"https?:\/\/(www\.)?instagram\.com"), # instagram ] @@ -81,56 +81,43 @@ def is_relevant_url(url: str) -> bool: """ clean_url = remove_get_parameters(url) - # favicons - if "favicon" in url: - return False - # ifnore icons - if clean_url.endswith(".ico"): - return False - # ignore SVGs - if remove_get_parameters(url).endswith(".svg"): - return False + IRRELEVANT_URLS = [ + # favicons + ("favicon",), + # twitter profile pictures + ("twimg.com/profile_images",), + ("twimg.com", "default_profile_images"), + # instagram profile pictures + ("https://scontent.cdninstagram.com/", "150x150"), + # instagram recurring images + ("https://static.cdninstagram.com/rsrc.php/",), + # telegram + ("https://telegram.org/img/emoji/",), + # youtube + ("https://www.youtube.com/s/gaming/emoji/",), + ("https://yt3.ggpht.com", "default-user="), + ("https://www.youtube.com/s/search/audio/",), + # ok + ("https://ok.ru/res/i/",), + ("https://vk.com/emoji/",), + ("vk.com/images/",), + ("vk.com/images/reaction/",), + # wikipedia + ("wikipedia.org/static",), + ] - # twitter profile pictures - if "twimg.com/profile_images" in url: - return False - if "twimg.com" in url and "/default_profile_images" in url: - return False + IRRELEVANT_ENDS_WITH = [ + ".svg", # ignore SVGs + ".ico", # ignore icons + ] - # instagram profile pictures - if "https://scontent.cdninstagram.com/" in url and "150x150" in url: - return False - # instagram recurring images - if "https://static.cdninstagram.com/rsrc.php/" in url: - return False + for end in IRRELEVANT_ENDS_WITH: + if clean_url.endswith(end): + return False - # telegram - if "https://telegram.org/img/emoji/" in url: - return False - - # youtube - if "https://www.youtube.com/s/gaming/emoji/" in url: - return False - if "https://yt3.ggpht.com" in url and "default-user=" in url: - return False - if "https://www.youtube.com/s/search/audio/" in url: - return False - - # ok - if " https://ok.ru/res/i/" in url: - return False - - # vk - if "https://vk.com/emoji/" in url: - return False - if "vk.com/images/" in url: - return False - if "vk.com/images/reaction/" in url: - return False - - # wikipedia - if "wikipedia.org/static" in url: - return False + for parts in IRRELEVANT_URLS: + if all(part in clean_url for part in parts): + return False return True diff --git a/tests/utils/test_urls.py b/tests/utils/test_urls.py new file mode 100644 index 0000000..81600ce --- /dev/null +++ b/tests/utils/test_urls.py @@ -0,0 +1,143 @@ +import pytest +from auto_archiver.utils.url import ( + is_auth_wall, + check_url_or_raise, + domain_for_url, + is_relevant_url, + remove_get_parameters, + twitter_best_quality_url, +) + + +@pytest.mark.parametrize( + "url, is_auth", + [ + ("https://example.com", False), + ("https://t.me/c/abc/123", True), + ("https://t.me/not-private/", False), + ("https://instagram.com", True), + ("https://www.instagram.com", True), + ("https://www.instagram.com/p/INVALID", True), + ("https://www.instagram.com/p/C4QgLbrIKXG/", True), + ], +) +def test_is_auth_wall(url, is_auth): + assert is_auth_wall(url) == is_auth + + +@pytest.mark.parametrize( + "url, raises", + [ + ("http://example.com", False), + ("https://example.com", False), + ("ftp://example.com", True), + ("http://localhost", True), + ("http://", True), + ], +) +def test_check_url_or_raise(url, raises): + if raises: + with pytest.raises(ValueError): + check_url_or_raise(url) + else: + assert check_url_or_raise(url) + + +@pytest.mark.parametrize( + "url, domain", + [ + ("https://example.com", "example.com"), + ("https://www.example.com", "www.example.com"), + ("https://www.example.com/path", "www.example.com"), + ("https://", ""), + ("http://localhost", "localhost"), + ], +) +def test_domain_for_url(url, domain): + assert domain_for_url(url) == domain + + +@pytest.mark.parametrize( + "url, without_get", + [ + ("https://example.com", "https://example.com"), + ("https://example.com?utm_source=example", "https://example.com"), + ("https://example.com?utm_source=example&other=1", "https://example.com"), + ("https://example.com/something", "https://example.com/something"), + ("https://example.com/something?utm_source=example", "https://example.com/something"), + ], +) +def test_remove_get_parameters(url, without_get): + assert remove_get_parameters(url) == without_get + + # IRRELEVANT_URLS = [ + # # favicons + # ("favicon",), + # # twitter profile pictures + # ("twimg.com/profile_images",), + # ("twimg.com", "default_profile_images"), + # # instagram profile pictures + # ("https://scontent.cdninstagram.com/", "150x150"), + # # instagram recurring images + # ("https://static.cdninstagram.com/rsrc.php/",), + # # telegram + # ("https://telegram.org/img/emoji/",), + # # youtube + # ("https://www.youtube.com/s/gaming/emoji/",), + # ("https://yt3.ggpht.com", "default-user="), + # ("https://www.youtube.com/s/search/audio/",), + # # ok + # ("https://ok.ru/res/i/",), + # ("https://vk.com/emoji/",), + # ("vk.com/images/",), + # ("vk.com/images/reaction/",), + # # wikipedia + # ("wikipedia.org/static",), + # ] + + # IRRELEVANT_ENDS_WITH = [ + # ".svg", # ignore SVGs + # ".ico", # ignore icons + # ] + + +@pytest.mark.parametrize( + "url, relevant", + [ + ("https://example.com", True), + ("https://example.com/favicon.ico", False), + ("https://twimg.com/profile_images", False), + ("https://twimg.com/something/default_profile_images", False), + ("https://scontent.cdninstagram.com/username/150x150.jpg", False), + ("https://static.cdninstagram.com/rsrc.php/", False), + ("https://telegram.org/img/emoji/", False), + ("https://www.youtube.com/s/gaming/emoji/", False), + ("https://yt3.ggpht.com/default-user=", False), + ("https://www.youtube.com/s/search/audio/", False), + ("https://ok.ru/res/i/", False), + ("https://vk.com/emoji/", False), + ("https://vk.com/images/", False), + ("https://vk.com/images/reaction/", False), + ("https://wikipedia.org/static", False), + ("https://example.com/file.svg", False), + ("https://example.com/file.ico", False), + ("https://example.com/file.mp4", True), + ("https://example.com/150x150.jpg", True), + ("https://example.com/rsrc.php/", True), + ("https://example.com/img/emoji/", True), + ], +) +def test_is_relevant_url(url, relevant): + assert is_relevant_url(url) == relevant + + +@pytest.mark.parametrize( + "url, best_quality", + [ + ("https://twitter.com/some_image.jpg?name=small", "https://twitter.com/some_image.jpg?name=orig"), + ("https://twitter.com/some_image.jpg", "https://twitter.com/some_image.jpg"), + ("https://twitter.com/some_image.jpg?name=orig", "https://twitter.com/some_image.jpg?name=orig"), + ], +) +def test_twitter_best_quality_url(url, best_quality): + assert twitter_best_quality_url(url) == best_quality From 5b131996c6067a7020a6099dd5dd60afb2c1f338 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 11:55:12 +0400 Subject: [PATCH 19/27] Add return type for auth_for_site --- src/auto_archiver/core/base_module.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py index 642b8ee..bcaa59b 100644 --- a/src/auto_archiver/core/base_module.py +++ b/src/auto_archiver/core/base_module.py @@ -71,7 +71,16 @@ class BaseModule(ABC): :param site: the domain of the site to get authentication information for :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar). - :returns: authdict dict of login information for the given site + :returns: authdict dict -> { + "username": str, + "password": str, + "api_key": str, + "api_secret": str, + "cookie": str, + "cookies_file": str, + "cookies_from_browser": str, + "cookies_jar": CookieJar + } **Global options:**\n * cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n @@ -85,6 +94,7 @@ class BaseModule(ABC): * cookie: str - a cookie string to use for login (specific to this site)\n * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n + """ # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com) # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code? From 14c56f4916feaa661575af4531fd5ac4a3027cf9 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 12:05:47 +0400 Subject: [PATCH 20/27] Provide better logs for screenshot enricher when auth is/isn't supported (cookies only) --- .../screenshot_enricher.py | 17 ++++++--- src/auto_archiver/utils/webdriver.py | 22 ++++++------ tests/enrichers/test_screenshot_enricher.py | 36 +++++++++++++++++-- 3 files changed, 58 insertions(+), 17 deletions(-) diff --git a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py index 491bd51..4e01357 100644 --- a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py +++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py @@ -19,12 +19,21 @@ class ScreenshotEnricher(Enricher): def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() - if UrlUtil.is_auth_wall(url): - logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}") - return - logger.debug(f"Enriching screenshot for {url=}") auth = self.auth_for_site(url) + + # screenshot enricher only supports cookie-type auth (selenium) + has_valid_auth = auth and (auth.get("cookies") or auth.get("cookies_jar") or auth.get("cookie")) + + if UrlUtil.is_auth_wall(url) and not has_valid_auth: + logger.warning(f"[SKIP] SCREENSHOT since url is behind AUTH WALL and no login details provided: {url=}") + if any(auth.get(key) for key in ["username", "password", "api_key", "api_secret"]): + logger.warning( + f"Screenshot enricher only supports cookie-type authentication, you have provided {auth.keys()} which are not supported.\ + Consider adding 'cookie', 'cookies_file' or 'cookies_from_browser' to your auth for this site." + ) + return + with self.webdriver_factory( self.width, self.height, diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index 43e7817..0b5fa57 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -22,35 +22,35 @@ from loguru import logger class CookieSettingDriver(webdriver.Firefox): facebook_accept_cookies: bool - cookies: str - cookiejar: MozillaCookieJar + cookie: str + cookie_jar: MozillaCookieJar - def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs): + def __init__(self, cookie, cookie_jar, facebook_accept_cookies, *args, **kwargs): if os.environ.get("RUNNING_IN_DOCKER"): # Selenium doesn't support linux-aarch64 driver, we need to set this manually kwargs["service"] = webdriver.FirefoxService(executable_path="/usr/local/bin/geckodriver") super(CookieSettingDriver, self).__init__(*args, **kwargs) - self.cookies = cookies - self.cookiejar = cookiejar + self.cookie = cookie + self.cookie_jar = cookie_jar self.facebook_accept_cookies = facebook_accept_cookies def get(self, url: str): - if self.cookies or self.cookiejar: + if self.cookie_jar or self.cookie: # set up the driver to make it not 'cookie averse' (needs a context/URL) # get the 'robots.txt' file which should be quick and easy robots_url = urlunparse(urlparse(url)._replace(path="/robots.txt", query="", fragment="")) super(CookieSettingDriver, self).get(robots_url) - if self.cookies: + if self.cookie: # an explicit cookie is set for this site, use that first for cookie in self.cookies.split(";"): for name, value in cookie.split("="): self.driver.add_cookie({"name": name, "value": value}) - elif self.cookiejar: + elif self.cookie_jar: domain = urlparse(url).netloc regex = re.compile(f"(www)?.?{domain}$") - for cookie in self.cookiejar: + for cookie in self.cookie_jar: if regex.match(cookie.domain): try: self.add_cookie( @@ -145,8 +145,8 @@ class Webdriver: try: self.driver = CookieSettingDriver( - cookies=self.auth.get("cookies"), - cookiejar=self.auth.get("cookies_jar"), + cookie=self.auth.get("cookie"), + cookie_jar=self.auth.get("cookies_jar"), facebook_accept_cookies=self.facebook_accept_cookies, options=options, ) diff --git a/tests/enrichers/test_screenshot_enricher.py b/tests/enrichers/test_screenshot_enricher.py index b86bb17..ec56345 100644 --- a/tests/enrichers/test_screenshot_enricher.py +++ b/tests/enrichers/test_screenshot_enricher.py @@ -85,8 +85,8 @@ def test_enrich_adds_screenshot( mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env screenshot_enricher.enrich(metadata_with_video) mock_driver_class.assert_called_once_with( - cookies=None, - cookiejar=None, + cookie=None, + cookie_jar=None, facebook_accept_cookies=False, options=mock_options_instance, ) @@ -124,6 +124,38 @@ def test_enrich_auth_wall( assert metadata_with_video.media[1].properties.get("id") == "screenshot" +def test_skip_authwall_no_cookies(screenshot_enricher, caplog): + with caplog.at_level("WARNING"): + screenshot_enricher.enrich(Metadata().set_url("https://instagram.com")) + assert "[SKIP] SCREENSHOT since url" in caplog.text + + +@pytest.mark.parametrize( + "auth", + [ + {"cookie": "cookie"}, + {"cookies_jar": "cookie"}, + ], +) +def test_dont_skip_authwall_with_cookies(screenshot_enricher, caplog, mocker, mock_selenium_env, auth): + mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=True) + + # patch the authentication dict: + screenshot_enricher.authentication = {"example.com": auth} + with caplog.at_level("WARNING"): + screenshot_enricher.enrich(Metadata().set_url("https://example.com")) + assert "[SKIP] SCREENSHOT since url" not in caplog.text + + +def test_show_warning_wrong_auth_type(screenshot_enricher, caplog, mocker, mock_selenium_env): + mock_driver, mock_driver_class, _ = mock_selenium_env + mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=True) + screenshot_enricher.authentication = {"example.com": {"username": "user", "password": "pass"}} + with caplog.at_level("WARNING"): + screenshot_enricher.enrich(Metadata().set_url("https://example.com")) + assert "Screenshot enricher only supports cookie-type authentication" in caplog.text + + def test_handle_timeout_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker): mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env From 4b5a8c019966f3e078a1cfd8dba3ad33c5982a35 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 12:09:58 +0400 Subject: [PATCH 21/27] Add warning *inside* instagram_extractor that it's not actively maintained --- .../modules/instagram_extractor/instagram_extractor.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py index 294b4e7..d559c47 100644 --- a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py +++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py @@ -29,6 +29,9 @@ class InstagramExtractor(Extractor): # TODO: links to stories def setup(self) -> None: + logger.warning("Instagram Extractor is not actively maintained, and may not work as expected.") + logger.warning("Please consider using the Instagram Tbot Extractor or Instagram API Extractor instead.") + self.insta = instaloader.Instaloader( download_geotags=True, download_comments=True, From aacb874b562f1a828495e03d4ab23cd25d2d6565 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 12:21:56 +0400 Subject: [PATCH 22/27] removeprefix for www. is required here --- src/auto_archiver/utils/webdriver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index 0b5fa57..bd47f9d 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -48,7 +48,7 @@ class CookieSettingDriver(webdriver.Firefox): for name, value in cookie.split("="): self.driver.add_cookie({"name": name, "value": value}) elif self.cookie_jar: - domain = urlparse(url).netloc + domain = urlparse(url).netloc.removeprefix("www.") regex = re.compile(f"(www)?.?{domain}$") for cookie in self.cookie_jar: if regex.match(cookie.domain): From 2233af81f75e8bd9c89a1adaa42ccaeceb82fbf7 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 14:33:08 +0400 Subject: [PATCH 23/27] Version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 89bd4eb..6720ce8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [project] name = "auto-archiver" -version = "0.13.7" +version = "0.13.8" description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)." requires-python = ">=3.10,<3.13" From a066bf4ca95cc589eeb1727048b1561aa9de02fd Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Fri, 21 Mar 2025 14:47:50 +0400 Subject: [PATCH 24/27] Clean up comments --- tests/utils/test_urls.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/tests/utils/test_urls.py b/tests/utils/test_urls.py index 81600ce..7871847 100644 --- a/tests/utils/test_urls.py +++ b/tests/utils/test_urls.py @@ -70,36 +70,6 @@ def test_domain_for_url(url, domain): def test_remove_get_parameters(url, without_get): assert remove_get_parameters(url) == without_get - # IRRELEVANT_URLS = [ - # # favicons - # ("favicon",), - # # twitter profile pictures - # ("twimg.com/profile_images",), - # ("twimg.com", "default_profile_images"), - # # instagram profile pictures - # ("https://scontent.cdninstagram.com/", "150x150"), - # # instagram recurring images - # ("https://static.cdninstagram.com/rsrc.php/",), - # # telegram - # ("https://telegram.org/img/emoji/",), - # # youtube - # ("https://www.youtube.com/s/gaming/emoji/",), - # ("https://yt3.ggpht.com", "default-user="), - # ("https://www.youtube.com/s/search/audio/",), - # # ok - # ("https://ok.ru/res/i/",), - # ("https://vk.com/emoji/",), - # ("vk.com/images/",), - # ("vk.com/images/reaction/",), - # # wikipedia - # ("wikipedia.org/static",), - # ] - - # IRRELEVANT_ENDS_WITH = [ - # ".svg", # ignore SVGs - # ".ico", # ignore icons - # ] - @pytest.mark.parametrize( "url, relevant", From 7b454baa02c43ee89b21ddbd1e87f68d5b6e3758 Mon Sep 17 00:00:00 2001 From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 24 Mar 2025 10:49:36 +0000 Subject: [PATCH 25/27] Create dependabot.yml --- .github/dependabot.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c77d007 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" + From ad373ae73345a53e764f1148e181e3a6d7a63de1 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 17:52:30 +0400 Subject: [PATCH 26/27] Add explicit dependabots for pip/poetry, GH actiona and npm --- .github/dependabot.yml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c77d007..34e7a24 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,36 @@ version: 2 updates: - - package-ecosystem: "" # See documentation for possible values - directory: "/" # Location of package manifests + - package-ecosystem: "pip" + directory: "/" + groups: + python: + patterns: + - "*" + schedule: + interval: "weekly" + + - package-ecosystem: "github-actions" + directory: "/" + groups: + actions: + patterns: + - "*" schedule: interval: "weekly" + - package-ecosystem: "npm" + directory: "/scripts/settings/" + groups: + actions: + patterns: + - "*" + schedule: + interval: "weekly" + + - package-ecosystem: "docker" + # Look for a `Dockerfile` in the `root` directory + directory: "/" + # Check for updates once a week + schedule: + interval: "weekly" \ No newline at end of file From ace97ac7fd2c4b048bd2663d28645803e99e79fa Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Mon, 24 Mar 2025 18:00:14 +0400 Subject: [PATCH 27/27] Don't run ruff on non-python file changes --- .github/workflows/ruff.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml index 5ccbb1c..b030e80 100644 --- a/.github/workflows/ruff.yaml +++ b/.github/workflows/ruff.yaml @@ -3,8 +3,18 @@ name: Ruff Formatting & Linting on: push: branches: [ main ] + paths-ignore: + - "README.md" + - ".github" + - "poetry.lock" + - "scripts/settings" pull_request: branches: [ main ] + paths-ignore: + - "README.md" + - ".github" + - "poetry.lock" + - "scripts/settings" jobs: build: