orchestrator design structure

orchestrate.yaml

@@ -0,0 +1,48 @@
+steps:
+  # only 1 feeder allowed
+  # a feeder could be in an "infinite loop" for example: gsheets_infinite feeder which holds-> this could be an easy logic addiction by modifying for each to while not feeder.done() if it becomes necessary
+  feeder: gsheets_feeder # default -> only expects URL from CLI
+  archivers: # order matters
+    - tiktok
+    - telethon
+    - twitter
+    - instagram
+    - webarchive # this way it runs as a failsafe only
+  enrichments:
+    - screenshot
+    - wacz
+    - webarchive # this way it runs for every case, webarchive extends archiver and enrichment
+    - thumbnails
+  formatters:
+    - HTMLFormater
+    - PDFFormater
+  storages:
+    - local_storage
+    - s3
+  databases:
+    - gsheets_db
+    - mongo_db
+
+
+
+configurations:
+  gsheets_feeder:
+    - sheet: "Auto archiver"
+    - header: "" # defaults to 1 in GSheetsFeeder
+    - service_account: "secrets/service_account.json"
+  tiktok:
+    username: "abc"
+    password: "123"
+    token: "here"
+  screenshot:
+    width: 1280
+    height: 720
+  wacz:
+    profile: secrets/profile.tar.gz
+  webarchive:
+    api_key: "12345"
+  s3: 
+    - bucket: 123
+    - region: "nyc3"
+    - cdn: "{region}{bucket}"
+