use different ical prefix, better print diffs

config/sakuracon.json

@@ -12,5 +12,5 @@
         "Summit Stage": "106d7620bbe3a8e14ee593758e7db9c71a311615e3a17ada938f50974b0b9ccf@group.calendar.google.com"
     },
     "tzSuffix": "-07:00",
-    "uidPrefix": "sak2025-"
+    "uidPrefix": "sakuracon2025-"
 }

poetry.lock

@@ -582,6 +582,24 @@ files = [
 [package.extras]
 all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"]
 
+[[package]]
+name = "jsondiff"
+version = "2.2.1"
+description = "Diff JSON and JSON-like structures in Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "jsondiff-2.2.1-py3-none-any.whl", hash = "sha256:b1f0f7e2421881848b1d556d541ac01a91680cfcc14f51a9b62cdf4da0e56722"},
+    {file = "jsondiff-2.2.1.tar.gz", hash = "sha256:658d162c8a86ba86de26303cd86a7b37e1b2c1ec98b569a60e2ca6180545f7fe"},
+]
+
+[package.dependencies]
+pyyaml = "*"
+
+[package.extras]
+dev = ["build", "hypothesis", "pytest", "setuptools-scm"]
+
 [[package]]
 name = "multidict"
 version = "6.3.2"
@@ -918,6 +936,69 @@ files = [
     {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"},
 ]
 
+[[package]]
+name = "pyyaml"
+version = "6.0.2"
+description = "YAML parser and emitter for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
+    {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"},
+    {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"},
+    {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"},
+    {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"},
+    {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"},
+    {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"},
+    {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"},
+    {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"},
+    {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"},
+    {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"},
+    {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"},
+    {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"},
+    {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"},
+    {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"},
+    {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"},
+    {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"},
+    {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"},
+    {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"},
+    {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"},
+    {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"},
+    {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"},
+    {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"},
+    {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"},
+    {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"},
+    {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"},
+]
+
 [[package]]
 name = "requests"
 version = "2.32.3"
@@ -1238,4 +1319,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.9"
-content-hash = "4491d7bcb43340cdddcedd2a3111165ce0a43f782b569592d08405e0eedb3f7e"
+content-hash = "06976e001b4080474473ef0cb78ee3ab086eea319c391f759d74d72206e2bfa3"

pyproject.toml

@@ -14,7 +14,8 @@ dependencies = [
     "aiohttp (>=3.11.16,<4.0.0)",
     "google-api-python-client (>=2.166.0,<3.0.0)",
     "google-auth-httplib2 (>=0.2.0,<0.3.0)",
-    "google-auth-oauthlib (>=1.2.1,<2.0.0)"
+    "google-auth-oauthlib (>=1.2.1,<2.0.0)",
+    "jsondiff (>=2.2.1,<3.0.0)"
 ]
 
 [project.scripts]

src/events_plus/scrapers/sakuracon.py

@@ -7,7 +7,8 @@ import aiohttp
 import json
 import os
 import base64
-import time
+import jsondiff as jd
+import pprint
 
 from google.oauth2 import service_account
 from googleapiclient.discovery import build
@@ -22,13 +23,14 @@ BATCH_LIMIT = 190
 async def clear_sakuracon_events(config_path="config/sakuracon.json"):
     with open(config_path) as f:
         cfg = json.load(f)
-        print(cfg)
+        print("using config:")
+        pprint.pp(cfg, indent=2)
         clear_gcals(cfg)
 
 async def collect_sakuracon_events(config_path="config/sakuracon.json"):
     with open(config_path) as f:
         cfg = json.load(f)
-        print(cfg)
+        pprint.pp(cfg, indent=2)
     events, tracks = await get_event_data()
     cals = convert_events_to_icals(events, tracks, cfg)
     write_ics(cals)
@@ -103,15 +105,16 @@ async def insert_fields(events):
         event["description"] = f"Track: {event['tag_title']}\n{tags_str}\n{desc}"
     return events
 
-def batch_create_handler(req_id, resp, exception):
+def batch_create_handler(_, resp, exception):
     if exception is not None:
         print(exception)
         return
     print(f"Event created: g {resp['id']}")
 
-def batch_exception_handler(req_id, resp, exception):
+def batch_exception_handler(_, resp, exception):
     if exception is not None:
-        print(exception)
+        pprint.pp(resp)
+        pprint.pp(exception)
 
 def convert_events_to_icals(all_events, all_tracks, cfg) -> dict[str, Calendar]:
     # Group events by track_title
@@ -211,6 +214,12 @@ def update_gcal(scraped_events, tracks, cfg):
             for e in events:
                 ical_uid = uid_prefix + e["id"]
                 try: # sometimes events have weird start/end times that fail gcal's validation, but that shouldn't kill our vibe
+                    start = datetime.fromisoformat(e["start_calendar"])
+                    end = datetime.fromisoformat(e["end_calendar"])
+
+                    if start >= end:
+                        raise Exception((f"Event {e['title']} (uid: {ical_uid}) has an invalid duration, {start} -> {end}"))
+
                     e_content = {
                         "summary": e["title"],
                         "start": e["start_calendar"] + tz_suffix,
@@ -249,33 +258,32 @@ def update_gcal(scraped_events, tracks, cfg):
                     
                         # if changes exist, update existing event
                         if not changed:
-                            print(f"Event {e['title']} seems to be the same, leaving as is")
+                            print(f"Event {e['title']} (uid: {ical_uid}) seems to be the same, leaving as is")
                         else:
                             print(f"Updating event {e['title']} (ical: {ical_uid}/g: {g_e['id']})")
+                            pprint.pp(jd.diff(g_content, e_content, syntax="symmetric"), depth=3)
                             # print(e_content)
                             # print(g_content)
                             g_e.update(gcal_body)
                             batch.add(service.events().patch(calendarId=track_gcal_id, eventId=g_e["id"], body=g_e), callback=batch_exception_handler)
                             batchlen += 1
                     else: # if event is new, insert
-                        print(f"Event with uid {ical_uid} seems to be new, adding...")
+                        print(f"Event {e['title']} (uid: {ical_uid}) seems to be new, adding...")
-                        service.events().insert(calendarId=track_gcal_id, body=gcal_body).execute()
+                        #service.events().insert(calendarId=track_gcal_id, body=gcal_body).execute()
-                        #batch.add(service.events().insert(calendarId=track_gcal_id, body=gcal_body), callback=batch_create_handler)
+                        batch.add(service.events().insert(calendarId=track_gcal_id, body=gcal_body), callback=batch_create_handler)
-                        #batchlen += 1
+                        batchlen += 1
 
                     if batchlen == BATCH_LIMIT:
                         batch.execute()
                         batchlen = 0
                         batch = service.new_batch_http_request()
-                        print("batch zzz")
-                        time.sleep(60)
                 except HttpError as error:
-                    print(f"An error occurred: {error}")
+                    print(f"An error occurred while processing the event: {error}")
                     print(gcal_body)
+                except Exception as error:
+                    print(f"An error occurred while processing the event: {error}")
             batch.execute()
             batchlen = 0 
-            print("batch zzz")
-            time.sleep(60)
         
         # delete all gcal_events which are not in the events array
         # we have the list of gcal events
@@ -285,24 +293,13 @@ def update_gcal(scraped_events, tracks, cfg):
         
         #print(gcalevent_gcal_map)
 
-        batch = service.new_batch_http_request()
+        # dont bother batching the deletes b/c cal has to be same in batch, lazy...
-        batchlen = 0
         for event in gcal_events:
-            #print(event)
-            #print(f"icaluid: {event.get('iCalUID')}")
             if "iCalUID" not in event or event["iCalUID"] not in uids and event["status"] != "cancelled":
                 cal = gcalevent_gcal_map[event["id"]]
                 print(f"gcal event g {event['id']} not in latest scrape, deleting...")
 
-                batch.add(service.events().delete(calendarId=cal, eventId=event["id"]), callback=batch_exception_handler)
+                service.events().delete(calendarId=cal, eventId=event["id"]).execute()
-                batchlen += 1
-                if batchlen == BATCH_LIMIT:
-                    batch.execute()
-                    batch = service.new_batch_http_request()
-                    batchlen = 0
-        batch.execute()
-        batch = service.new_batch_http_request()
-        batchlen = 0
 
     except HttpError as error:
         print(f"An error occurred: {error}")