From 1dfaac75d253fa2a6f4377f21b64c79341063ef8 Mon Sep 17 00:00:00 2001 From: Rufei Zhou Date: Thu, 10 Apr 2025 05:09:57 -0700 Subject: [PATCH] there's a lot to unpack here --- config/sakuracon.json | 16 ++ poetry.lock | 312 +++++++++++++++++++++++++- pyproject.toml | 6 +- src/events_plus/console.py | 6 +- src/events_plus/scrapers/sakuracon.py | 226 ++++++++++++++++++- 5 files changed, 556 insertions(+), 10 deletions(-) create mode 100755 config/sakuracon.json diff --git a/config/sakuracon.json b/config/sakuracon.json new file mode 100755 index 0000000..4003cf1 --- /dev/null +++ b/config/sakuracon.json @@ -0,0 +1,16 @@ +{ + "calendars": { + "Sakura-Con Presents": "f2802717d1727828bf1bfe9f6cc35844a36ffff6daad7c3fa293ab35bf51c495@group.calendar.google.com", + "Fan Panel": "8062c73edfb8db11c385b255531d97ba7a30672a5e6ded4e24cd589b25282d65@group.calendar.google.com", + "Gaming": "46c95e247f9e449f2968b44edfe659560a232efbf57576efc9f15ba8668ee0ff@group.calendar.google.com", + "Main Stage": "ae20459e6d4c41b46496663b04ec87279fb1a0ed858351ea796d82c217e20579@group.calendar.google.com", + "Cultural Panel": "633adb5a3a96b984092c8e641dda4011b80a8a314d03fc0c6e941895caabf2d2@group.calendar.google.com", + "Guest of Honor": "da7b9a6845eb7868457147046aa388101ef2601cffbaf4d29f1a1c8d00a8bea9@group.calendar.google.com", + "Charity Auction": "a51250c24fd2c4fd2d4bbefc94c055eb1f392ccf89d93799d4a92ff2c319f436@group.calendar.google.com", + "Industry Panel": "fe3a9a7e1e839dd41ac934bb0b79287d4ff705f577f726d06b0f4249bae1ac40@group.calendar.google.com", + "Autographs": "1959b9a550cbdf73d7997c070009ea836c1af93d8330353bbfdbbc2f82aff27a@group.calendar.google.com", + "Summit Stage": "106d7620bbe3a8e14ee593758e7db9c71a311615e3a17ada938f50974b0b9ccf@group.calendar.google.com" + }, + "tzSuffix": "-07:00", + "uidPrefix": "sak2025-" +} \ No newline at end of file diff --git a/poetry.lock b/poetry.lock index 7683935..0a88fff 100755 --- a/poetry.lock +++ b/poetry.lock @@ -164,6 +164,18 @@ docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphi tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"] tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""] +[[package]] +name = "cachetools" +version = "5.5.2" +description = "Extensible memoizing collections and decorators" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, + {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, +] + [[package]] name = "certifi" version = "2025.1.31" @@ -396,6 +408,146 @@ files = [ {file = "frozenlist-1.5.0.tar.gz", hash = "sha256:81d5af29e61b9c8348e876d442253723928dce6433e0e76cd925cd83f1b4b817"}, ] +[[package]] +name = "google-api-core" +version = "2.24.2" +description = "Google API client core library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_api_core-2.24.2-py3-none-any.whl", hash = "sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9"}, + {file = "google_api_core-2.24.2.tar.gz", hash = "sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696"}, +] + +[package.dependencies] +google-auth = ">=2.14.1,<3.0.0" +googleapis-common-protos = ">=1.56.2,<2.0.0" +proto-plus = [ + {version = ">=1.25.0,<2.0.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.22.3,<2.0.0", markers = "python_version < \"3.13\""}, +] +protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" +requests = ">=2.18.0,<3.0.0" + +[package.extras] +async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"] +grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""] +grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] +grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] + +[[package]] +name = "google-api-python-client" +version = "2.166.0" +description = "Google API Client Library for Python" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_api_python_client-2.166.0-py2.py3-none-any.whl", hash = "sha256:dd8cc74d9fc18538ab05cbd2e93cb4f82382f910c5f6945db06c91f1deae6e45"}, + {file = "google_api_python_client-2.166.0.tar.gz", hash = "sha256:b8cf843bd9d736c134aef76cf1dc7a47c9283a2ef24267b97207b9dd43b30ef7"}, +] + +[package.dependencies] +google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0" +google-auth = ">=1.32.0,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0" +google-auth-httplib2 = ">=0.2.0,<1.0.0" +httplib2 = ">=0.19.0,<1.0.0" +uritemplate = ">=3.0.1,<5" + +[[package]] +name = "google-auth" +version = "2.38.0" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "google_auth-2.38.0-py2.py3-none-any.whl", hash = "sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a"}, + {file = "google_auth-2.38.0.tar.gz", hash = "sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4"}, +] + +[package.dependencies] +cachetools = ">=2.0.0,<6.0" +pyasn1-modules = ">=0.2.1" +rsa = ">=3.1.4,<5" + +[package.extras] +aiohttp = ["aiohttp (>=3.6.2,<4.0.0.dev0)", "requests (>=2.20.0,<3.0.0.dev0)"] +enterprise-cert = ["cryptography", "pyopenssl"] +pyjwt = ["cryptography (>=38.0.3)", "pyjwt (>=2.0)"] +pyopenssl = ["cryptography (>=38.0.3)", "pyopenssl (>=20.0.0)"] +reauth = ["pyu2f (>=0.1.5)"] +requests = ["requests (>=2.20.0,<3.0.0.dev0)"] + +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +description = "Google Authentication Library: httplib2 transport" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05"}, + {file = "google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d"}, +] + +[package.dependencies] +google-auth = "*" +httplib2 = ">=0.19.0" + +[[package]] +name = "google-auth-oauthlib" +version = "1.2.1" +description = "Google Authentication Library" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "google_auth_oauthlib-1.2.1-py2.py3-none-any.whl", hash = "sha256:2d58a27262d55aa1b87678c3ba7142a080098cbc2024f903c62355deb235d91f"}, + {file = "google_auth_oauthlib-1.2.1.tar.gz", hash = "sha256:afd0cad092a2eaa53cd8e8298557d6de1034c6cb4a740500b5357b648af97263"}, +] + +[package.dependencies] +google-auth = ">=2.15.0" +requests-oauthlib = ">=0.7.0" + +[package.extras] +tool = ["click (>=6.0.0)"] + +[[package]] +name = "googleapis-common-protos" +version = "1.69.2" +description = "Common protobufs used in Google APIs" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "googleapis_common_protos-1.69.2-py3-none-any.whl", hash = "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212"}, + {file = "googleapis_common_protos-1.69.2.tar.gz", hash = "sha256:3e1b904a27a33c821b4b749fd31d334c0c9c30e6113023d495e48979a3dc9c5f"}, +] + +[package.dependencies] +protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0" + +[package.extras] +grpc = ["grpcio (>=1.44.0,<2.0.0)"] + +[[package]] +name = "httplib2" +version = "0.22.0" +description = "A comprehensive HTTP client library." +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +groups = ["main"] +files = [ + {file = "httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, + {file = "httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, +] + +[package.dependencies] +pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""} + [[package]] name = "icalendar" version = "6.1.3" @@ -535,6 +687,23 @@ files = [ [package.dependencies] typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} +[[package]] +name = "oauthlib" +version = "3.2.2" +description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] + +[package.extras] +rsa = ["cryptography (>=3.0.0)"] +signals = ["blinker (>=1.4.0)"] +signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] + [[package]] name = "propcache" version = "0.3.1" @@ -643,6 +812,85 @@ files = [ {file = "propcache-0.3.1.tar.gz", hash = "sha256:40d980c33765359098837527e18eddefc9a24cea5b45e078a7f3bb5b032c6ecf"}, ] +[[package]] +name = "proto-plus" +version = "1.26.1" +description = "Beautiful, Pythonic protocol buffers" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"}, + {file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"}, +] + +[package.dependencies] +protobuf = ">=3.19.0,<7.0.0" + +[package.extras] +testing = ["google-api-core (>=1.31.5)"] + +[[package]] +name = "protobuf" +version = "6.30.2" +description = "" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "protobuf-6.30.2-cp310-abi3-win32.whl", hash = "sha256:b12ef7df7b9329886e66404bef5e9ce6a26b54069d7f7436a0853ccdeb91c103"}, + {file = "protobuf-6.30.2-cp310-abi3-win_amd64.whl", hash = "sha256:7653c99774f73fe6b9301b87da52af0e69783a2e371e8b599b3e9cb4da4b12b9"}, + {file = "protobuf-6.30.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:0eb523c550a66a09a0c20f86dd554afbf4d32b02af34ae53d93268c1f73bc65b"}, + {file = "protobuf-6.30.2-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:50f32cc9fd9cb09c783ebc275611b4f19dfdfb68d1ee55d2f0c7fa040df96815"}, + {file = "protobuf-6.30.2-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:4f6c687ae8efae6cf6093389a596548214467778146b7245e886f35e1485315d"}, + {file = "protobuf-6.30.2-cp39-cp39-win32.whl", hash = "sha256:524afedc03b31b15586ca7f64d877a98b184f007180ce25183d1a5cb230ee72b"}, + {file = "protobuf-6.30.2-cp39-cp39-win_amd64.whl", hash = "sha256:acec579c39c88bd8fbbacab1b8052c793efe83a0a5bd99db4a31423a25c0a0e2"}, + {file = "protobuf-6.30.2-py3-none-any.whl", hash = "sha256:ae86b030e69a98e08c77beab574cbcb9fff6d031d57209f574a5aea1445f4b51"}, + {file = "protobuf-6.30.2.tar.gz", hash = "sha256:35c859ae076d8c56054c25b59e5e59638d86545ed6e2b6efac6be0b6ea3ba048"}, +] + +[[package]] +name = "pyasn1" +version = "0.6.1" +description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, + {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +description = "A collection of ASN.1-based protocols modules" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a"}, + {file = "pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6"}, +] + +[package.dependencies] +pyasn1 = ">=0.6.1,<0.7.0" + +[[package]] +name = "pyparsing" +version = "3.2.3" +description = "pyparsing module - Classes and methods to define and execute parsing grammars" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, + {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, +] + +[package.extras] +diagrams = ["jinja2", "railroad-diagrams"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -692,6 +940,56 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-oauthlib" +version = "2.0.0" +description = "OAuthlib authentication support for Requests." +optional = false +python-versions = ">=3.4" +groups = ["main"] +files = [ + {file = "requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9"}, + {file = "requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36"}, +] + +[package.dependencies] +oauthlib = ">=3.0.0" +requests = ">=2.0.0" + +[package.extras] +rsa = ["oauthlib[signedtoken] (>=3.0.0)"] + +[[package]] +name = "rsa" +version = "4.2" +description = "Pure-Python RSA implementation" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version >= \"3.13\"" +files = [ + {file = "rsa-4.2.tar.gz", hash = "sha256:aaefa4b84752e3e99bd8333a2e1e3e7a7da64614042bd66f775573424370108a"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + +[[package]] +name = "rsa" +version = "4.9" +description = "Pure-Python RSA implementation" +optional = false +python-versions = ">=3.6,<4" +groups = ["main"] +markers = "python_version < \"3.13\"" +files = [ + {file = "rsa-4.9-py3-none-any.whl", hash = "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7"}, + {file = "rsa-4.9.tar.gz", hash = "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"}, +] + +[package.dependencies] +pyasn1 = ">=0.1.3" + [[package]] name = "setuptools" version = "75.3.2" @@ -750,6 +1048,18 @@ files = [ {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, ] +[[package]] +name = "uritemplate" +version = "4.1.1" +description = "Implementation of RFC 6570 URI Templates" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "uritemplate-4.1.1-py2.py3-none-any.whl", hash = "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"}, + {file = "uritemplate-4.1.1.tar.gz", hash = "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0"}, +] + [[package]] name = "urllib3" version = "2.2.3" @@ -928,4 +1238,4 @@ testing = ["coverage[toml]", "zope.event", "zope.testing"] [metadata] lock-version = "2.1" python-versions = ">=3.9" -content-hash = "b5f9acef18a15409718a71521e2a39a7cae20793f55425d28e46f6b37ad3b3cd" +content-hash = "4491d7bcb43340cdddcedd2a3111165ce0a43f782b569592d08405e0eedb3f7e" diff --git a/pyproject.toml b/pyproject.toml index 4d6b682..8314669 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,11 +11,15 @@ dependencies = [ "icalendar (>=6.1.3,<7.0.0)", "datetime (>=5.5,<6.0)", "requests (>=2.32.3,<3.0.0)", - "aiohttp (>=3.11.16,<4.0.0)" + "aiohttp (>=3.11.16,<4.0.0)", + "google-api-python-client (>=2.166.0,<3.0.0)", + "google-auth-httplib2 (>=0.2.0,<0.3.0)", + "google-auth-oauthlib (>=1.2.1,<2.0.0)" ] [project.scripts] generate-ics = 'events_plus.console:generate_all' +clear-gcals = 'events_plus.console:clear_all' [build-system] diff --git a/src/events_plus/console.py b/src/events_plus/console.py index 960f405..32711cd 100755 --- a/src/events_plus/console.py +++ b/src/events_plus/console.py @@ -1,7 +1,11 @@ -from .scrapers.sakuracon import collect_sakuracon_events +from .scrapers.sakuracon import collect_sakuracon_events, clear_sakuracon_events import asyncio def generate_all(): loop = asyncio.new_event_loop() loop.run_until_complete(collect_sakuracon_events()) + +def clear_all(): + loop = asyncio.new_event_loop() + loop.run_until_complete(clear_sakuracon_events()) \ No newline at end of file diff --git a/src/events_plus/scrapers/sakuracon.py b/src/events_plus/scrapers/sakuracon.py index e46afc7..22a1408 100755 --- a/src/events_plus/scrapers/sakuracon.py +++ b/src/events_plus/scrapers/sakuracon.py @@ -1,20 +1,37 @@ import requests from icalendar import Calendar, Event -from datetime import datetime +from datetime import datetime, timezone from collections import defaultdict import asyncio import aiohttp import json import os +import base64 + +from google.oauth2 import service_account +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + EVENTNY_ENDPOINT = "https://www.eventeny.com/funcs/event/event-page-elements-2022-03-06.php" SAK_BIZ_ID = "233997" SAK_EVENT_ID = "13462" +BATCH_LIMIT = 500 -async def collect_sakuracon_events(): +async def clear_sakuracon_events(config_path="config/sakuracon.json"): + with open(config_path) as f: + cfg = json.load(f) + print(cfg) + clear_gcals(cfg) + +async def collect_sakuracon_events(config_path="config/sakuracon.json"): + with open(config_path) as f: + cfg = json.load(f) + print(cfg) events, tracks = await get_event_data() - cals = convert_events_to_icals(events, tracks) + cals = convert_events_to_icals(events, tracks, cfg) write_ics(cals) + update_gcal(events, tracks, cfg) async def get_event_data(): @@ -48,7 +65,7 @@ async def get_event_data(): all_events.extend(data['list']) all_tracks.update(data['track']) - all_events = await insert_descriptions(all_events) + all_events = await insert_fields(all_events) return all_events, all_tracks @@ -67,7 +84,7 @@ async def get_description(eventid): results = json.loads(data.decode()) return results["schedule"]["overview"]["description"] or "" -async def insert_descriptions(events): +async def insert_fields(events): tasks = [] for event in events: t = asyncio.create_task( @@ -79,11 +96,25 @@ async def insert_descriptions(events): for desc, event in zip(descs, events): event["raw_description"] = desc + tags_str = "" + if 'hashtag_title' in event and event['hashtag_title']: + tags_str = f"Tags: {','.join(event['hashtag_title'])}\n" + event["description"] = f"Track: {event['tag_title']}\n{tags_str}\n{desc}" return events - -def convert_events_to_icals(all_events, all_tracks) -> dict[str, Calendar]: +def batch_create_handler(req_id, resp, exception): + if exception is not None: + print(exception) + return + print(f"Event created: g {resp['id']}") + +def batch_exception_handler(req_id, resp, exception): + if exception is not None: + print(exception) + +def convert_events_to_icals(all_events, all_tracks, cfg) -> dict[str, Calendar]: # Group events by track_title + uid_prefix = cfg["uidPrefix"] calendars = defaultdict(Calendar) for event in all_events: @@ -100,6 +131,7 @@ def convert_events_to_icals(all_events, all_tracks) -> dict[str, Calendar]: # Create event ical_event = Event() + ical_event.add("uid", uid_prefix + event["id"]) # we are assuming the ids generated by eventny are unique ical_event.add("summary", event['title']) ical_event.add("dtstart", datetime.fromisoformat(event['start_calendar'])) ical_event.add("dtend", datetime.fromisoformat(event['end_calendar'])) @@ -124,3 +156,183 @@ def write_ics(calendars, output_dir="output/sakuracon"): with open(filename, 'wb') as f: f.write(cal.to_ical()) print(f"Wrote: {filename}") + +def update_gcal(scraped_events, tracks, cfg): + SCOPES = ["https://www.googleapis.com/auth/calendar.events"] + creds_b64 = os.getenv("GOOGLE_SERVICE_ACCOUNT_CREDS") + creds_body = json.loads(base64.b64decode(creds_b64)) + creds = service_account.Credentials.from_service_account_info(creds_body, scopes=SCOPES) + tz_suffix = cfg["tzSuffix"] + gcal_map = cfg["calendars"] + uid_prefix = cfg["uidPrefix"] + + uids = set([uid_prefix + e["id"] for e in scraped_events]) + print(f"There seem to be {len(uids)} unique events by ical uids") + + try: + service = build("calendar", "v3", credentials=creds) + # list all events in the gcals + gcal_events = [] + gcalevent_gcal_map = {} + batch = service.new_batch_http_request() + batchlen = 0 + for _, gcal in gcal_map.items(): + page_token = None + while True: + resp = service.events().list(calendarId=gcal, maxResults=BATCH_LIMIT, pageToken=page_token, showDeleted=True).execute() + for e in resp["items"]: + #print(e.get("iCalUID")) + gcalevent_gcal_map[e["id"]] = gcal + + gcal_events.extend(resp["items"]) + page_token = resp.get("nextPageToken") + if not page_token: + break + + print(f"Retrieved {len(gcal_events)} gcal events") + + # associate icaluid + uid_gcalevent_map = {} + for event in gcal_events: + uid = event.get("iCalUID") + if uid and uid.startswith(uid_prefix): + uid_gcalevent_map[uid] = event + + # go through scraped events and note changes from gcal (correlated by icaluid), if any + events_by_track = defaultdict(list) + for e in scraped_events: + events_by_track[e["tag_title"]].append(e) + + + for track_name, events in events_by_track.items(): + batch = service.new_batch_http_request() + track_gcal_id = gcal_map[track_name] + for e in events: + ical_uid = uid_prefix + e["id"] + try: # sometimes events have weird start/end times that fail gcal's validation, but that shouldn't kill our vibe + e_content = { + "summary": e["title"], + "start": e["start_calendar"] + tz_suffix, + "end": e["end_calendar"] + tz_suffix, + "location": e['location'].replace('&', '&'), + "description": e["description"], + "status": "confirmed" + } + gcal_body = { + "summary": e["title"], + "iCalUID": ical_uid, + "start": { + "dateTime": e["start_calendar"] + tz_suffix, + }, + "end": { + "dateTime": e["end_calendar"] + tz_suffix, + }, + "location": e['location'].replace('&', '&'), + "description": e["description"], + "status": "confirmed" + } + + if ical_uid in uid_gcalevent_map: + g_e = uid_gcalevent_map[ical_uid] + #print(g_e) + g_content = { + "summary": g_e.get("summary"), + "start": g_e["start"]["dateTime"], + "end": g_e["end"]["dateTime"], + "location": g_e.get("location"), + "description": g_e.get("description"), + "status": g_e["status"] # this will likely either be confirmed or cancelled, and because we are using icaluids, we have to fish the cancelled/trashed event out of the bin using this field + } + + changed = e_content != g_content + + # if changes exist, update existing event + if not changed: + print(f"Event {e['title']} seems to be the same, leaving as is") + else: + print(f"Updating event {e['title']} (ical: {ical_uid}/g: {g_e['id']})") + # print(e_content) + # print(g_content) + g_e.update(gcal_body) + batch.add(service.events().patch(calendarId=track_gcal_id, eventId=g_e["id"], body=g_e), callback=batch_exception_handler) + batchlen += 1 + else: # if event is new, insert + print(f"Event with uid {ical_uid} seems to be new, adding...") + service.events().insert(calendarId=track_gcal_id, body=gcal_body).execute() + #batch.add(service.events().insert(calendarId=track_gcal_id, body=gcal_body), callback=batch_create_handler) + #batchlen += 1 + + if batchlen == BATCH_LIMIT: + batch.execute() + batchlen = 0 + batch = service.new_batch_http_request() + except HttpError as error: + print(f"An error occurred: {error}") + print(gcal_body) + batch.execute() + batchlen = 0 + + # delete all gcal_events which are not in the events array + # we have the list of gcal events + # we have a set of event uids + # if the gcal event does not have a uid or its uid is not in the uids set, then delete + # but we need the cal of the gcal event in order to delete it + + #print(gcalevent_gcal_map) + + batch = service.new_batch_http_request() + batchlen = 0 + for event in gcal_events: + #print(event) + #print(f"icaluid: {event.get('iCalUID')}") + if "iCalUID" not in event or event["iCalUID"] not in uids and event["status"] != "cancelled": + cal = gcalevent_gcal_map[event["id"]] + print(f"gcal event g {event['id']} not in latest scrape, deleting...") + + batch.add(service.events().delete(calendarId=cal, eventId=event["id"]), callback=batch_exception_handler) + batchlen += 1 + if batchlen == BATCH_LIMIT: + batch.execute() + batch = service.new_batch_http_request() + batchlen = 0 + batch.execute() + batch = service.new_batch_http_request() + batchlen = 0 + + except HttpError as error: + print(f"An error occurred: {error}") + print(gcal_body) + + +def clear_gcals(cfg): + SCOPES = ["https://www.googleapis.com/auth/calendar.events"] + creds_b64 = os.getenv("GOOGLE_SERVICE_ACCOUNT_CREDS") + creds_body = json.loads(base64.b64decode(creds_b64)) + creds = service_account.Credentials.from_service_account_info(creds_body, scopes=SCOPES) + gcal_map = cfg["calendars"] + try: + service = build("calendar", "v3", credentials=creds) + # list all events in the gcals + count = 0 + + for _, gcal in gcal_map.items(): + page_token = None + while True: + batch = service.new_batch_http_request() + resp = service.events().list(calendarId=gcal, maxResults=BATCH_LIMIT, pageToken=page_token).execute() + for e in resp["items"]: + print(e.get("iCalUID")) + + events = resp["items"] + print(events) + for e in events: + batch.add(service.events().delete(calendarId=gcal, eventId=e["id"]), callback=batch_exception_handler) + count += len(events) + batch.execute() + page_token = resp.get("nextPageToken") + if not page_token: + break + + print(f"Removed {count} gcal events") + except HttpError as error: + print(f"An error occurred: {error}") \ No newline at end of file