rufei/events-plus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 33 Wiki Activity

there's a lot to unpack here

Browse Source
This commit is contained in:
Rufei Zhou
2025-04-10 05:09:57 -07:00
parent 1985a196b7
commit 1dfaac75d2
5 changed files with 556 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/events_plus/console.py
+5 -1
View File
@@ -1,7 +1,11 @@
from .scrapers.sakuracon import collect_sakuracon_events
from .scrapers.sakuracon import collect_sakuracon_events, clear_sakuracon_events
import asyncio
def generate_all():
loop = asyncio.new_event_loop()
loop.run_until_complete(collect_sakuracon_events())
def clear_all():
loop = asyncio.new_event_loop()
loop.run_until_complete(clear_sakuracon_events())
src/events_plus/scrapers/sakuracon.py
+219 -7
View File
@@ -1,20 +1,37 @@
import requests
from icalendar import Calendar, Event
from datetime import datetime
from datetime import datetime, timezone
from collections import defaultdict
import asyncio
import aiohttp
import json
import os
import base64
from google.oauth2 import service_account
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
EVENTNY_ENDPOINT = "https://www.eventeny.com/funcs/event/event-page-elements-2022-03-06.php"
SAK_BIZ_ID = "233997"
SAK_EVENT_ID = "13462"
BATCH_LIMIT = 500
async def collect_sakuracon_events():
async def clear_sakuracon_events(config_path="config/sakuracon.json"):
with open(config_path) as f:
cfg = json.load(f)
print(cfg)
clear_gcals(cfg)
async def collect_sakuracon_events(config_path="config/sakuracon.json"):
with open(config_path) as f:
cfg = json.load(f)
print(cfg)
events, tracks = await get_event_data()
cals = convert_events_to_icals(events, tracks)
cals = convert_events_to_icals(events, tracks, cfg)
write_ics(cals)
update_gcal(events, tracks, cfg)
async def get_event_data():
@@ -48,7 +65,7 @@ async def get_event_data():
all_events.extend(data['list'])
all_tracks.update(data['track'])
all_events = await insert_descriptions(all_events)
all_events = await insert_fields(all_events)
return all_events, all_tracks
@@ -67,7 +84,7 @@ async def get_description(eventid):
results = json.loads(data.decode())
return results["schedule"]["overview"]["description"] or ""
async def insert_descriptions(events):
async def insert_fields(events):
tasks = []
for event in events:
t = asyncio.create_task(
@@ -79,11 +96,25 @@ async def insert_descriptions(events):
for desc, event in zip(descs, events):
event["raw_description"] = desc
tags_str = ""
if 'hashtag_title' in event and event['hashtag_title']:
tags_str = f"Tags: {','.join(event['hashtag_title'])}\n"
event["description"] = f"Track: {event['tag_title']}\n{tags_str}\n{desc}"
return events
def convert_events_to_icals(all_events, all_tracks) -> dict[str, Calendar]:
def batch_create_handler(req_id, resp, exception):
if exception is not None:
print(exception)
return
print(f"Event created: g {resp['id']}")
def batch_exception_handler(req_id, resp, exception):
if exception is not None:
print(exception)
def convert_events_to_icals(all_events, all_tracks, cfg) -> dict[str, Calendar]:
# Group events by track_title
uid_prefix = cfg["uidPrefix"]
calendars = defaultdict(Calendar)
for event in all_events:
@@ -100,6 +131,7 @@ def convert_events_to_icals(all_events, all_tracks) -> dict[str, Calendar]:
# Create event
ical_event = Event()
ical_event.add("uid", uid_prefix + event["id"]) # we are assuming the ids generated by eventny are unique
ical_event.add("summary", event['title'])
ical_event.add("dtstart", datetime.fromisoformat(event['start_calendar']))
ical_event.add("dtend", datetime.fromisoformat(event['end_calendar']))
@@ -124,3 +156,183 @@ def write_ics(calendars, output_dir="output/sakuracon"):
with open(filename, 'wb') as f:
f.write(cal.to_ical())
print(f"Wrote: {filename}")
def update_gcal(scraped_events, tracks, cfg):
SCOPES = ["https://www.googleapis.com/auth/calendar.events"]
creds_b64 = os.getenv("GOOGLE_SERVICE_ACCOUNT_CREDS")
creds_body = json.loads(base64.b64decode(creds_b64))
creds = service_account.Credentials.from_service_account_info(creds_body, scopes=SCOPES)
tz_suffix = cfg["tzSuffix"]
gcal_map = cfg["calendars"]
uid_prefix = cfg["uidPrefix"]
uids = set([uid_prefix + e["id"] for e in scraped_events])
print(f"There seem to be {len(uids)} unique events by ical uids")
try:
service = build("calendar", "v3", credentials=creds)
# list all events in the gcals
gcal_events = []
gcalevent_gcal_map = {}
batch = service.new_batch_http_request()
batchlen = 0
for _, gcal in gcal_map.items():
page_token = None
while True:
resp = service.events().list(calendarId=gcal, maxResults=BATCH_LIMIT, pageToken=page_token, showDeleted=True).execute()
for e in resp["items"]:
#print(e.get("iCalUID"))
gcalevent_gcal_map[e["id"]] = gcal
gcal_events.extend(resp["items"])
page_token = resp.get("nextPageToken")
if not page_token:
break
print(f"Retrieved {len(gcal_events)} gcal events")
# associate icaluid
uid_gcalevent_map = {}
for event in gcal_events:
uid = event.get("iCalUID")
if uid and uid.startswith(uid_prefix):
uid_gcalevent_map[uid] = event
# go through scraped events and note changes from gcal (correlated by icaluid), if any
events_by_track = defaultdict(list)
for e in scraped_events:
events_by_track[e["tag_title"]].append(e)
for track_name, events in events_by_track.items():
batch = service.new_batch_http_request()
track_gcal_id = gcal_map[track_name]
for e in events:
ical_uid = uid_prefix + e["id"]
try: # sometimes events have weird start/end times that fail gcal's validation, but that shouldn't kill our vibe
e_content = {
"summary": e["title"],
"start": e["start_calendar"] + tz_suffix,
"end": e["end_calendar"] + tz_suffix,
"location": e['location'].replace('&', '&'),
"description": e["description"],
"status": "confirmed"
}
gcal_body = {
"summary": e["title"],
"iCalUID": ical_uid,
"start": {
"dateTime": e["start_calendar"] + tz_suffix,
},
"end": {
"dateTime": e["end_calendar"] + tz_suffix,
},
"location": e['location'].replace('&', '&'),
"description": e["description"],
"status": "confirmed"
}
if ical_uid in uid_gcalevent_map:
g_e = uid_gcalevent_map[ical_uid]
#print(g_e)
g_content = {
"summary": g_e.get("summary"),
"start": g_e["start"]["dateTime"],
"end": g_e["end"]["dateTime"],
"location": g_e.get("location"),
"description": g_e.get("description"),
"status": g_e["status"] # this will likely either be confirmed or cancelled, and because we are using icaluids, we have to fish the cancelled/trashed event out of the bin using this field
}
changed = e_content != g_content
# if changes exist, update existing event
if not changed:
print(f"Event {e['title']} seems to be the same, leaving as is")
else:
print(f"Updating event {e['title']} (ical: {ical_uid}/g: {g_e['id']})")
# print(e_content)
# print(g_content)
g_e.update(gcal_body)
batch.add(service.events().patch(calendarId=track_gcal_id, eventId=g_e["id"], body=g_e), callback=batch_exception_handler)
batchlen += 1
else: # if event is new, insert
print(f"Event with uid {ical_uid} seems to be new, adding...")
service.events().insert(calendarId=track_gcal_id, body=gcal_body).execute()
#batch.add(service.events().insert(calendarId=track_gcal_id, body=gcal_body), callback=batch_create_handler)
#batchlen += 1
if batchlen == BATCH_LIMIT:
batch.execute()
batchlen = 0
batch = service.new_batch_http_request()
except HttpError as error:
print(f"An error occurred: {error}")
print(gcal_body)
batch.execute()
batchlen = 0
# delete all gcal_events which are not in the events array
# we have the list of gcal events
# we have a set of event uids
# if the gcal event does not have a uid or its uid is not in the uids set, then delete
# but we need the cal of the gcal event in order to delete it
#print(gcalevent_gcal_map)
batch = service.new_batch_http_request()
batchlen = 0
for event in gcal_events:
#print(event)
#print(f"icaluid: {event.get('iCalUID')}")
if "iCalUID" not in event or event["iCalUID"] not in uids and event["status"] != "cancelled":
cal = gcalevent_gcal_map[event["id"]]
print(f"gcal event g {event['id']} not in latest scrape, deleting...")
batch.add(service.events().delete(calendarId=cal, eventId=event["id"]), callback=batch_exception_handler)
batchlen += 1
if batchlen == BATCH_LIMIT:
batch.execute()
batch = service.new_batch_http_request()
batchlen = 0
batch.execute()
batch = service.new_batch_http_request()
batchlen = 0
except HttpError as error:
print(f"An error occurred: {error}")
print(gcal_body)
def clear_gcals(cfg):
SCOPES = ["https://www.googleapis.com/auth/calendar.events"]
creds_b64 = os.getenv("GOOGLE_SERVICE_ACCOUNT_CREDS")
creds_body = json.loads(base64.b64decode(creds_b64))
creds = service_account.Credentials.from_service_account_info(creds_body, scopes=SCOPES)
gcal_map = cfg["calendars"]
try:
service = build("calendar", "v3", credentials=creds)
# list all events in the gcals
count = 0
for _, gcal in gcal_map.items():
page_token = None
while True:
batch = service.new_batch_http_request()
resp = service.events().list(calendarId=gcal, maxResults=BATCH_LIMIT, pageToken=page_token).execute()
for e in resp["items"]:
print(e.get("iCalUID"))
events = resp["items"]
print(events)
for e in events:
batch.add(service.events().delete(calendarId=gcal, eventId=e["id"]), callback=batch_exception_handler)
count += len(events)
batch.execute()
page_token = resp.get("nextPageToken")
if not page_token:
break
print(f"Removed {count} gcal events")
except HttpError as error:
print(f"An error occurred: {error}")