aboutsummaryrefslogtreecommitdiff
path: root/jb/flow/events.py
diff options
context:
space:
mode:
authorstuppie2026-11-04 12:11:03 -0600
committerstuppie2026-11-04 12:11:03 -0600
commit671a7b1b02503cfa5c9aecd40c27a0c2729a285b (patch)
tree6cb1bd7c33e3e1d243ee5ae74decc57faaa5742d /jb/flow/events.py
parente99ec6b7e38113a2237dc1ff402d8f4337c651d5 (diff)
downloadamt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.tar.gz
amt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.zip
strip out AMT
Diffstat (limited to 'jb/flow/events.py')
-rw-r--r--jb/flow/events.py98
1 files changed, 0 insertions, 98 deletions
diff --git a/jb/flow/events.py b/jb/flow/events.py
deleted file mode 100644
index bee63fd..0000000
--- a/jb/flow/events.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import time
-from concurrent import futures
-from concurrent.futures import Executor, ThreadPoolExecutor
-from typing import cast
-
-import redis
-
-from jb.config import (
- CONSUMER_GROUP,
- CONSUMER_NAME,
- JB_EVENTS_STREAM,
-)
-from jb.decorators import LOG, get_redis
-from jb.flow.assignment_tasks import process_assignment_submitted
-from jb.flow.monitoring import emit_error_event
-from jb.models.event import MTurkEvent
-
-StreamMessages = list[tuple[str, list[tuple[bytes, dict[bytes, bytes]]]]]
-
-
-def process_mturk_events_task():
- executor = ThreadPoolExecutor(max_workers=5)
- create_consumer_group()
- while True:
- try:
- process_mturk_events(executor=executor)
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(1)
-
-
-def process_mturk_events(executor: Executor):
- while True:
- n = process_mturk_events_chunk(executor=executor)
- if n is None or n < 10:
- break
-
-
-def create_consumer_group():
- try:
- redis_client = get_redis()
- redis_client.xgroup_create(
- JB_EVENTS_STREAM, CONSUMER_GROUP, id="0", mkstream=True
- )
- except redis.exceptions.ResponseError as e:
- if "BUSYGROUP Consumer Group name already exists" in str(e):
- pass # group already exists
- else:
- raise
-
-
-def process_mturk_events_chunk(executor: Executor) -> int | None:
- redis_client = get_redis()
- msgs_raw = redis_client.xreadgroup(
- groupname=CONSUMER_GROUP,
- consumername=CONSUMER_NAME,
- streams={JB_EVENTS_STREAM: ">"},
- count=10,
- )
- if not msgs_raw:
- return None
-
- msgs = cast(StreamMessages, msgs_raw)[0][1] # The queue, we only have 1
-
- fs = []
- for msg in msgs:
- msg_id, data = msg
- msg_json: str = data["data"]
-
- event = MTurkEvent.model_validate_json(json_data=msg_json)
- if event.event_type == "AssignmentSubmitted":
- fs.append(
- executor.submit(process_assignment_submitted_event, event, str(msg_id))
- )
- else:
- LOG.info(f"Discarding {event}")
- redis_client.xdel(JB_EVENTS_STREAM, msg_id)
-
- futures.wait(fs, timeout=60)
- return len(msgs)
-
-
-def process_assignment_submitted_event(event: MTurkEvent, msg_id: str):
- from jb.decorators import AM, AMTM, BM, HM
-
- redis_client = get_redis()
-
- try:
- process_assignment_submitted(amtm=AMTM, am=AM, hm=HM, bm=BM, event=event)
- except Exception as e:
- LOG.exception(f"{event.amt_assignment_id=}, {e=}")
- emit_error_event(
- event_type="failed_process_assignment_submitted",
- amt_hit_type_id=event.amt_hit_type_id,
- )
-
- redis_client.xackdel(JB_EVENTS_STREAM, CONSUMER_GROUP, msg_id)