1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
import time
from concurrent import futures
from concurrent.futures import Executor, ThreadPoolExecutor
from typing import cast
import redis
from jb.config import (
CONSUMER_GROUP,
CONSUMER_NAME,
JB_EVENTS_STREAM,
)
from jb.decorators import LOG, REDIS
from jb.flow.assignment_tasks import process_assignment_submitted
from jb.flow.monitoring import emit_error_event
from jb.models.event import MTurkEvent
StreamMessages = list[tuple[str, list[tuple[bytes, dict[bytes, bytes]]]]]
def process_mturk_events_task():
executor = ThreadPoolExecutor(max_workers=5)
create_consumer_group()
while True:
try:
process_mturk_events(executor=executor)
except Exception as e:
LOG.exception(e)
finally:
time.sleep(1)
def process_mturk_events(executor: Executor):
while True:
n = process_mturk_events_chunk(executor=executor)
if n is None or n < 10:
break
def create_consumer_group():
try:
REDIS.xgroup_create(JB_EVENTS_STREAM, CONSUMER_GROUP, id="0", mkstream=True)
except redis.exceptions.ResponseError as e:
if "BUSYGROUP Consumer Group name already exists" in str(e):
pass # group already exists
else:
raise
def process_mturk_events_chunk(executor: Executor) -> int | None:
msgs_raw = REDIS.xreadgroup(
groupname=CONSUMER_GROUP,
consumername=CONSUMER_NAME,
streams={JB_EVENTS_STREAM: ">"},
count=10,
)
if not msgs_raw:
return None
msgs = cast(StreamMessages, msgs_raw)[0][1] # The queue, we only have 1
fs = []
for msg in msgs:
msg_id, data = msg
msg_json: str = data["data"]
event = MTurkEvent.model_validate_json(json_data=msg_json)
if event.event_type == "AssignmentSubmitted":
fs.append(
executor.submit(process_assignment_submitted_event, event, str(msg_id))
)
else:
LOG.info(f"Discarding {event}")
REDIS.xdel(JB_EVENTS_STREAM, msg_id)
futures.wait(fs, timeout=60)
return len(msgs)
def process_assignment_submitted_event(event: MTurkEvent, msg_id: str):
from jb.decorators import AM, AMTM, BM, HM
try:
process_assignment_submitted(amtm=AMTM, am=AM, hm=HM, bm=BM, event=event)
except Exception as e:
LOG.exception(f"{event.amt_assignment_id=}, {e=}")
emit_error_event(
event_type="failed_process_assignment_submitted",
amt_hit_type_id=event.amt_hit_type_id,
)
REDIS.xackdel(JB_EVENTS_STREAM, CONSUMER_GROUP, msg_id)
|
