aboutsummaryrefslogtreecommitdiff
path: root/jb/flow
diff options
context:
space:
mode:
Diffstat (limited to 'jb/flow')
-rw-r--r--jb/flow/__init__.py0
-rw-r--r--jb/flow/assignment_tasks.py124
-rw-r--r--jb/flow/events.py98
-rw-r--r--jb/flow/maintenance.py31
-rw-r--r--jb/flow/monitoring.py168
-rw-r--r--jb/flow/setup_tasks.py35
-rw-r--r--jb/flow/tasks.py109
7 files changed, 0 insertions, 565 deletions
diff --git a/jb/flow/__init__.py b/jb/flow/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/jb/flow/__init__.py
+++ /dev/null
diff --git a/jb/flow/assignment_tasks.py b/jb/flow/assignment_tasks.py
deleted file mode 100644
index b3c820a..0000000
--- a/jb/flow/assignment_tasks.py
+++ /dev/null
@@ -1,124 +0,0 @@
-from jb.decorators import LOG
-from jb.flow.monitoring import emit_assignment_event, emit_error_event
-from jb.managers.amt import (
- REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- AMTManager,
-)
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
-from jb.managers.hit import HitManager
-from jb.models.assignment import Assignment
-from jb.models.definitions import AssignmentStatus
-from jb.models.event import MTurkEvent
-
-
-def process_assignment_submitted(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- event: MTurkEvent,
-) -> None:
- """
- Reject any submitted assignments
- """
-
- #
- # Step 1: Attempt to get the Assignment out of the API
- #
- LOG.info(f"{event=}")
-
- # This is the assignment model from AMT. In the DB, we should only have
- # the AssignmentStub
- # This call is hitting AMT, not our db.
- # The API won't even return is unless it has been submitted. So if this
- # fails, we don't need to do anything (i.e. reject it).
- assignment = amtm.get_assignment_if_exists(
- amt_assignment_id=event.amt_assignment_id
- )
-
- if assignment is None:
- # It is not found in amt, either it is invalid, not yet submitted, or
- # already been approved/rejected, so we just do nothing ...
- # todo: maybe we confirm its state matches what we have in the db
- LOG.warning(f"No assignment found on AMT: {event.amt_assignment_id}")
- emit_error_event(
- event_type="assignment_not_found_in_amt",
- amt_hit_type_id=event.amt_hit_type_id,
- )
- return
-
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
-
- reject_assignment(
- amtm=amtm,
- am=am,
- hm=hm,
- amt_assignment_id=assignment.amt_assignment_id,
- msg=REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- amt_hit_type_id=hit.amt_hit_type_id,
- )
- review_hit(amtm=amtm, hm=hm, assignment=assignment)
- return
-
-
-def review_hit(amtm: AMTManager, hm: HitManager, assignment: Assignment) -> None:
- # Reviewable to Reviewing
- amtm.update_hit_review_status(amt_hit_id=assignment.amt_hit_id, revert=False)
- hit, _ = amtm.get_hit_if_exists(amt_hit_id=assignment.amt_hit_id)
-
- if hit is None:
- LOG.warning(f"Hit not found when trying to review hit: {assignment.amt_hit_id}")
- return
-
- # Update the db
- hm.update_hit(hit)
-
- return
-
-
-def reject_assignment(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- amt_assignment_id: str,
- msg: str,
- amt_hit_type_id: str,
-) -> Assignment:
- # Reject in AMT, update db
-
- res = amtm.reject_assignment_if_possible(
- amt_assignment_id=amt_assignment_id, msg=msg
- )
- if res is None:
- # We failed to reject this assignment. This happens if the assignment
- # is already approved or already rejected.
- emit_error_event(
- event_type="failed_to_reject_assignment",
- amt_hit_type_id=amt_hit_type_id,
- )
- LOG.exception(f"Failed to reject assignment: {amt_assignment_id}")
-
- # We just rejected this assignment, get it from amazon again
- assignment = amtm.get_assignment(amt_assignment_id=amt_assignment_id)
- assert assignment.status == AssignmentStatus.Rejected
-
- # And update the db. The assignment may not actually exist in the db (if
- # a baddie intercepted it and is trying to game us). So, we might
- # need to create as assignment first ...
- stub = am.get_stub_if_exists(amt_assignment_id=assignment.amt_assignment_id)
- if stub is None:
- LOG.warning(
- f"Rejected assignment doesn't exist in DB. Creating ... : {amt_assignment_id}"
- )
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
- assignment.hit_id = hit.id
- am.create(assignment=assignment)
- am.reject(assignment=assignment)
- emit_assignment_event(
- status=AssignmentStatus.Rejected, amt_hit_type_id=amt_hit_type_id, reason=msg
- )
- LOG.warning(f"Rejected assignment: {amt_assignment_id}")
- return assignment
diff --git a/jb/flow/events.py b/jb/flow/events.py
deleted file mode 100644
index bee63fd..0000000
--- a/jb/flow/events.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import time
-from concurrent import futures
-from concurrent.futures import Executor, ThreadPoolExecutor
-from typing import cast
-
-import redis
-
-from jb.config import (
- CONSUMER_GROUP,
- CONSUMER_NAME,
- JB_EVENTS_STREAM,
-)
-from jb.decorators import LOG, get_redis
-from jb.flow.assignment_tasks import process_assignment_submitted
-from jb.flow.monitoring import emit_error_event
-from jb.models.event import MTurkEvent
-
-StreamMessages = list[tuple[str, list[tuple[bytes, dict[bytes, bytes]]]]]
-
-
-def process_mturk_events_task():
- executor = ThreadPoolExecutor(max_workers=5)
- create_consumer_group()
- while True:
- try:
- process_mturk_events(executor=executor)
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(1)
-
-
-def process_mturk_events(executor: Executor):
- while True:
- n = process_mturk_events_chunk(executor=executor)
- if n is None or n < 10:
- break
-
-
-def create_consumer_group():
- try:
- redis_client = get_redis()
- redis_client.xgroup_create(
- JB_EVENTS_STREAM, CONSUMER_GROUP, id="0", mkstream=True
- )
- except redis.exceptions.ResponseError as e:
- if "BUSYGROUP Consumer Group name already exists" in str(e):
- pass # group already exists
- else:
- raise
-
-
-def process_mturk_events_chunk(executor: Executor) -> int | None:
- redis_client = get_redis()
- msgs_raw = redis_client.xreadgroup(
- groupname=CONSUMER_GROUP,
- consumername=CONSUMER_NAME,
- streams={JB_EVENTS_STREAM: ">"},
- count=10,
- )
- if not msgs_raw:
- return None
-
- msgs = cast(StreamMessages, msgs_raw)[0][1] # The queue, we only have 1
-
- fs = []
- for msg in msgs:
- msg_id, data = msg
- msg_json: str = data["data"]
-
- event = MTurkEvent.model_validate_json(json_data=msg_json)
- if event.event_type == "AssignmentSubmitted":
- fs.append(
- executor.submit(process_assignment_submitted_event, event, str(msg_id))
- )
- else:
- LOG.info(f"Discarding {event}")
- redis_client.xdel(JB_EVENTS_STREAM, msg_id)
-
- futures.wait(fs, timeout=60)
- return len(msgs)
-
-
-def process_assignment_submitted_event(event: MTurkEvent, msg_id: str):
- from jb.decorators import AM, AMTM, BM, HM
-
- redis_client = get_redis()
-
- try:
- process_assignment_submitted(amtm=AMTM, am=AM, hm=HM, bm=BM, event=event)
- except Exception as e:
- LOG.exception(f"{event.amt_assignment_id=}, {e=}")
- emit_error_event(
- event_type="failed_process_assignment_submitted",
- amt_hit_type_id=event.amt_hit_type_id,
- )
-
- redis_client.xackdel(JB_EVENTS_STREAM, CONSUMER_GROUP, msg_id)
diff --git a/jb/flow/maintenance.py b/jb/flow/maintenance.py
deleted file mode 100644
index 4a2fe81..0000000
--- a/jb/flow/maintenance.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from jb.decorators import HM
-from jb.flow.monitoring import emit_hit_event
-from jb.managers.amt import AMTManager
-from jb.models.definitions import HitStatus
-
-
-def check_hit_status(
- amtm: AMTManager,
- amt_hit_id: str,
- amt_hit_type_id: str,
- reason: str | None = None,
-) -> HitStatus:
- """
- (this used to be called "process_hit")
- Request information from Amazon regarding the status of a HIT ID. Update
- the local state from that response.
- """
-
- hit_status = amtm.get_hit_status(amt_hit_id=amt_hit_id)
-
- # We're assuming that in the db this Hit is marked as Assignable, or
- # else we wouldn't have called this function.
- if hit_status != HitStatus.Assignable:
- # TODO: should update also assignment_pending_count,
- # assignment_available_count, assignment_completed_count
- HM.update_status(amt_hit_id=amt_hit_id, hit_status=hit_status)
- emit_hit_event(
- status=hit_status, amt_hit_type_id=amt_hit_type_id, reason=reason
- )
-
- return hit_status
diff --git a/jb/flow/monitoring.py b/jb/flow/monitoring.py
deleted file mode 100644
index 22e9e93..0000000
--- a/jb/flow/monitoring.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import socket
-
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk.literals import EventTypeType
-
-from jb.config import settings
-from jb.decorators import influx_client
-from jb.models.definitions import AssignmentStatus, HitStatus
-
-
-def write_hit_gauge(status: HitStatus, amt_hit_type_id: str, cnt: int) -> None:
-
- tags = {
- "host": socket.gethostname(), # could be "amt-jb-0"
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.hits",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def write_assignment_gauge(
- status: AssignmentStatus, amt_hit_type_id: str, cnt: int
-) -> None:
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.assignments",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def emit_hit_event(
- status: HitStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. a HIT was created, Reviewable, etc. We don't have a "created"
- HitStatus, so it would just be when status=='Assignable'
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.hit_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_assignment_event(
- status: AssignmentStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. an Assignment was accepted/approved/reject
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.assignment_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_mturk_notification_event(
- event_type: EventTypeType, amt_hit_type_id: str
-) -> None:
- """
- e.g. a Mturk notification was received. We just put it in redis, we
- haven't processed it yet.
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.mturk_notification_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_error_event(event_type: str, amt_hit_type_id: str) -> None:
- """
- e.g. todo: structure the error_types
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.error_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_bonus_event(amount: USDCent, amt_hit_type_id: str) -> None:
- """
- An AMT bonus was awarded
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.bonus_events",
- "tags": tags,
- "fields": {"value": 1, "amount": int(amount)},
- }
-
- if influx_client:
- influx_client.write_points([point])
diff --git a/jb/flow/setup_tasks.py b/jb/flow/setup_tasks.py
deleted file mode 100644
index f5cda48..0000000
--- a/jb/flow/setup_tasks.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from jb.config import SUBSCRIPTION, TOPIC_ARN, settings
-from jb.decorators import AMT_CLIENT, SNS_CLIENT
-
-
-def initial_setup():
- # Run once for initial setup. Not on each server start or anything
- subscription = SNS_CLIENT.subscribe( # type: ignore
- TopicArn=TOPIC_ARN,
- Protocol="https",
- Endpoint=f"https://jamesbillings67.com/{settings.sns_path}/",
- ReturnSubscriptionArn=True,
- )
-
-
-def check_sns_configuration():
- SNS_CLIENT.get_topic_attributes(TopicArn=TOPIC_ARN)
-
- # check this TOPIC_ARN exists
- # (doesnt have permission, dont need this anyways)
- # res = SNS_CLIENT.list_topics()
- # arns = {x["TopicArn"] for x in res["Topics"]}
- # assert TOPIC_ARN in arns, f"SNS Topic {TOPIC_ARN} doesn't exist!"
-
- subs = SNS_CLIENT.list_subscriptions_by_topic(TopicArn=TOPIC_ARN)
- assert SUBSCRIPTION in subs["Subscriptions"]
-
- AMT_CLIENT.send_test_event_notification(
- Notification={
- "Destination": TOPIC_ARN,
- "Transport": "SNS",
- "Version": "2006-05-05",
- "EventTypes": ["AssignmentSubmitted"],
- },
- TestEventType="AssignmentSubmitted",
- )
diff --git a/jb/flow/tasks.py b/jb/flow/tasks.py
deleted file mode 100644
index c555021..0000000
--- a/jb/flow/tasks.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import time
-from typing import TypedDict, cast
-
-from generalresearch.config import is_debug
-
-from jb.decorators import AMTM, HM, HQM, HTM, LOG, pg_config
-from jb.flow.maintenance import check_hit_status
-from jb.flow.monitoring import emit_hit_event, write_hit_gauge
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class HitRow(TypedDict):
- amt_hit_id: str
- amt_hit_type_id: str
-
-
-def check_stale_hits():
- # Check live hits that haven't been modified in a long time. They may not
- # be expired yet, but maybe something is wrong?
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- ORDER BY modified_at
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_stale_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="cleanup",
- )
-
-
-def check_expired_hits():
- # Check live/assignable hits that are expired (based on AMT's expiration time)
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- AND expiration < now()
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_expired_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="expired",
- )
-
-
-def create_hit_from_hittype(hit_type: HitType) -> Hit:
- if is_debug():
- raise Exception("Handle AMT Sandbox issues.")
-
- else:
- question = HQM.get_or_create(
- HitQuestion(height=800, url="https://jamesbillings67.com/work/")
- )
-
- hit = AMTM.create_hit_with_hit_type(hit_type=hit_type, question=question)
- HM.create(hit)
- emit_hit_event(status=hit.status, amt_hit_type_id=hit.amt_hit_type_id)
- return hit
-
-
-def refill_hits() -> None:
-
- for hit_type in HTM.filter_active():
- assert hit_type.id
- assert hit_type.amt_hit_type_id
-
- active_count = HM.get_active_count(hit_type_id=hit_type.id)
- LOG.info(
- f"HitType: {hit_type.amt_hit_type_id}, {hit_type.min_active=}, active_count={active_count}"
- )
- write_hit_gauge(
- status=HitStatus.Assignable,
- amt_hit_type_id=hit_type.amt_hit_type_id,
- cnt=active_count,
- )
- if active_count < hit_type.min_active:
- cnt_todo = hit_type.min_active - active_count
- LOG.info(f"Refilling {cnt_todo} hits")
- for _ in range(cnt_todo):
- create_hit_from_hittype(hit_type)
-
-
-def refill_hits_task():
- while True:
- try:
- check_expired_hits()
- check_stale_hits()
- refill_hits()
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(5 * 60)