aboutsummaryrefslogtreecommitdiff
path: root/jb
diff options
context:
space:
mode:
authorstuppie2026-11-04 12:11:03 -0600
committerstuppie2026-11-04 12:11:03 -0600
commit671a7b1b02503cfa5c9aecd40c27a0c2729a285b (patch)
tree6cb1bd7c33e3e1d243ee5ae74decc57faaa5742d /jb
parente99ec6b7e38113a2237dc1ff402d8f4337c651d5 (diff)
downloadamt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.tar.gz
amt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.zip
strip out AMT
Diffstat (limited to 'jb')
-rw-r--r--jb/config.py15
-rw-r--r--jb/decorators.py51
-rw-r--r--jb/flow/__init__.py0
-rw-r--r--jb/flow/assignment_tasks.py124
-rw-r--r--jb/flow/events.py98
-rw-r--r--jb/flow/maintenance.py31
-rw-r--r--jb/flow/monitoring.py168
-rw-r--r--jb/flow/setup_tasks.py35
-rw-r--r--jb/flow/tasks.py109
-rw-r--r--jb/main.py10
-rw-r--r--jb/managers/amt.py270
-rw-r--r--jb/managers/assignment.py240
-rw-r--r--jb/managers/bonus.py51
-rw-r--r--jb/managers/hit.py327
-rw-r--r--jb/managers/worker.py14
-rw-r--r--jb/models/amt.py19
-rw-r--r--jb/models/api_response.py17
-rw-r--r--jb/models/assignment.py396
-rw-r--r--jb/models/bonus.py48
-rw-r--r--jb/models/custom_types.py19
-rw-r--r--jb/models/definitions.py64
-rw-r--r--jb/models/errors.py80
-rw-r--r--jb/models/event.py48
-rw-r--r--jb/models/hit.py260
-rw-r--r--jb/models/response.py21
-rw-r--r--jb/settings.py18
-rw-r--r--jb/views/common.py98
27 files changed, 2 insertions, 2629 deletions
diff --git a/jb/config.py b/jb/config.py
index 954dfe8..dea6bec 100644
--- a/jb/config.py
+++ b/jb/config.py
@@ -9,21 +9,6 @@ if settings.debug:
else:
LOG_LEVEL = logging.WARNING
-# The SNS topic that 1) JB Mturk will send notifications to, 2) will make http POSTs
-# back to us (here)
-TOPIC_ARN = f"arn:aws:sns:us-east-2:{settings.aws_owner_id}:amt-jb"
-SUBSCRIPTION = {
- "SubscriptionArn": settings.aws_subscription_arn,
- "Owner": settings.aws_owner_id,
- "Protocol": "https",
- "Endpoint": f"https://jamesbillings67.com/{settings.sns_path}/",
- "TopicArn": TOPIC_ARN,
-}
-
-JB_EVENTS_STREAM = "amt_jb_events"
-CONSUMER_GROUP = "amt-jb-0"
-# We'll only have 1 consumer atm, change this if we don't
-CONSUMER_NAME = "amt-jb-0"
if settings.sentry:
import sentry_sdk
diff --git a/jb/decorators.py b/jb/decorators.py
index 9c7a31c..e275af5 100644
--- a/jb/decorators.py
+++ b/jb/decorators.py
@@ -1,20 +1,11 @@
import logging
-import boto3
-from botocore.config import Config
-from generalresearch.managers.base import Permission
from generalresearch.pg_helper import PostgresConfig
from generalresearch.redis_helper import RedisConfig
from influxdb import InfluxDBClient
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_sns import SNSClient
from jb.config import settings
-from jb.managers.amt import AMTManager
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
from jb.managers.gr_api import GRApiManager
-from jb.managers.hit import HitManager, HitQuestionManager, HitTypeManager
redis_config = RedisConfig(
dsn=settings.redis,
@@ -41,16 +32,6 @@ logging.basicConfig(
)
LOG = logging.getLogger("amtjb")
-CLIENT_CONFIG = Config(
- # connect_timeout (float or int) – The time in seconds till a timeout
- # exception is thrown when attempting to make a connection. The default
- # is 60 seconds.
- connect_timeout=1,
- # read_timeout (float or int) – The time in seconds till a timeout
- # exception is thrown when attempting to read from a connection. The
- # default is 60 seconds.
- read_timeout=2.5,
-)
gr_api_manager = GRApiManager(
base_url=str(settings.gr_api_host),
@@ -58,23 +39,6 @@ gr_api_manager = GRApiManager(
product_id=settings.product_id,
)
-# We shouldn't use this directly. Use our AMTManager wrapper
-AMT_CLIENT: MTurkClient = boto3.client(
- service_name="mturk",
- region_name="us-east-1",
- endpoint_url=str(settings.amt_endpoint),
- aws_access_key_id=settings.amt_access_id,
- aws_secret_access_key=settings.amt_secret_key,
- config=CLIENT_CONFIG,
-)
-
-SNS_CLIENT: SNSClient = boto3.client(
- service_name="sns",
- region_name="us-east-2",
- aws_access_key_id=settings.amt_access_id,
- aws_secret_access_key=settings.amt_secret_key,
- config=CLIENT_CONFIG,
-)
pg_config = PostgresConfig(
dsn=settings.amt_jb_db,
@@ -82,21 +46,6 @@ pg_config = PostgresConfig(
statement_timeout=1,
)
-AMTM = AMTManager(amt_client=AMT_CLIENT)
-
-HTM = HitTypeManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-HM = HitManager(pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE])
-HQM = HitQuestionManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-AM = AssignmentManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-
-BM = BonusManager(pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE])
-
influx_client = None
if settings.influx_db:
influx_client = InfluxDBClient.from_dsn(str(settings.influx_db))
diff --git a/jb/flow/__init__.py b/jb/flow/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/jb/flow/__init__.py
+++ /dev/null
diff --git a/jb/flow/assignment_tasks.py b/jb/flow/assignment_tasks.py
deleted file mode 100644
index b3c820a..0000000
--- a/jb/flow/assignment_tasks.py
+++ /dev/null
@@ -1,124 +0,0 @@
-from jb.decorators import LOG
-from jb.flow.monitoring import emit_assignment_event, emit_error_event
-from jb.managers.amt import (
- REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- AMTManager,
-)
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
-from jb.managers.hit import HitManager
-from jb.models.assignment import Assignment
-from jb.models.definitions import AssignmentStatus
-from jb.models.event import MTurkEvent
-
-
-def process_assignment_submitted(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- event: MTurkEvent,
-) -> None:
- """
- Reject any submitted assignments
- """
-
- #
- # Step 1: Attempt to get the Assignment out of the API
- #
- LOG.info(f"{event=}")
-
- # This is the assignment model from AMT. In the DB, we should only have
- # the AssignmentStub
- # This call is hitting AMT, not our db.
- # The API won't even return is unless it has been submitted. So if this
- # fails, we don't need to do anything (i.e. reject it).
- assignment = amtm.get_assignment_if_exists(
- amt_assignment_id=event.amt_assignment_id
- )
-
- if assignment is None:
- # It is not found in amt, either it is invalid, not yet submitted, or
- # already been approved/rejected, so we just do nothing ...
- # todo: maybe we confirm its state matches what we have in the db
- LOG.warning(f"No assignment found on AMT: {event.amt_assignment_id}")
- emit_error_event(
- event_type="assignment_not_found_in_amt",
- amt_hit_type_id=event.amt_hit_type_id,
- )
- return
-
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
-
- reject_assignment(
- amtm=amtm,
- am=am,
- hm=hm,
- amt_assignment_id=assignment.amt_assignment_id,
- msg=REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- amt_hit_type_id=hit.amt_hit_type_id,
- )
- review_hit(amtm=amtm, hm=hm, assignment=assignment)
- return
-
-
-def review_hit(amtm: AMTManager, hm: HitManager, assignment: Assignment) -> None:
- # Reviewable to Reviewing
- amtm.update_hit_review_status(amt_hit_id=assignment.amt_hit_id, revert=False)
- hit, _ = amtm.get_hit_if_exists(amt_hit_id=assignment.amt_hit_id)
-
- if hit is None:
- LOG.warning(f"Hit not found when trying to review hit: {assignment.amt_hit_id}")
- return
-
- # Update the db
- hm.update_hit(hit)
-
- return
-
-
-def reject_assignment(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- amt_assignment_id: str,
- msg: str,
- amt_hit_type_id: str,
-) -> Assignment:
- # Reject in AMT, update db
-
- res = amtm.reject_assignment_if_possible(
- amt_assignment_id=amt_assignment_id, msg=msg
- )
- if res is None:
- # We failed to reject this assignment. This happens if the assignment
- # is already approved or already rejected.
- emit_error_event(
- event_type="failed_to_reject_assignment",
- amt_hit_type_id=amt_hit_type_id,
- )
- LOG.exception(f"Failed to reject assignment: {amt_assignment_id}")
-
- # We just rejected this assignment, get it from amazon again
- assignment = amtm.get_assignment(amt_assignment_id=amt_assignment_id)
- assert assignment.status == AssignmentStatus.Rejected
-
- # And update the db. The assignment may not actually exist in the db (if
- # a baddie intercepted it and is trying to game us). So, we might
- # need to create as assignment first ...
- stub = am.get_stub_if_exists(amt_assignment_id=assignment.amt_assignment_id)
- if stub is None:
- LOG.warning(
- f"Rejected assignment doesn't exist in DB. Creating ... : {amt_assignment_id}"
- )
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
- assignment.hit_id = hit.id
- am.create(assignment=assignment)
- am.reject(assignment=assignment)
- emit_assignment_event(
- status=AssignmentStatus.Rejected, amt_hit_type_id=amt_hit_type_id, reason=msg
- )
- LOG.warning(f"Rejected assignment: {amt_assignment_id}")
- return assignment
diff --git a/jb/flow/events.py b/jb/flow/events.py
deleted file mode 100644
index bee63fd..0000000
--- a/jb/flow/events.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import time
-from concurrent import futures
-from concurrent.futures import Executor, ThreadPoolExecutor
-from typing import cast
-
-import redis
-
-from jb.config import (
- CONSUMER_GROUP,
- CONSUMER_NAME,
- JB_EVENTS_STREAM,
-)
-from jb.decorators import LOG, get_redis
-from jb.flow.assignment_tasks import process_assignment_submitted
-from jb.flow.monitoring import emit_error_event
-from jb.models.event import MTurkEvent
-
-StreamMessages = list[tuple[str, list[tuple[bytes, dict[bytes, bytes]]]]]
-
-
-def process_mturk_events_task():
- executor = ThreadPoolExecutor(max_workers=5)
- create_consumer_group()
- while True:
- try:
- process_mturk_events(executor=executor)
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(1)
-
-
-def process_mturk_events(executor: Executor):
- while True:
- n = process_mturk_events_chunk(executor=executor)
- if n is None or n < 10:
- break
-
-
-def create_consumer_group():
- try:
- redis_client = get_redis()
- redis_client.xgroup_create(
- JB_EVENTS_STREAM, CONSUMER_GROUP, id="0", mkstream=True
- )
- except redis.exceptions.ResponseError as e:
- if "BUSYGROUP Consumer Group name already exists" in str(e):
- pass # group already exists
- else:
- raise
-
-
-def process_mturk_events_chunk(executor: Executor) -> int | None:
- redis_client = get_redis()
- msgs_raw = redis_client.xreadgroup(
- groupname=CONSUMER_GROUP,
- consumername=CONSUMER_NAME,
- streams={JB_EVENTS_STREAM: ">"},
- count=10,
- )
- if not msgs_raw:
- return None
-
- msgs = cast(StreamMessages, msgs_raw)[0][1] # The queue, we only have 1
-
- fs = []
- for msg in msgs:
- msg_id, data = msg
- msg_json: str = data["data"]
-
- event = MTurkEvent.model_validate_json(json_data=msg_json)
- if event.event_type == "AssignmentSubmitted":
- fs.append(
- executor.submit(process_assignment_submitted_event, event, str(msg_id))
- )
- else:
- LOG.info(f"Discarding {event}")
- redis_client.xdel(JB_EVENTS_STREAM, msg_id)
-
- futures.wait(fs, timeout=60)
- return len(msgs)
-
-
-def process_assignment_submitted_event(event: MTurkEvent, msg_id: str):
- from jb.decorators import AM, AMTM, BM, HM
-
- redis_client = get_redis()
-
- try:
- process_assignment_submitted(amtm=AMTM, am=AM, hm=HM, bm=BM, event=event)
- except Exception as e:
- LOG.exception(f"{event.amt_assignment_id=}, {e=}")
- emit_error_event(
- event_type="failed_process_assignment_submitted",
- amt_hit_type_id=event.amt_hit_type_id,
- )
-
- redis_client.xackdel(JB_EVENTS_STREAM, CONSUMER_GROUP, msg_id)
diff --git a/jb/flow/maintenance.py b/jb/flow/maintenance.py
deleted file mode 100644
index 4a2fe81..0000000
--- a/jb/flow/maintenance.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from jb.decorators import HM
-from jb.flow.monitoring import emit_hit_event
-from jb.managers.amt import AMTManager
-from jb.models.definitions import HitStatus
-
-
-def check_hit_status(
- amtm: AMTManager,
- amt_hit_id: str,
- amt_hit_type_id: str,
- reason: str | None = None,
-) -> HitStatus:
- """
- (this used to be called "process_hit")
- Request information from Amazon regarding the status of a HIT ID. Update
- the local state from that response.
- """
-
- hit_status = amtm.get_hit_status(amt_hit_id=amt_hit_id)
-
- # We're assuming that in the db this Hit is marked as Assignable, or
- # else we wouldn't have called this function.
- if hit_status != HitStatus.Assignable:
- # TODO: should update also assignment_pending_count,
- # assignment_available_count, assignment_completed_count
- HM.update_status(amt_hit_id=amt_hit_id, hit_status=hit_status)
- emit_hit_event(
- status=hit_status, amt_hit_type_id=amt_hit_type_id, reason=reason
- )
-
- return hit_status
diff --git a/jb/flow/monitoring.py b/jb/flow/monitoring.py
deleted file mode 100644
index 22e9e93..0000000
--- a/jb/flow/monitoring.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import socket
-
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk.literals import EventTypeType
-
-from jb.config import settings
-from jb.decorators import influx_client
-from jb.models.definitions import AssignmentStatus, HitStatus
-
-
-def write_hit_gauge(status: HitStatus, amt_hit_type_id: str, cnt: int) -> None:
-
- tags = {
- "host": socket.gethostname(), # could be "amt-jb-0"
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.hits",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def write_assignment_gauge(
- status: AssignmentStatus, amt_hit_type_id: str, cnt: int
-) -> None:
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.assignments",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def emit_hit_event(
- status: HitStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. a HIT was created, Reviewable, etc. We don't have a "created"
- HitStatus, so it would just be when status=='Assignable'
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.hit_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_assignment_event(
- status: AssignmentStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. an Assignment was accepted/approved/reject
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.assignment_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_mturk_notification_event(
- event_type: EventTypeType, amt_hit_type_id: str
-) -> None:
- """
- e.g. a Mturk notification was received. We just put it in redis, we
- haven't processed it yet.
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.mturk_notification_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_error_event(event_type: str, amt_hit_type_id: str) -> None:
- """
- e.g. todo: structure the error_types
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.error_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_bonus_event(amount: USDCent, amt_hit_type_id: str) -> None:
- """
- An AMT bonus was awarded
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.bonus_events",
- "tags": tags,
- "fields": {"value": 1, "amount": int(amount)},
- }
-
- if influx_client:
- influx_client.write_points([point])
diff --git a/jb/flow/setup_tasks.py b/jb/flow/setup_tasks.py
deleted file mode 100644
index f5cda48..0000000
--- a/jb/flow/setup_tasks.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from jb.config import SUBSCRIPTION, TOPIC_ARN, settings
-from jb.decorators import AMT_CLIENT, SNS_CLIENT
-
-
-def initial_setup():
- # Run once for initial setup. Not on each server start or anything
- subscription = SNS_CLIENT.subscribe( # type: ignore
- TopicArn=TOPIC_ARN,
- Protocol="https",
- Endpoint=f"https://jamesbillings67.com/{settings.sns_path}/",
- ReturnSubscriptionArn=True,
- )
-
-
-def check_sns_configuration():
- SNS_CLIENT.get_topic_attributes(TopicArn=TOPIC_ARN)
-
- # check this TOPIC_ARN exists
- # (doesnt have permission, dont need this anyways)
- # res = SNS_CLIENT.list_topics()
- # arns = {x["TopicArn"] for x in res["Topics"]}
- # assert TOPIC_ARN in arns, f"SNS Topic {TOPIC_ARN} doesn't exist!"
-
- subs = SNS_CLIENT.list_subscriptions_by_topic(TopicArn=TOPIC_ARN)
- assert SUBSCRIPTION in subs["Subscriptions"]
-
- AMT_CLIENT.send_test_event_notification(
- Notification={
- "Destination": TOPIC_ARN,
- "Transport": "SNS",
- "Version": "2006-05-05",
- "EventTypes": ["AssignmentSubmitted"],
- },
- TestEventType="AssignmentSubmitted",
- )
diff --git a/jb/flow/tasks.py b/jb/flow/tasks.py
deleted file mode 100644
index c555021..0000000
--- a/jb/flow/tasks.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import time
-from typing import TypedDict, cast
-
-from generalresearch.config import is_debug
-
-from jb.decorators import AMTM, HM, HQM, HTM, LOG, pg_config
-from jb.flow.maintenance import check_hit_status
-from jb.flow.monitoring import emit_hit_event, write_hit_gauge
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class HitRow(TypedDict):
- amt_hit_id: str
- amt_hit_type_id: str
-
-
-def check_stale_hits():
- # Check live hits that haven't been modified in a long time. They may not
- # be expired yet, but maybe something is wrong?
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- ORDER BY modified_at
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_stale_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="cleanup",
- )
-
-
-def check_expired_hits():
- # Check live/assignable hits that are expired (based on AMT's expiration time)
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- AND expiration < now()
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_expired_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="expired",
- )
-
-
-def create_hit_from_hittype(hit_type: HitType) -> Hit:
- if is_debug():
- raise Exception("Handle AMT Sandbox issues.")
-
- else:
- question = HQM.get_or_create(
- HitQuestion(height=800, url="https://jamesbillings67.com/work/")
- )
-
- hit = AMTM.create_hit_with_hit_type(hit_type=hit_type, question=question)
- HM.create(hit)
- emit_hit_event(status=hit.status, amt_hit_type_id=hit.amt_hit_type_id)
- return hit
-
-
-def refill_hits() -> None:
-
- for hit_type in HTM.filter_active():
- assert hit_type.id
- assert hit_type.amt_hit_type_id
-
- active_count = HM.get_active_count(hit_type_id=hit_type.id)
- LOG.info(
- f"HitType: {hit_type.amt_hit_type_id}, {hit_type.min_active=}, active_count={active_count}"
- )
- write_hit_gauge(
- status=HitStatus.Assignable,
- amt_hit_type_id=hit_type.amt_hit_type_id,
- cnt=active_count,
- )
- if active_count < hit_type.min_active:
- cnt_todo = hit_type.min_active - active_count
- LOG.info(f"Refilling {cnt_todo} hits")
- for _ in range(cnt_todo):
- create_hit_from_hittype(hit_type)
-
-
-def refill_hits_task():
- while True:
- try:
- check_expired_hits()
- check_stale_hits()
- refill_hits()
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(5 * 60)
diff --git a/jb/main.py b/jb/main.py
index b3b7470..52d1ca1 100644
--- a/jb/main.py
+++ b/jb/main.py
@@ -1,4 +1,3 @@
-from multiprocessing import Process
from typing import Any
from fastapi import FastAPI
@@ -9,7 +8,6 @@ from starlette.middleware.trustedhost import TrustedHostMiddleware
from jb.config import settings
from jb.settings import render_base_html
from jb.views.auth import auth_router
-from jb.views.common import common_router
from jb.views.wallet import wallet_router
app = FastAPI(
@@ -33,7 +31,6 @@ app.add_middleware(
)
app.add_middleware(TrustedHostMiddleware, allowed_hosts=["*"])
-app.include_router(router=common_router)
app.include_router(router=auth_router)
app.include_router(router=wallet_router)
@@ -60,11 +57,8 @@ def serve_react_app(full_path: str):
def schedule_tasks():
- from jb.flow.events import process_mturk_events_task
- from jb.flow.tasks import refill_hits_task
-
- Process(target=process_mturk_events_task).start()
- Process(target=refill_hits_task).start()
+ pass
+ # Process(target=example_task).start()
if not settings.debug:
diff --git a/jb/managers/amt.py b/jb/managers/amt.py
deleted file mode 100644
index 488bbb8..0000000
--- a/jb/managers/amt.py
+++ /dev/null
@@ -1,270 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import Any
-
-import botocore.exceptions
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_mturk.type_defs import (
- AssignmentTypeDef,
- BonusPaymentTypeDef,
- CreateHITTypeResponseTypeDef,
- CreateHITWithHITTypeResponseTypeDef,
- GetHITResponseTypeDef,
-)
-from pydantic import ValidationError
-
-from jb.config import TOPIC_ARN
-from jb.models.amt import AMTAccount
-from jb.models.assignment import Assignment
-from jb.models.bonus import Bonus
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-LOG = logging.getLogger("amtjb")
-
-REJECT_MESSAGE_UNKNOWN_ASSIGNMENT = "Unknown assignment"
-REJECT_MESSAGE_NO_WORK = "Assignment was submitted with no attempted work."
-REJECT_MESSAGE_BADDIE = "Quality has dropped below an acceptable level"
-APPROVAL_MESSAGE = "Thank you!"
-NO_WORK_APPROVAL_MESSAGE = (
- REJECT_MESSAGE_NO_WORK + " In the future, if you are not sent into a task, "
- "please return the assignment, otherwise it will be rejected!"
-)
-BONUS_MESSAGE = "Great job! Bonus for a survey complete"
-
-
-class AMTManager:
- """
- I am type annotating this more than needed (depending on the editor),
- however it's only because AMT on boto3 is not intuitive... at all.
- """
-
- def __init__(
- self,
- amt_client: MTurkClient,
- **kwargs, # type: ignore
- ):
- super().__init__(**kwargs)
- self.amt_client = amt_client
-
- def fetch_account(self) -> AMTAccount:
- res = self.amt_client.get_account_balance()
-
- return AMTAccount.model_validate(
- {
- "available_balance": res["AvailableBalance"],
- "onhold_balance": res.get("OnHoldBalance", 0),
- }
- )
-
- def get_hit_if_exists(self, amt_hit_id: str) -> tuple[Hit | None, str | None]:
- try:
- res: GetHITResponseTypeDef = self.amt_client.get_hit(HITId=amt_hit_id)
-
- except self.amt_client.exceptions.RequestError as e:
- msg = e.response.get("Error", {}).get("Message", "")
- return None, msg
-
- hit = Hit.from_amt_get_hit(res["HIT"])
- return hit, None
-
- def get_hit_status(self, amt_hit_id: str) -> HitStatus:
- res, msg = self.get_hit_if_exists(amt_hit_id=amt_hit_id)
-
- if res is None:
- if msg is None:
- return HitStatus.Unassignable
-
- if " does not exist. (" in msg:
- return HitStatus.Disposed
-
- else:
- LOG.warning(msg)
- return HitStatus.Unassignable
-
- return res.status
-
- def create_hit_type(self, hit_type: HitType) -> HitType:
- assert hit_type.amt_hit_type_id is None
-
- res: CreateHITTypeResponseTypeDef = self.amt_client.create_hit_type(
- **hit_type.to_api_request_body()
- ) # type: ignore
-
- hit_type.amt_hit_type_id = res["HITTypeId"]
-
- # TODO: Assert / Check that the SNS Notification was
- # successfully created.
- self.amt_client.update_notification_settings(
- HITTypeId=hit_type.amt_hit_type_id,
- Notification={
- "Destination": TOPIC_ARN,
- "Transport": "SNS",
- "Version": "2006-05-05",
- # you can add more events, see mypy_boto3_mturk.literals.EventTypeType
- "EventTypes": ["AssignmentSubmitted"],
- },
- Active=True,
- )
-
- return hit_type
-
- def create_hit_with_hit_type(self, hit_type: HitType, question: HitQuestion) -> Hit:
- """
- HITTypeId: str
- LifetimeInSeconds: int
- MaxAssignments: NotRequired[int]
- Question: NotRequired[str]
- UniqueRequestToken: NotRequired[str]
- """
- assert hit_type.id is not None
- assert hit_type.amt_hit_type_id is not None
-
- data = hit_type.generate_hit_amt_request(question=question)
- res: CreateHITWithHITTypeResponseTypeDef = (
- self.amt_client.create_hit_with_hit_type(**data)
- )
-
- return Hit.from_amt_create_hit(res["HIT"], hit_type=hit_type, question=question)
-
- def get_assignment(self, amt_assignment_id: str) -> Assignment:
- """
- You CANNOT get an Assignment if it has been only ACCEPTED (by the
- worker). The api is stupid, it will only show up once it is Submitted
- """
- res = self.amt_client.get_assignment(AssignmentId=amt_assignment_id)
- ass_res: AssignmentTypeDef = res["Assignment"]
- try:
- assignment = Assignment.from_amt_get_assignment(ass_res)
- except ValidationError as e:
- # Baddies have been known to submit assignments with purposely
- # malformed "answer" (xml) section, which will raise
- # a pydantic validation error. Try to parse again with no Answer.
- LOG.exception(e)
- ass_res["Answer"] = None
- # If it wasn't the Answer that caused the ValidationError, it'll raise again
- assignment = Assignment.from_amt_get_assignment(ass_res)
-
- # to be clear, this has not checked whether it exists in our db
- assert assignment.id is None
- return assignment
-
- def get_assignment_if_exists(self, amt_assignment_id: str) -> Assignment | None:
- expected_err_msg = f"Assignment {amt_assignment_id} does not exist"
-
- try:
- return self.get_assignment(amt_assignment_id=amt_assignment_id)
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- error_code = e.response["Error"]["Code"]
- error_msg = e.response["Error"]["Message"]
- if error_code == "RequestError" and expected_err_msg in error_msg:
- return None
- raise e
-
- def reject_assignment_if_possible(
- self, amt_assignment_id: str, msg: str = REJECT_MESSAGE_UNKNOWN_ASSIGNMENT
- ) -> dict[str, Any] | None:
-
- # Unclear to me when this would fail
- try:
- return self.amt_client.reject_assignment(
- AssignmentId=amt_assignment_id, RequesterFeedback=msg
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- return None
-
- def approve_assignment_if_possible(
- self,
- amt_assignment_id: str,
- msg: str = APPROVAL_MESSAGE,
- override_rejection: bool = False,
- ) -> dict[str, Any] | None:
-
- # Unclear to me when this would fail
- try:
- return self.amt_client.approve_assignment(
- AssignmentId=amt_assignment_id,
- RequesterFeedback=msg,
- OverrideRejection=override_rejection,
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- return None
-
- def update_hit_review_status(self, amt_hit_id: str, revert: bool = False) -> None:
- try:
- # Reviewable to Reviewing
- self.amt_client.update_hit_review_status(HITId=amt_hit_id, Revert=revert)
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(f"{amt_hit_id=}, {e}")
- error_msg = e.response["Error"]["Message"]
-
- if "does not exist" in error_msg:
- raise ValueError(error_msg)
-
- # elif "This HIT is currently in the state 'Reviewing'" in error_msg:
- # logging.warning(error_msg)
-
- def send_bonus(
- self,
- amt_worker_id: str,
- amount: USDCent,
- amt_assignment_id: str,
- reason: str,
- unique_request_token: str,
- ) -> dict[str, Any] | None:
- try:
- return self.amt_client.send_bonus(
- WorkerId=amt_worker_id,
- BonusAmount=str(amount.to_usd()),
- AssignmentId=amt_assignment_id,
- Reason=reason,
- UniqueRequestToken=unique_request_token,
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(f"{amt_worker_id=} {amt_assignment_id=}, {e}")
- return None
-
- def get_bonus(self, amt_assignment_id: str, payout_event_id: str) -> Bonus | None:
-
- res: list[BonusPaymentTypeDef] = self.amt_client.list_bonus_payments(
- AssignmentId=amt_assignment_id
- )["BonusPayments"]
-
- assert len(res) <= 1, (
- f"{amt_assignment_id=} Expected 1 or 0 bonuses, got {len(res)}"
- )
- d = res[0] if res else None
-
- if d:
- return Bonus.model_validate(
- {
- "amt_worker_id": d["WorkerId"],
- "amount": USDCent(round(float(d["BonusAmount"]) * 100)),
- "amt_assignment_id": d["AssignmentId"],
- "reason": d["Reason"],
- "grant_time": d["GrantTime"].astimezone(tz=timezone.utc),
- "payout_event_id": payout_event_id,
- }
- )
-
- return None
-
- def expire_all_hits(self) -> None:
- # Used in testing only (or in an emergency I guess)
- now = datetime.now(tz=timezone.utc)
- paginator = self.amt_client.get_paginator("list_hits")
-
- for page in paginator.paginate():
- for hit in page["HITs"]:
- if hit["HITStatus"] in ("Assignable", "Reviewable", "Reviewing"):
- self.amt_client.update_expiration_for_hit(
- HITId=hit["HITId"], ExpireAt=now
- )
diff --git a/jb/managers/assignment.py b/jb/managers/assignment.py
deleted file mode 100644
index 2740aee..0000000
--- a/jb/managers/assignment.py
+++ /dev/null
@@ -1,240 +0,0 @@
-from datetime import datetime, timezone
-
-from psycopg import sql
-from pydantic import NonNegativeInt, PositiveInt
-
-from jb.managers.base import PostgresManager
-from jb.models.assignment import Assignment, AssignmentStub
-from jb.models.definitions import AssignmentStatus
-
-
-class AssignmentManager(PostgresManager):
-
- def create_stub(self, stub: AssignmentStub) -> None:
- assert stub.id is None
- data = stub.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_assignment
- (amt_assignment_id, amt_worker_id, status,
- created_at, modified_at, hit_id)
- VALUES
- (%(amt_assignment_id)s, %(amt_worker_id)s, %(status)s,
- %(created_at)s, %(modified_at)s, %(hit_id)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- stub.id = pk
-
- def create(self, assignment: Assignment) -> None:
- # Typically this is NOT used (we'd create the stub when HIT is
- # accepted, then update it once the assignment is submitted), however
- # there may be cases where an assignment is submitted and the stub
- # was never created (probably due to error/downtime/baddie).
-
- assert assignment.id is None
- data = assignment.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_assignment
- (amt_assignment_id, amt_worker_id, status,
- created_at, modified_at, hit_id,
- auto_approval_time, accept_time, submit_time,
- approval_time, rejection_time, requester_feedback,
- tsid)
- VALUES
- (%(amt_assignment_id)s, %(amt_worker_id)s, %(status)s,
- %(created_at)s, %(modified_at)s, %(hit_id)s,
- %(auto_approval_time)s, %(accept_time)s, %(submit_time)s,
- %(approval_time)s, %(rejection_time)s, %(requester_feedback)s,
- %(tsid)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- assignment.id = pk
-
- def get_stub(self, amt_assignment_id: str) -> AssignmentStub:
- res = self.pg_config.execute_sql_query(
- """
- SELECT ma.id, ma.hit_id,
- ma.amt_assignment_id, ma.amt_worker_id,
- ma.status, ma.created_at, ma.modified_at,
- mh.amt_hit_id
- FROM mtwerk_assignment ma
- JOIN mtwerk_hit mh ON ma.hit_id = mh.id
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
- assert len(res) == 1
- return AssignmentStub.model_validate(res[0])
-
- def get_stub_if_exists(self, amt_assignment_id: str) -> AssignmentStub | None:
- try:
- return self.get_stub(amt_assignment_id=amt_assignment_id)
- except AssertionError:
- return None
-
- def get(self, amt_assignment_id: str) -> Assignment:
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT mh.amt_hit_id, ma.id, ma.amt_assignment_id,
- ma.amt_worker_id, ma.status, ma.auto_approval_time,
- ma.accept_time, ma.submit_time, ma.approval_time,
- ma.rejection_time, ma.requester_feedback,
- ma.created_at, ma.modified_at, ma.tsid, ma.hit_id
- FROM mtwerk_assignment ma
- JOIN mtwerk_hit mh ON ma.hit_id = mh.id
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
- assert len(res) == 1
- return Assignment.model_validate(res[0])
-
- def update_answer(self, assignment: Assignment) -> None:
- # We're assuming a stub already exists
- # The assignment was submitted, but we haven't made a decision yet
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "auto_approval_time": assignment.auto_approval_time,
- "tsid": assignment.tsid,
- "amt_assignment_id": assignment.amt_assignment_id,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- auto_approval_time = %(auto_approval_time)s,
- status = %(status)s,
- tsid = %(tsid)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def reject(self, assignment: Assignment) -> None:
- assert assignment.status == AssignmentStatus.Rejected
- assert assignment.rejection_time is not None
- assert assignment.approval_time is None
- assert assignment.requester_feedback is not None
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "rejection_time": assignment.rejection_time,
- "requester_feedback": assignment.requester_feedback,
- "amt_assignment_id": assignment.amt_assignment_id,
- "auto_approval_time": assignment.auto_approval_time,
- "accept_time": assignment.accept_time,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- rejection_time = %(rejection_time)s,
- status = %(status)s,
- requester_feedback = %(requester_feedback)s,
- auto_approval_time = %(auto_approval_time)s,
- accept_time = %(accept_time)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def approve(self, assignment: Assignment) -> None:
- assert assignment.status == AssignmentStatus.Approved
- # # We can override a rejection and the rejection time is not None
- # assert assignment.rejection_time is None
- assert assignment.approval_time is not None
- assert assignment.requester_feedback is not None
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "approval_time": assignment.approval_time,
- "requester_feedback": assignment.requester_feedback,
- "amt_assignment_id": assignment.amt_assignment_id,
- "auto_approval_time": assignment.auto_approval_time,
- "accept_time": assignment.accept_time,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- approval_time = %(approval_time)s,
- status = %(status)s,
- requester_feedback = %(requester_feedback)s,
- auto_approval_time = %(auto_approval_time)s,
- accept_time = %(accept_time)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def missing_tsid_count(
- self, amt_worker_id: str, lookback_hrs: PositiveInt = 24
- ) -> NonNegativeInt:
- """
- Look at this user's previous N hrs of submitted assignments.
- Count how many assignments they have submitted without a tsid.
- """
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) AS c
- FROM mtwerk_assignment
- WHERE amt_worker_id = %(amt_worker_id)s
- AND submit_time > NOW() - (%(lookback_interval)s)::interval
- AND tsid IS NULL;
- """,
- params={
- "amt_worker_id": amt_worker_id,
- "lookback_interval": f"{lookback_hrs} hour",
- },
- )
- return int(res[0]["c"]) # type: ignore
-
- def rejected_count(
- self, amt_worker_id: str, lookback_hrs: int = 24
- ) -> NonNegativeInt:
- """
- Look at this user's previous N hrs of submitted assignments.
- Count how many rejected assignments they have.
- """
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) AS c
- FROM mtwerk_assignment
- WHERE amt_worker_id = %(amt_worker_id)s
- AND submit_time > NOW() - (%(lookback_interval)s)::interval
- AND status = %(status)s;
- """,
- params={
- "amt_worker_id": amt_worker_id,
- "lookback_interval": f"{lookback_hrs} hour",
- "status": AssignmentStatus.Rejected.value,
- },
- )
- return int(res[0]["c"]) # type: ignore
diff --git a/jb/managers/bonus.py b/jb/managers/bonus.py
deleted file mode 100644
index b649103..0000000
--- a/jb/managers/bonus.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from typing import Any
-
-from psycopg import sql
-
-from jb.managers.base import PostgresManager
-from jb.models.bonus import Bonus
-
-
-class BonusManager(PostgresManager):
-
- def create(self, bonus: Bonus) -> None:
- assert bonus.id is None
- data = bonus.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_bonus
- (payout_event_id, amt_worker_id, amount, grant_time, assignment_id, reason)
- VALUES (
- %(payout_event_id)s,
- %(amt_worker_id)s,
- %(amount)s,
- %(grant_time)s,
- (
- SELECT id
- FROM mtwerk_assignment
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1
- ),
- %(reason)s
- )
- RETURNING id, assignment_id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- res = c.fetchone()
- conn.commit()
- bonus.id = res["id"] # type: ignore
- bonus.assignment_id = res["assignment_id"] # type: ignore
-
- def filter(self, amt_assignment_id: str) -> list[Bonus]:
- res: list[Any] = self.pg_config.execute_sql_query(
- """
- SELECT mb.*, ma.amt_assignment_id
- FROM mtwerk_bonus mb
- JOIN mtwerk_assignment ma ON ma.id = mb.assignment_id
- WHERE amt_assignment_id = %(amt_assignment_id)s;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
-
- return [Bonus.from_postgres(data=x) for x in res]
diff --git a/jb/managers/hit.py b/jb/managers/hit.py
deleted file mode 100644
index bbae92b..0000000
--- a/jb/managers/hit.py
+++ /dev/null
@@ -1,327 +0,0 @@
-from datetime import datetime, timezone
-
-from psycopg import sql
-
-from jb.managers.base import PostgresManager
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class HitQuestionManager(PostgresManager):
-
- def create(self, question: HitQuestion) -> None:
- assert question.id is None
- data = question.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_question (url, height)
- VALUES (%(url)s, %(height)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- question.id = pk
-
- def get_by_id(self, question_id: int) -> HitQuestion:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_question
- WHERE id = %(question_id)s
- LIMIT 1;
- """,
- params={"question_id": question_id},
- )
- assert len(res) == 1
- return HitQuestion.model_validate(res[0])
-
- def get_by_values_if_exists(self, url: str, height: int) -> HitQuestion | None:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_question
- WHERE url = %(url)s AND height = %(height)s
- LIMIT 2;
- """,
- params={"url": url, "height": height},
- )
- assert len(res) != 2, "More than 1 result!"
- if len(res) == 0:
- return None
-
- return HitQuestion.model_validate(res[0])
-
- def get_or_create(self, question: HitQuestion) -> HitQuestion:
- res = self.get_by_values_if_exists(url=question.url, height=question.height)
- if res:
- return res
- self.create(question=question)
- return question
-
-
-class HitTypeManager(PostgresManager):
-
- def create(self, hit_type: HitType) -> None:
- assert hit_type.id is None
- assert hit_type.amt_hit_type_id is not None
-
- data = hit_type.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_hittype (
- amt_hit_type_id,
- title,
- description,
- reward,
- assignment_duration,
- auto_approval_delay,
- keywords,
- min_active
- )
- VALUES (
- %(amt_hit_type_id)s,
- %(title)s,
- %(description)s,
- %(reward)s,
- %(assignment_duration)s,
- %(auto_approval_delay)s,
- %(keywords)s,
- %(min_active)s
- )
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- hit_type.id = pk
-
- def filter_active(self) -> list[HitType]:
- res = self.pg_config.execute_sql_query("""
- SELECT *
- FROM mtwerk_hittype
- WHERE min_active > 0
- LIMIT 50
- """)
-
- if len(res) == 50:
- raise ValueError("Too many HitTypes!")
-
- return [HitType.from_postgres(i) for i in res]
-
- def get(self, amt_hit_type_id: str) -> HitType:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_hittype
- WHERE amt_hit_type_id = %(amt_hit_type_id)s
- """,
- params={"amt_hit_type_id": amt_hit_type_id},
- )
- assert len(res) == 1
- return HitType.from_postgres(res[0])
-
- def get_if_exists(self, amt_hit_type_id: str) -> HitType | None:
- try:
- return self.get(amt_hit_type_id=amt_hit_type_id)
- except AssertionError:
- return None
-
- def get_or_create(self, hit_type: HitType) -> HitType:
- res = self.get_if_exists(amt_hit_type_id=hit_type.amt_hit_type_id)
- if res:
- return res
-
- self.create(hit_type=hit_type)
- return self.get(amt_hit_type_id=hit_type.amt_hit_type_id)
-
- def set_min_active(self, hit_type: HitType) -> None:
- assert hit_type.id, "must be in the db first!"
- query = sql.SQL("""
- UPDATE mtwerk_hittype
- SET min_active = %(min_active)s
- WHERE id = %(id)s
- """)
- data = {"id": hit_type.id, "min_active": hit_type.min_active}
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- row_cnt = c.rowcount
-
- assert row_cnt == 1, f"Expected 1 row updated, got {row_cnt}"
-
-
-class HitManager(PostgresManager):
-
- def create(self, hit: Hit):
- assert hit.amt_hit_id is not None
- assert hit.id is None
- data = hit.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_hit (
- amt_hit_id,
- hit_type_id,
- amt_group_id,
- status,
- review_status,
- creation_time,
- expiration,
- question_id,
- created_at,
- modified_at,
- max_assignments,
- assignment_pending_count,
- assignment_completed_count,
- assignment_available_count
- )
- VALUES (
- %(amt_hit_id)s,
- %(hit_type_id)s,
- %(amt_group_id)s,
- %(status)s,
- %(review_status)s,
- %(creation_time)s,
- %(expiration)s,
- %(question_id)s,
- %(created_at)s,
- %(modified_at)s,
- %(max_assignments)s,
- %(assignment_pending_count)s,
- %(assignment_completed_count)s,
- %(assignment_available_count)s
- )
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- hit.id = pk
- return hit
-
- def update_status(self, amt_hit_id: str, hit_status: HitStatus):
- now = datetime.now(tz=timezone.utc)
- query = sql.SQL("""
- UPDATE mtwerk_hit
- SET status = %(status)s, modified_at = %(modified_at)s
- WHERE amt_hit_id = %(amt_hit_id)s;
- """)
-
- data = {
- "amt_hit_id": amt_hit_id,
- "status": hit_status.value,
- "modified_at": now,
- }
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- assert c.rowcount == 1, c.rowcount
-
- def update_hit(self, hit: Hit):
- hit.modified_at = datetime.now(tz=timezone.utc)
- # fields expected to change:
- fields = {
- "status",
- "review_status",
- "assignment_pending_count",
- "assignment_available_count",
- "assignment_completed_count",
- "amt_hit_id",
- "modified_at",
- }
- query = sql.SQL("""
- UPDATE mtwerk_hit
- SET status = %(status)s, review_status = %(review_status)s,
- assignment_pending_count = %(assignment_pending_count)s,
- assignment_available_count = %(assignment_available_count)s,
- assignment_completed_count = %(assignment_completed_count)s,
- modified_at = %(modified_at)s
- WHERE amt_hit_id = %(amt_hit_id)s
- RETURNING id;
- """)
-
- data = hit.model_dump(mode="json", include=fields)
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- assert c.rowcount == 1, c.rowcount
- hit.id = c.fetchone()["id"] # type: ignore
-
- def get_from_amt_id(self, amt_hit_id: str) -> Hit:
- res = self.pg_config.execute_sql_query(
- """
- SELECT
- mh.id,
- mh.amt_hit_id,
- mh.amt_group_id,
- mh.status,
- mh.review_status,
- mh.creation_time,
- mh.expiration,
- mh.created_at,
- mh.modified_at,
- mh.assignment_available_count,
- mh.assignment_completed_count,
- mh.assignment_pending_count,
- mh.max_assignments,
- mht.amt_hit_type_id,
- mht.title,
- mht.description,
- mht.reward,
- mht.assignment_duration,
- mht.auto_approval_delay,
- mht.keywords,
- mq.id as question_id,
- mq.height,
- mq.url
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mh.hit_type_id = mht.id
- JOIN mtwerk_question mq ON mh.question_id = mq.id
- WHERE amt_hit_id = %(amt_hit_id)s
- LIMIT 2;
- """,
- params={"amt_hit_id": amt_hit_id},
- )
- assert len(res) == 1, "Incorrect number of results"
- res = res[0]
-
- question_xml = HitQuestion.model_validate(
- {"height": res.pop("height"), "url": res.pop("url")}
- ).xml
-
- res["question_id"] = res["question_id"]
- res["hit_question_xml"] = question_xml
-
- return Hit.from_postgres(res)
-
- def get_from_amt_id_if_exists(self, amt_hit_id: str) -> Hit | None:
- try:
- return self.get_from_amt_id(amt_hit_id=amt_hit_id)
-
- except (AssertionError, Exception):
- return None
-
- def get_active_count(self, hit_type_id: int) -> int:
- return self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) as active_count
- FROM mtwerk_hit
- WHERE status = %(status)s
- AND hit_type_id = %(hit_type_id)s;
- """,
- params={"status": HitStatus.Assignable, "hit_type_id": hit_type_id},
- )[0]["active_count"]
-
- def filter_active_ids(self, hit_type_id: int) -> set[str]:
- res = self.pg_config.execute_sql_query(
- """
- SELECT mh.amt_hit_id
- FROM mtwerk_hit mh
- WHERE hit_type_id = %(hit_type_id)s;
- """,
- params={"hit_type_id": hit_type_id},
- )
- return {x["amt_hit_id"] for x in res}
diff --git a/jb/managers/worker.py b/jb/managers/worker.py
deleted file mode 100644
index 9e7d5e3..0000000
--- a/jb/managers/worker.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from mypy_boto3_mturk.type_defs import WorkerBlockTypeDef
-
-from jb.decorators import AMT_CLIENT
-
-
-class WorkerManager:
-
- @staticmethod
- def fetch_worker_blocks() -> list[WorkerBlockTypeDef]:
- p = AMT_CLIENT.get_paginator("list_worker_blocks")
- res: list[WorkerBlockTypeDef] = []
- for item in p.paginate():
- res.extend(item["WorkerBlocks"])
- return res
diff --git a/jb/models/amt.py b/jb/models/amt.py
deleted file mode 100644
index e012741..0000000
--- a/jb/models/amt.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from decimal import Decimal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class AMTAccount(BaseModel):
- model_config = ConfigDict(extra="ignore", validate_assignment=True)
-
- # Remaining available AWS Billing usage if you have enabled AWS Billing.
- available_balance: Decimal = Field()
- onhold_balance: Decimal = Field(default=Decimal(0))
-
- # --- Properties ---
-
- @property
- def is_healthy(self) -> bool:
- # A healthy account is one with at least $2,500 worth of
- # credit available to it
- return self.available_balance >= 2_500
diff --git a/jb/models/api_response.py b/jb/models/api_response.py
deleted file mode 100644
index 6b29e51..0000000
--- a/jb/models/api_response.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from jb.models.assignment import Assignment
-from jb.models.hit import Hit
-
-
-class AssignmentResponse(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- assignment: Assignment = Field(alias="Assignment")
- hit: Hit = Field(alias="HIT")
-
- @model_validator(mode="after")
- def check_consistent_hit_id(self) -> "AssignmentResponse":
- if self.hit.id != self.assignment.hit_id:
- raise ValueError("Inconsistent Hit IDs")
- return self
diff --git a/jb/models/assignment.py b/jb/models/assignment.py
deleted file mode 100644
index fa6ccd5..0000000
--- a/jb/models/assignment.py
+++ /dev/null
@@ -1,396 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import Any, TypedDict
-from xml.etree import ElementTree
-
-from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
-from mypy_boto3_mturk.type_defs import AssignmentTypeDef
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- PositiveInt,
- TypeAdapter,
- ValidationError,
- model_validator,
-)
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-from jb.models.definitions import AssignmentStatus
-
-logger = logging.getLogger("amtjb")
-
-
-class AnswerDict(TypedDict):
- amt_assignment_id: str
- amt_worker_id: str
- tsid: str
-
-
-class AssignmentStub(BaseModel):
- # todo: we need an "AssignmentStub" model that just has
- # the IDs, this is used when a user accepts an assignment
- # but hasn't submitted it yet. We want to create it in the db
- # at that point.
-
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
-
- id: PositiveInt | None = Field(default=None)
- hit_id: PositiveInt | None = Field(default=None)
- amt_assignment_id: AMTBoto3ID = Field()
- amt_hit_id: AMTBoto3ID = Field()
- amt_worker_id: str = Field(min_length=3, max_length=50)
-
- status: AssignmentStatus = Field()
-
- # GRL Specific
- created_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was saved in the database",
- )
-
- modified_at: AwareDatetimeISO | None = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was updated / modified in the database",
- )
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- return d
-
-
-class Assignment(AssignmentStub):
- """
- The Assignment data structure represents a single assignment of a HIT to
- a Worker. The assignment tracks the Worker's efforts to complete the HIT,
- and contains the results for later retrieval.
-
- The Assignment data structure is used as a response element for the
- following operations:
-
- GetAssignment
- GetAssignmentsForHIT
-
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html
- """
-
- auto_approval_time: AwareDatetimeISO = Field(
- description="If results have been submitted, AutoApprovalTime is the "
- "date and time the results of the assignment results are "
- "considered Approved automatically if they have not already "
- "been explicitly approved or rejected by the Requester. "
- "This value is derived from the auto-approval delay "
- "specified by the Requester in the HIT. This value is "
- "omitted from the assignment if the Worker has not yet "
- "submitted results.",
- )
-
- accept_time: AwareDatetimeISO = Field(
- description="The date and time the Worker accepted the assignment.",
- )
-
- submit_time: AwareDatetimeISO = Field(
- description="The date and time the assignment was submitted. This value "
- "is omitted from the assignment if the Worker has not yet "
- "submitted results.",
- )
-
- approval_time: AwareDatetimeISO | None = Field(
- default=None,
- description="The date and time the Requester approved the results. This "
- "value is omitted from the assignment if the Requester has "
- "not yet approved the results.",
- )
- rejection_time: AwareDatetimeISO | None = Field(
- default=None,
- description="The date and time the Requester rejected the results.",
- )
-
- requester_feedback: str | None = Field(
- # Default: None. This field isn't returned with assignment data by
- # default. To request this field, specify a response group of
- # AssignmentFeedback. For information about response groups, see
- # Common Parameters.
- default=None,
- min_length=3,
- max_length=2_000,
- json_schema_extra={
- "help_text": "The feedback string included with the call to the "
- "ApproveAssignment operation or the RejectAssignment "
- "operation, if the Requester approved or rejected the "
- "assignment and specified feedback."
- },
- )
-
- answer_xml: str | None = Field(default=None, exclude=True)
-
- # GRL Specific
-
- tsid: UUIDStr | None = Field(default=None)
-
- # --- Validators ---
-
- @model_validator(mode="before")
- def set_tsid(cls, values: dict[str, Any]) -> dict[str, Any]:
- if values.get("tsid") is None and (answer_xml := values.get("answer_xml")):
- answer_dict = cls.parse_answer_xml(answer_xml)
- tsid = answer_dict.get("tsid")
- try:
- values["tsid"] = TypeAdapter(UUIDStr).validate_python(tsid)
- except ValidationError as e:
- # Don't break the model validation if a baddie messes with the tsid in the answer.
- logger.warning(e)
-
- values["tsid"] = None
- return values
-
- @model_validator(mode="after")
- def check_time_sequences(self) -> Self:
- if self.accept_time > self.submit_time:
- raise ValueError("Assignment times invalid")
-
- return self
-
- @model_validator(mode="after")
- def check_answers_alignment(self) -> Self:
- if self.answers_dict is None:
- return self
- if "amt_worker_id" not in self.answers_dict:
- raise ValueError("answers_dict missing amt_worker_id")
- if self.amt_worker_id != self.answers_dict["amt_worker_id"]:
- raise ValueError("Assignment answer invalid worker_id")
- if "amt_assignment_id" not in self.answers_dict:
- raise ValueError("answers_dict missing amt_assignment_id")
- if self.amt_assignment_id != self.answers_dict["amt_assignment_id"]:
- raise ValueError("Assignment answer invalid amt_assignment_id")
- if self.tsid:
- if "tsid" not in self.answers_dict:
- raise ValueError("answers_dict missing tsid")
- if self.answers_dict["tsid"] and self.tsid != self.answers_dict["tsid"]:
- raise ValueError("Assignment answer invalid tsid")
- return self
-
- # --- Properties ---
-
- @property
- def answers_dict(self) -> AnswerDict | None:
- # See https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html
- # https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMechanicalTurkRequester/Concepts_NotificationsArticle.html
- if self.answer_xml is None:
- return None
-
- return self.parse_answer_xml(self.answer_xml) # type: ignore
-
- @staticmethod
- def parse_answer_xml(answer_xml: str) -> dict[str, Any]:
- root = ElementTree.fromstring(answer_xml)
- ns = {
- "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd"
- }
- res = {}
-
- for a in root.findall("mt:Answer", ns):
- name = a.find("mt:QuestionIdentifier", ns).text # type: ignore
- value = a.find("mt:FreeText", ns).text # type: ignore
- res[name] = value or ""
-
- EXPECTED_KEYS = {"amt_assignment_id", "amt_worker_id", "tsid"}
- # We don't want validation to fail if a baddie inserts or changes url
- # params, which will result in new or missing keys. Amazon generates the xml
- # so that should always be correct
- # assert all(k in res for k in EXPECTED_KEYS), list(res.keys())
- res = {k: v for k, v in res.items() if k in EXPECTED_KEYS}
- return res
-
- @classmethod
- def from_amt_get_assignment(cls, data: AssignmentTypeDef) -> Self:
- assignment = cls(
- amt_assignment_id=data["AssignmentId"],
- amt_hit_id=data["HITId"],
- amt_worker_id=data["WorkerId"],
- status=AssignmentStatus[data["AssignmentStatus"]],
- auto_approval_time=data["AutoApprovalTime"].astimezone(tz=timezone.utc),
- accept_time=data["AcceptTime"].astimezone(tz=timezone.utc),
- submit_time=data["SubmitTime"].astimezone(tz=timezone.utc),
- approval_time=(
- data["ApprovalTime"].astimezone(tz=timezone.utc)
- if data.get("ApprovalTime")
- else None
- ),
- rejection_time=(
- data["RejectionTime"].astimezone(tz=timezone.utc)
- if data.get("RejectionTime")
- else None
- ),
- answer_xml=data["Answer"],
- requester_feedback=data.get("RequesterFeedback"),
- )
- return assignment
-
- def to_stub(self) -> AssignmentStub:
- return AssignmentStub.model_validate(
- self.model_dump(include=set(AssignmentStub.model_fields.keys()))
- )
-
- # --- Methods ---
- #
- # def refresh(self) -> Self:
- # from tasks.mtwerk.managers.assignment import AssignmentManager
- # return AssignmentManager.fetch_by_id(self)
- #
- # def reject(self, msg: str = REJECT_MESSAGE_UNKNOWN_ASSIGNMENT):
- # """
- # Save in the database that the Assignment was rejected, and also
- # Report to Amazon Mechanical Turk that this Assignment should be
- # rejected
- #
- # TODO: can this only occur when the Assignment is in a certain status?
- #
- # :return:
- # """
- # now = datetime.now(tz=None)
- #
- # MYSQLC.execute_sql_query("""
- # UPDATE `amt-jb`.`mtwerk_assignment`
- # SET submit_time = %s, rejection_time = %s, status = %s,
- # requester_feedback = %s
- # WHERE assignment_id = %s""",
- # params=[
- # now, now,
- # AssignmentStatus.Rejected.value,
- # msg, self.id],
- # commit=True)
- #
- # CLIENT.reject_assignment(
- # AssignmentId=self.id,
- # RequesterFeedback=msg)
- #
- # def approve(self, msg: str = "Approved."):
- # """
- # Report to Amazon Mechanical Turk that this Assignment should be
- # approved
- #
- # TODO: can this only occur when the Assignment is in a certain status?
- #
- # :return:
- # """
- # CLIENT.approve_assignment(
- # AssignmentId=self.id,
- # RequesterFeedback=msg)
- #
- # def submit_and_complete_request(self) -> Optional[str]:
- # """
- # This approves the Assignment and issues the Reward
- # amount (typically $.05)
- #
- # :return:
- # """
- # worker = self.worker
- # amount = DecimalUSDDollars(self.hit.reward)
- #
- # # If successful, returns the cashout id, otherwise, returns None
- # cashout: Optional[dict] = worker.cashout_request(
- # amount=amount,
- # cashout_method_id=AMT_ASSIGNMENT_CASHOUT_METHOD)
- #
- # if cashout is None or cashout.get('status') != PayoutStatus.PENDING:
- # return None
- #
- # cashout_id: str = cashout[id]
- #
- # approval: Optional[dict] = Bonus.manage_pending_cashout(
- # cashout_id=cashout_id,
- # action=PayoutStatus.APPROVED)
- #
- # if approval is None or approval['status'] != PayoutStatus.APPROVED:
- # return None
- #
- # completion: Optional[dict] = Bonus.manage_pending_cashout(
- # cashout_id=cashout_id,
- # action=PayoutStatus.COMPLETE)
- #
- # if completion is None or completion['status'] != PayoutStatus.COMPLETE:
- # return None
- #
- # return cashout_id
- #
- # # --- ORM ---
- #
- # def model_dump_mysql(self, *args, **kwargs) -> dict:
- # d = self.model_dump(mode='json', *args, **kwargs)
- #
- # d['auto_approval_time'] = self.auto_approval_time.replace(tzinfo=None)
- # d['accept_time'] = self.accept_time.replace(tzinfo=None)
- # d['submit_time'] = self.submit_time.replace(tzinfo=None)
- #
- # if self.approval_time:
- # d['approval_time'] = self.approval_time.replace(tzinfo=None)
- #
- # if self.rejection_time:
- # d['rejection_time'] = self.rejection_time.replace(tzinfo=None)
- #
- # # created is automatically added by the database
- # d['created'] = self.created.replace(tzinfo=None)
- #
- # if self.modified:
- # d['modified'] = self.modified.replace(tzinfo=None)
- #
- # d['tsid'] = self.answers.get('tsid')
- #
- # return d
- #
- # def save(self) -> bool:
- # """
- # Either INSERTS or UPDATES the Assignment instance to a Mysql
- # record.
- # """
- #
- # # We're modifying the record, so set the time to right now!
- # self.modified = datetime.now(tz=timezone.utc)
- #
- # query = """
- # INSERT `amt-jb`.`mtwerk_assignment` (
- # id, worker_id, hit_id, status,
- # auto_approval_time, accept_time, submit_time,
- # approval_time, rejection_time,
- # requester_feedback, created, modified, tsid
- # )
- # VALUES (
- # %(id)s, %(worker_id)s, %(hit_id)s, %(status)s,
- # %(auto_approval_time)s, %(accept_time)s, %(submit_time)s,
- # %(approval_time)s, %(rejection_time)s,
- # %(requester_feedback)s, %(created)s, %(modified)s, %(tsid)s
- # )
- # ON DUPLICATE KEY UPDATE
- # worker_id = %(worker_id)s,
- # hit_id = %(hit_id)s,
- # status = %(status)s,
- #
- # auto_approval_time = %(auto_approval_time)s,
- # accept_time = %(accept_time)s,
- # submit_time = %(submit_time)s,
- #
- # approval_time = %(approval_time)s,
- # rejection_time = %(rejection_time)s,
- #
- # requester_feedback = %(requester_feedback)s,
- # -- Not going to update created just incase it changed
- # -- in pydantic for some reason
- # modified = %(modified)s,
- # tsid = %(tsid)s
- # """
- #
- # try:
- # MYSQLC.execute_sql_query(query, params=self.model_dump_mysql(), commit=True)
- # return True
- #
- # except Exception as e:
- # return False
- #
-
-
-# REJECT_MESSAGE_UNKNOWN_ASSIGNMENT = "Unknown assignment"
diff --git a/jb/models/bonus.py b/jb/models/bonus.py
deleted file mode 100644
index c6da3c4..0000000
--- a/jb/models/bonus.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from typing import Any
-
-from generalresearch.currency import USDCent
-from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
-from pydantic import BaseModel, ConfigDict, Field, PositiveInt
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-
-
-class Bonus(BaseModel):
- """
- A Bonus is created (in our DB) ONLY associated with an APPROVED
- thl-payout-event, AFTER the bonus has actually been sent to
- the worker.
- We have the payout_event uuid as the unique request token to make
- sure it only gets sent once (param in the boto request).
- """
-
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
- id: PositiveInt | None = Field(default=None)
- assignment_id: PositiveInt | None = Field(default=None)
-
- amt_worker_id: str = Field(min_length=3, max_length=50)
- amt_assignment_id: AMTBoto3ID = Field()
-
- amount: USDCent = Field()
- reason: str = Field(min_length=5)
- grant_time: AwareDatetimeISO = Field()
-
- # -- GRL Specific ---
- payout_event_id: UUIDStr = Field()
- # created: Optional[AwareDatetimeISO] = Field(default=None)
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["amount"] = self.amount.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["amount"] = USDCent(round(data["amount"] * 100))
- fields = set(cls.model_fields.keys())
- data = {k: v for k, v in data.items() if k in fields}
- return cls.model_validate(data)
diff --git a/jb/models/custom_types.py b/jb/models/custom_types.py
deleted file mode 100644
index 385c1ba..0000000
--- a/jb/models/custom_types.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import re
-from typing import Annotated
-
-from pydantic import StringConstraints
-from pydantic.functional_validators import AfterValidator
-
-
-def check_valid_amt_boto3_id(v: str) -> str:
- # Test ids from amazon have 20 chars
- if not re.fullmatch(r"[A-Z0-9]{20}|[A-Z0-9]{30}", v):
- raise ValueError("Invalid AMT Boto3 ID")
- return v
-
-
-AMTBoto3ID = Annotated[
- str,
- StringConstraints(min_length=20, max_length=30),
- AfterValidator(check_valid_amt_boto3_id),
-]
diff --git a/jb/models/definitions.py b/jb/models/definitions.py
deleted file mode 100644
index 4ae7a21..0000000
--- a/jb/models/definitions.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from enum import IntEnum
-
-
-class AssignmentStatus(IntEnum):
- # boto3.mturk specific
- Submitted = 0 # same thing as Reviewable
- Approved = 1
- Rejected = 2
-
- # GRL specific
- Accepted = 3
- PreviewState = 4
- # Invalid = 5
- # NotExist = 6
-
-
-class HitStatus(IntEnum):
- """
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_HITDataStructureArticle.html
- """
-
- # Official boto3.mturk
- Assignable = 0
- Unassignable = 1
- Reviewable = 2
- Reviewing = 3
- Disposed = 4
-
- # GRL Specific
- NotExist = 5
-
-
-class HitReviewStatus(IntEnum):
- NotReviewed = 0
- MarkedForReview = 1
- ReviewedAppropriate = 2
- ReviewedInappropriate = 3
-
-
-class ReportValue(IntEnum):
- """
- The reason a user reported a task.
- """
-
- # Used to indicate the user exited the task without giving feedback
- REASON_UNKNOWN = 0
- # Task is in the wrong language/country, unanswerable question, won't proceed to
- # next question, loading forever, error message
- TECHNICAL_ERROR = 1
- # Task ended (completed or failed, and showed the user some dialog
- # indicating the task was over), but failed to redirect
- NO_REDIRECT = 2
- # Asked for full name, home address, identity on another site, cc#
- PRIVACY_INVASION = 3
- # Asked about children, employer, medical issues, drug use, STDs, etc.
- UNCOMFORTABLE_TOPICS = 4
- # Asked to install software, signup/login to external site, access webcam,
- # promise to pay using external site, etc.
- ASKED_FOR_NOT_ALLOWED_ACTION = 5
- # Task doesn't work well on a mobile device
- BAD_ON_MOBILE = 6
- # Too long, too boring, confusing, complicated, too many
- # open-ended/free-response questions
- DIDNT_LIKE = 7
diff --git a/jb/models/errors.py b/jb/models/errors.py
deleted file mode 100644
index 1fe71df..0000000
--- a/jb/models/errors.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import re
-from enum import Enum
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from jb.models.response import ResponseMetadata
-
-
-class BotoRequestErrorOperation(str, Enum):
- GET_ASSIGNMENT = "GetAssignment"
- GET_HIT = "GetHIT"
-
-
-class TurkErrorCode(str, Enum):
- # Unclear: maybe when it's new?
- HIT_NOT_EXIST = "AWS.MechanicalTurk.HITDoesNotExist"
-
- # This seems to be for really old Assignments
- # Also maybe when it's only a Preview?
- # Happens 2+ years back, and also from past 24hrs
- INVALID_ASSIGNEMENT_STATE = "AWS.MechanicalTurk.InvalidAssignmentState"
-
- # If random assignmentId is used
- ASSIGNMENT_NOT_EXIST = "AWS.MechanicalTurk.AssignmentDoesNotExist"
-
-
-class BotoRequestErrorResponseErrorCodes(str, Enum):
- REQUEST_ERROR = "RequestError"
-
-
-class BotoRequestErrorResponseError(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- message: str = Field(alias="Message")
- code: BotoRequestErrorResponseErrorCodes = Field(alias="Code")
-
-
-class BotoRequestErrorResponse(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- error: BotoRequestErrorResponseError = Field(alias="Error")
- response_metadata: ResponseMetadata = Field(alias="ResponseMetadata")
- message: str = Field(alias="Message", min_length=50)
- error_code: TurkErrorCode = Field(alias="TurkErrorCode")
-
- @model_validator(mode="after")
- def check_consistent_hit_id(self) -> "BotoRequestErrorResponse":
-
- match self.error_code:
- case TurkErrorCode.HIT_NOT_EXIST:
- if not re.match(
- r"Hit [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message
- ):
- raise ValueError("Unknown message for TurkErrorCode.HIT_NOT_EXIST")
-
- case TurkErrorCode.INVALID_ASSIGNEMENT_STATE:
- if not re.match(
- r"This operation can be called with a status of: Reviewable,Approved,Rejected \(\d{13}\)",
- self.message,
- ):
- raise ValueError(
- "Unknown message for TurkErrorCode.INVALID_ASSIGNEMENT_STATE"
- )
-
- case TurkErrorCode.ASSIGNMENT_NOT_EXIST:
- if not re.match(
- r"Assignment [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message
- ):
- raise ValueError(
- "Unknown message for TurkErrorCode.ASSIGNMENT_NOT_EXIST"
- )
-
- return self
-
-
-class BotoRequestError(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- response: BotoRequestErrorResponse = Field()
- operation_name: BotoRequestErrorOperation = Field()
diff --git a/jb/models/event.py b/jb/models/event.py
deleted file mode 100644
index fb5735b..0000000
--- a/jb/models/event.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from typing import Any
-
-from generalresearch.models.custom_types import AwareDatetimeISO
-from mypy_boto3_mturk.literals import EventTypeType
-from pydantic import BaseModel, Field
-
-from jb.models.custom_types import AMTBoto3ID
-
-
-class MTurkEvent(BaseModel):
- """
- What AWS SNS will POST to our mturk_notifications endpoint (inside the request body)
- """
-
- event_type: EventTypeType = Field(
- json_schema_extra={"example": "AssignmentSubmitted"}
- )
- event_timestamp: AwareDatetimeISO = Field(
- json_schema_extra={"example": "2025-10-16T18:45:51Z"}
- )
- amt_hit_id: AMTBoto3ID = Field(
- json_schema_extra={"example": "12345678901234567890"}
- )
- amt_assignment_id: str = Field(
- max_length=64,
- json_schema_extra={"example": "1234567890123456789012345678901234567890"},
- )
- amt_hit_type_id: AMTBoto3ID = Field(
- json_schema_extra={"example": "09876543210987654321"}
- )
-
- @classmethod
- def from_sns(cls, data: dict[str, Any]):
- return cls.model_validate(
- {
- "event_type": data["EventType"],
- "event_timestamp": cls.fix_mturk_timestamp(data["EventTimestamp"]),
- "amt_hit_id": data["HITId"],
- "amt_assignment_id": data["AssignmentId"],
- "amt_hit_type_id": data["HITTypeId"],
- }
- )
-
- @staticmethod
- def fix_mturk_timestamp(ts: str) -> str:
- if ts.endswith("Z") and "." not in ts:
- ts = ts[:-1] + ".000Z"
- return ts
diff --git a/jb/models/hit.py b/jb/models/hit.py
deleted file mode 100644
index a550943..0000000
--- a/jb/models/hit.py
+++ /dev/null
@@ -1,260 +0,0 @@
-from datetime import datetime, timedelta, timezone
-from typing import Any
-from uuid import uuid4
-from xml.etree import ElementTree
-
-from generalresearch.currency import USDCent
-from generalresearch.models.custom_types import AwareDatetimeISO, HttpsUrlStr
-from mypy_boto3_mturk.type_defs import HITTypeDef
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- NonNegativeInt,
- PositiveInt,
-)
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-from jb.models.definitions import HitReviewStatus, HitStatus
-
-
-class HitQuestion(BaseModel):
- id: PositiveInt | None = Field(default=None)
-
- url: HttpsUrlStr = Field()
- height: PositiveInt = Field(default=1_200, ge=100, le=4_000)
-
- # --- Properties ---
-
- def to_postgres(self):
- return self.model_dump(mode="json")
-
- @property
- def xml(self) -> str:
- return f"""<?xml version="1.0" encoding="UTF-8"?>
- <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
- <ExternalURL>{self.url!s}</ExternalURL>
- <FrameHeight>{self.height}</FrameHeight>
- </ExternalQuestion>"""
-
-
-class HitTypeCommon(BaseModel):
- """
- Fields on both the HitType and Hit
- """
-
- model_config = ConfigDict(
- extra="forbid", validate_assignment=True, ser_json_timedelta="float"
- )
-
- title: str = Field(
- min_length=3,
- max_length=200,
- description="The HIT post title that appears in the listing view",
- )
- description: str = Field(
- min_length=3,
- max_length=2_000,
- description="The expand more about textarea, has a max of 2000 characters",
- )
- reward: USDCent = Field(
- description="The amount of money the Requester will pay a Worker for successfully completing the HIT."
- )
-
- assignment_duration: timedelta = Field(
- default=timedelta(minutes=90),
- description="The amount of time, in seconds, that a Worker has to complete "
- "the HIT after accepting it.",
- )
- auto_approval_delay: timedelta = Field(
- default=timedelta(days=7),
- description="The number of seconds after an assignment for the HIT has "
- "been submitted, after which the assignment is considered "
- "Approved automatically unless the Requester explicitly "
- "rejects it.",
- )
- keywords: str = Field(min_length=3, max_length=999)
-
-
-class HitType(HitTypeCommon):
- """
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/mturk/client/create_hit_type.html
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_CreateHITTypeOperation.html
- """
-
- id: PositiveInt | None = Field(default=None)
- amt_hit_type_id: AMTBoto3ID | None = Field(default=None)
-
- # --- GRL Specific ---
- min_active: NonNegativeInt = Field(default=0, le=100_000)
-
- def to_api_request_body(self) -> dict[str, Any]:
- return {
- "AutoApprovalDelayInSeconds": round(
- self.auto_approval_delay.total_seconds()
- ),
- "AssignmentDurationInSeconds": round(
- self.assignment_duration.total_seconds()
- ),
- "Reward": str(self.reward.to_usd()),
- "Title": self.title,
- "Keywords": self.keywords,
- "Description": self.description,
- }
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["reward"] = self.reward.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["reward"] = USDCent(round(data["reward"] * 100))
- return cls.model_validate(data)
-
- def generate_hit_amt_request(self, question: HitQuestion) -> dict[str, Any]:
- d = {}
- d["HITTypeId"] = self.amt_hit_type_id
- d["MaxAssignments"] = 1
- d["LifetimeInSeconds"] = round(timedelta(days=14).total_seconds())
- d["Question"] = question.xml
- d["UniqueRequestToken"] = uuid4().hex
- return d
-
-
-class Hit(HitTypeCommon):
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
-
- id: PositiveInt | None = Field(default=None)
- hit_type_id: PositiveInt | None = Field(default=None)
- question_id: PositiveInt | None = Field(default=None)
-
- amt_hit_id: AMTBoto3ID = Field()
- amt_hit_type_id: AMTBoto3ID = Field()
- amt_group_id: AMTBoto3ID = Field()
- hit_question_xml: str = Field()
-
- status: HitStatus = Field()
- review_status: HitReviewStatus = Field()
-
- # TODO: Check if this is actually ever going to be None. I type fixed it,
- # but I don't have anything to suggest it isn't requred. -- Max 2026-02-24
- creation_time: AwareDatetimeISO | None = Field(default=None, description="From aws")
- expiration: AwareDatetimeISO | None = Field(default=None)
-
- # GRL Specific
- created_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was saved in the database",
- )
- modified_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was last modified",
- )
-
- # -- Hit specific
-
- qualification_requirements: list[dict[str, Any]] | None = Field(default=None)
- max_assignments: int = Field()
-
- # # this comes back as expiration. only for the request
- # lifetime: timedelta = Field(
- # default=timedelta(days=14),
- # description="An amount of time, in seconds, after which the HIT is no longer "
- # "available for users to accept.",
- # )
- assignment_pending_count: NonNegativeInt = Field()
- assignment_available_count: NonNegativeInt = Field()
- assignment_completed_count: NonNegativeInt = Field()
-
- @classmethod
- def from_amt_create_hit(
- cls, data: HITTypeDef, question: HitQuestion, hit_type: HitType
- ) -> Self:
- assert question.id is not None
- assert hit_type.id is not None
- assert hit_type.amt_hit_type_id is not None
-
- h = cls.model_validate(
- {
- "amt_hit_id": data["HITId"],
- "amt_hit_type_id": data["HITTypeId"],
- "amt_group_id": data["HITGroupId"],
- "status": HitStatus[data["HITStatus"]],
- "review_status": HitReviewStatus[data["HITReviewStatus"]],
- "creation_time": data["CreationTime"].astimezone(tz=timezone.utc),
- "expiration": data["Expiration"].astimezone(tz=timezone.utc),
- "hit_question_xml": data["Question"],
- "qualification_requirements": data["QualificationRequirements"],
- "max_assignments": data["MaxAssignments"],
- "assignment_pending_count": data["NumberOfAssignmentsPending"],
- "assignment_available_count": data["NumberOfAssignmentsAvailable"],
- "assignment_completed_count": data["NumberOfAssignmentsCompleted"],
- "description": data["Description"],
- "keywords": data["Keywords"],
- "reward": USDCent(round(float(data["Reward"]) * 100)),
- "title": data["Title"],
- "question_id": question.id,
- "hit_type_id": hit_type.id,
- }
- )
-
- return h
-
- @classmethod
- def from_amt_get_hit(cls, data: HITTypeDef) -> Self:
- h = cls.model_validate(
- {
- "amt_hit_id": data["HITId"],
- "amt_hit_type_id": data["HITTypeId"],
- "amt_group_id": data["HITGroupId"],
- "status": HitStatus[data["HITStatus"]],
- "review_status": HitReviewStatus[data["HITReviewStatus"]],
- "creation_time": data["CreationTime"].astimezone(tz=timezone.utc),
- "expiration": data["Expiration"].astimezone(tz=timezone.utc),
- "hit_question_xml": data["Question"],
- "qualification_requirements": data["QualificationRequirements"],
- "max_assignments": data["MaxAssignments"],
- "assignment_pending_count": data["NumberOfAssignmentsPending"],
- "assignment_available_count": data["NumberOfAssignmentsAvailable"],
- "assignment_completed_count": data["NumberOfAssignmentsCompleted"],
- "description": data["Description"],
- "keywords": data["Keywords"],
- "reward": USDCent(round(float(data["Reward"]) * 100)),
- "title": data["Title"],
- "question_id": None,
- "hit_type_id": None,
- }
- )
- return h
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["reward"] = self.reward.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["reward"] = USDCent(round(data["reward"] * 100))
- return cls.model_validate(data)
-
- @property
- def hit_question(self) -> HitQuestion:
- root = ElementTree.fromstring(self.hit_question_xml)
-
- ns = {
- "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"
- }
- res = {}
-
- lookup_table = {"ExternalURL": "url", "FrameHeight": "height"}
- for a in root.findall("mt:*", ns):
- key = lookup_table[a.tag.split("}")[1]]
- val = a.text
- res[key] = val
-
- return HitQuestion.model_validate(res, from_attributes=True)
diff --git a/jb/models/response.py b/jb/models/response.py
deleted file mode 100644
index 22985af..0000000
--- a/jb/models/response.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class HTTPHeaders(BaseModel):
- request_id: str = Field(alias="x-amzn-requestid", min_length=36, max_length=36)
- content_type: str = Field(alias="content-type", min_length=26, max_length=26)
- # 'content-length': '1255',
- content_length: str = Field(alias="content-length", min_length=2)
- # 'Mon, 15 Jan 2024 23:40:32 GMT'
- date: str = Field()
-
- connection: str | None = Field(default=None) # 'close'
-
-
-class ResponseMetadata(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- request_id: str = Field(alias="RequestId", min_length=36, max_length=36)
- status_code: int = Field(alias="HTTPStatusCode", ge=200, le=599)
- headers: HTTPHeaders = Field(alias="HTTPHeaders")
- retry_attempts: int = Field(alias="RetryAttempts", ge=0)
diff --git a/jb/settings.py b/jb/settings.py
index 3508a54..0c6be21 100644
--- a/jb/settings.py
+++ b/jb/settings.py
@@ -53,13 +53,6 @@ class Settings(GRLBaseSettings):
amt_jb_db: PostgresDsn | None = Field(default=None)
- amt_endpoint: HttpUrl | None = Field(default=None)
- amt_access_id: str | None = Field(default=None)
- amt_secret_key: str | None = Field(default=None)
-
- aws_owner_id: str | None = Field(default=None)
- aws_subscription_arn: str | None = Field(default=None)
-
# --- Pytest ---
app_name: str = "AMT JB API"
@@ -73,8 +66,6 @@ class Settings(GRLBaseSettings):
influx_db: InfluxDsn | None = Field(default=None)
- sns_path: str | None = Field(default=None)
-
session_token_ttl_seconds: int = Field(default=30 * 24 * 60 * 60, gt=0) # 30 days
session_jwt_secret: SecretStr | None = Field(default=None, min_length=32)
@@ -97,18 +88,9 @@ class Settings(GRLBaseSettings):
if not self.amt_jb_db:
raise ValueError("amt_jb_db is required")
- if not self.aws_owner_id:
- raise ValueError("aws_owner_id is required")
-
- if not self.aws_subscription_arn:
- raise ValueError("aws_subscription_arn is required")
-
if not self.product_id:
raise ValueError("product_id is required")
- if not self.sns_path:
- raise ValueError("sns_path is required")
-
if not self.session_jwt_secret:
raise ValueError("session_jwt_secret is required")
diff --git a/jb/views/common.py b/jb/views/common.py
deleted file mode 100644
index f58acda..0000000
--- a/jb/views/common.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import json
-from typing import Annotated, Any
-
-import requests
-from fastapi import APIRouter, Depends, HTTPException, Request
-from fastapi.responses import HTMLResponse
-from generalresearch.redis_helper import RedisConfig
-from starlette.responses import RedirectResponse
-
-from jb.api.auth import get_authenticated_user
-from jb.config import JB_EVENTS_STREAM, settings
-from jb.decorators import get_redis_config
-from jb.flow.monitoring import emit_mturk_notification_event
-from jb.models.auth import User
-from jb.models.event import MTurkEvent
-from jb.settings import render_base_html
-
-common_router = APIRouter(prefix="", tags=["API"], include_in_schema=True)
-
-
-@common_router.get(path="/work/", response_class=HTMLResponse)
-async def work(request: Request):
- """
- HTML Page that the worker lands on in an iFrame.
- They can either be previewing the HIT, or have already accepted it.
- """
- amt_assignment_id = request.query_params.get("assignmentId", None)
- worker_id = request.query_params.get("workerId", None)
- amt_hit_id = request.query_params.get("hitId", None)
- print(f"work: {amt_assignment_id=} {worker_id=} {amt_hit_id=}")
-
- if (
- not worker_id
- or not amt_assignment_id
- or amt_assignment_id == "ASSIGNMENT_ID_NOT_AVAILABLE"
- ):
- return RedirectResponse(
- url=f"/preview/?{request.url.query}" if request.url.query else "/preview/",
- status_code=302,
- )
-
- return HTMLResponse(render_base_html())
-
-
-RedisConfigDep = Annotated[RedisConfig, Depends(get_redis_config)]
-
-
-@common_router.post(path=f"/{settings.sns_path}/", include_in_schema=False)
-async def mturk_notifications(request: Request, redis_config: RedisConfigDep):
- """
- Our SNS topic will POST to this endpoint whenever we get a new message
- """
- try:
- message = await request.json()
- except Exception:
- raise HTTPException(status_code=500, detail="No POST data")
-
- match message.get("Type"):
- case "SubscriptionConfirmation":
- subscribe_url = message["SubscribeURL"]
- print("Confirming SNS subscription...")
- requests.get(subscribe_url)
-
- case "Notification":
- msg = json.loads(message["Message"])
- print("Received MTurk event:", msg)
- enqueue_mturk_notifications(msg=msg, redis_config=redis_config)
-
- case _:
- raise HTTPException(status_code=500, detail="Invalid JSON")
-
- return {"status": "ok"}
-
-
-def enqueue_mturk_notifications(msg: dict[str, Any], redis_config: RedisConfig) -> None:
- redis_client = redis_config.create_redis_client()
-
- for evt in msg["Events"]:
- event = MTurkEvent.from_sns(evt)
- emit_mturk_notification_event(
- event_type=event.event_type, amt_hit_type_id=event.amt_hit_type_id
- )
-
- print("enqueue_mturk_notifications", event.model_dump_json())
- redis_client.xadd(JB_EVENTS_STREAM, {"data": event.model_dump_json()})
-
-
-@common_router.get(path="/work/direct/", response_class=HTMLResponse)
-async def work_direct(
- request: Request,
- user: Annotated[User, Depends(get_authenticated_user)],
-):
- """
- View for direct work (not on AMT).
- Makes sure user is authenticated.
- """
- # todo: emit event
- return HTMLResponse(render_base_html())