aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--carer/README.md20
-rw-r--r--carer/__init__.py0
-rw-r--r--carer/app/__init__.py0
-rw-r--r--carer/app/manage.py9
-rw-r--r--carer/app/settings.py24
-rw-r--r--carer/apps.py9
-rw-r--r--carer/mtwerk/__init__.py0
-rw-r--r--carer/mtwerk/migrations/0001_initial.py157
-rw-r--r--carer/mtwerk/migrations/__init__.py0
-rw-r--r--carer/mtwerk/models.py130
-rw-r--r--carer/requirements.txt4
-rw-r--r--jb/config.py15
-rw-r--r--jb/decorators.py51
-rw-r--r--jb/flow/__init__.py0
-rw-r--r--jb/flow/assignment_tasks.py124
-rw-r--r--jb/flow/events.py98
-rw-r--r--jb/flow/maintenance.py31
-rw-r--r--jb/flow/monitoring.py168
-rw-r--r--jb/flow/setup_tasks.py35
-rw-r--r--jb/flow/tasks.py109
-rw-r--r--jb/main.py10
-rw-r--r--jb/managers/amt.py270
-rw-r--r--jb/managers/assignment.py240
-rw-r--r--jb/managers/bonus.py51
-rw-r--r--jb/managers/hit.py327
-rw-r--r--jb/managers/worker.py14
-rw-r--r--jb/models/amt.py19
-rw-r--r--jb/models/api_response.py17
-rw-r--r--jb/models/assignment.py396
-rw-r--r--jb/models/bonus.py48
-rw-r--r--jb/models/custom_types.py19
-rw-r--r--jb/models/definitions.py64
-rw-r--r--jb/models/errors.py80
-rw-r--r--jb/models/event.py48
-rw-r--r--jb/models/hit.py260
-rw-r--r--jb/models/response.py21
-rw-r--r--jb/settings.py18
-rw-r--r--jb/views/common.py98
-rw-r--r--requirements.txt166
-rw-r--r--tests/__init__.py7
-rw-r--r--tests/conftest.py319
-rw-r--r--tests/fixtures/amt.py264
-rw-r--r--tests/fixtures/flow.py308
-rw-r--r--tests/fixtures/http.py50
-rw-r--r--tests/fixtures/managers.py88
-rw-r--r--tests/fixtures/models.py296
-rw-r--r--tests/flow/__init__.py16
-rw-r--r--tests/flow/test_tasks.py352
-rw-r--r--tests/http/test_auth.py14
-rw-r--r--tests/http/test_magic.py7
-rw-r--r--tests/http/test_notifications.py115
-rw-r--r--tests/http/test_preview.py71
-rw-r--r--tests/http/test_wallet.py96
-rw-r--r--tests/http/test_work.py122
-rw-r--r--tests/managers/__init__.py0
-rw-r--r--tests/managers/test_amt.py132
-rw-r--r--tests/managers/test_hit.py50
-rw-r--r--tests/models/__init__.py0
-rw-r--r--tests/models/test_assignment.py42
-rw-r--r--tests/models/test_event.py12
-rw-r--r--tests/models/test_hit.py11
-rw-r--r--tests/test_postgres.py69
62 files changed, 193 insertions, 5398 deletions
diff --git a/carer/README.md b/carer/README.md
deleted file mode 100644
index e5f9123..0000000
--- a/carer/README.md
+++ /dev/null
@@ -1,20 +0,0 @@
-# amt-jb
-
-This refers to the Django carer that handles DB models
-
-## Environment
-
-- amt-jb-carer: (amt-jb-carer-venv) Python 3.13.7
-
-
-```shell
-
-python3.13 -m venv /root/amt-jb-carer-venv
-source /root/amt-jb-carer-venv/bin/activate
-pip install -U pip setuptools wheel
-cd ~/amt-jb/carer/
-pip install -r requirements.txt
-
-python3 manage.py makemigrations mtwerk --settings=carer.settings.production
-python3 manage.py migrate mtwerk --settings=carer.settings.production
-``` \ No newline at end of file
diff --git a/carer/__init__.py b/carer/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/carer/__init__.py
+++ /dev/null
diff --git a/carer/app/__init__.py b/carer/app/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/carer/app/__init__.py
+++ /dev/null
diff --git a/carer/app/manage.py b/carer/app/manage.py
deleted file mode 100644
index 9ae482e..0000000
--- a/carer/app/manage.py
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if __name__ == "__main__":
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "carer.app.settings")
- from django.core.management import execute_from_command_line
-
- execute_from_command_line(sys.argv)
diff --git a/carer/app/settings.py b/carer/app/settings.py
deleted file mode 100644
index 55dd93b..0000000
--- a/carer/app/settings.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import os
-
-INSTALLED_APPS = [
- "django.contrib.postgres",
- "django.contrib.contenttypes",
- "carer.mtwerk",
-]
-
-DATABASES = {
- "default": {
- "ENGINE": "django.db.backends.postgresql",
- "NAME": os.environ.get("DB_NAME", "thl-test"),
- "USER": os.environ.get("DB_USER", "postgres"),
- "PASSWORD": os.environ.get("DB_PASSWORD", "password"),
- "HOST": os.environ.get("DB_HOST", "127.0.0.1"),
- "PORT": "5432",
- }
-}
-DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
-LANGUAGE_CODE = "en-us"
-TIME_ZONE = "UTC"
-USE_I18N = True
-USE_L10N = True
-USE_TZ = True
diff --git a/carer/apps.py b/carer/apps.py
deleted file mode 100644
index c96687b..0000000
--- a/carer/apps.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from django.apps import AppConfig
-
-
-class MTwerkSchemaConfig(AppConfig):
- name = "carer.mtwerk"
- label = "mtwerk"
-
- def ready(self):
- from .mtwerk import models # pycharm: keep
diff --git a/carer/mtwerk/__init__.py b/carer/mtwerk/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/carer/mtwerk/__init__.py
+++ /dev/null
diff --git a/carer/mtwerk/migrations/0001_initial.py b/carer/mtwerk/migrations/0001_initial.py
deleted file mode 100644
index d5c856a..0000000
--- a/carer/mtwerk/migrations/0001_initial.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Generated by Django 5.2.7 on 2025-10-28 22:06
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
- initial = True
-
- dependencies = [
- ]
-
- operations = [
- migrations.CreateModel(
- name='Hit',
- fields=[
- ('id', models.BigAutoField(primary_key=True, serialize=False)),
- ('amt_hit_id', models.CharField(max_length=30, unique=True)),
- ('amt_group_id', models.CharField(max_length=30)),
- ('status', models.SmallIntegerField()),
- ('review_status', models.SmallIntegerField()),
- ('creation_time', models.DateTimeField()),
- ('expiration', models.DateTimeField()),
- ('created_at', models.DateTimeField(auto_now_add=True)),
- ('modified_at', models.DateTimeField(auto_now=True)),
- ('max_assignments', models.IntegerField(default=1)),
- ('assignment_pending_count', models.IntegerField(default=0)),
- ('assignment_available_count', models.IntegerField(default=0)),
- ('assignment_completed_count', models.IntegerField(default=0)),
- ],
- options={
- 'db_table': 'mtwerk_hit',
- },
- ),
- migrations.CreateModel(
- name='Assignment',
- fields=[
- ('id', models.BigAutoField(primary_key=True, serialize=False)),
- ('amt_assignment_id', models.CharField(max_length=30, unique=True)),
- ('amt_worker_id', models.CharField(max_length=30)),
- ('status', models.SmallIntegerField()),
- ('auto_approval_time', models.DateTimeField(null=True)),
- ('accept_time', models.DateTimeField(null=True)),
- ('submit_time', models.DateTimeField(null=True)),
- ('approval_time', models.DateTimeField(null=True)),
- ('rejection_time', models.DateTimeField(null=True)),
- ('requester_feedback', models.CharField(max_length=1024, null=True)),
- ('created_at', models.DateTimeField(auto_now_add=True)),
- ('modified_at', models.DateTimeField(auto_now=True)),
- ('tsid', models.UUIDField(null=True)),
- ('hit', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='mtwerk.hit')),
- ],
- options={
- 'db_table': 'mtwerk_assignment',
- },
- ),
- migrations.CreateModel(
- name='HitType',
- fields=[
- ('id', models.AutoField(primary_key=True, serialize=False)),
- ('amt_hit_type_id', models.CharField(max_length=30, unique=True)),
- ('title', models.CharField(max_length=200)),
- ('description', models.CharField(max_length=2000)),
- ('reward', models.DecimalField(decimal_places=2, max_digits=8)),
- ('assignment_duration', models.PositiveIntegerField()),
- ('auto_approval_delay', models.PositiveIntegerField()),
- ('keywords', models.CharField(max_length=256)),
- ('min_active', models.IntegerField(default=0)),
- ],
- options={
- 'db_table': 'mtwerk_hittype',
- 'indexes': [models.Index(fields=['amt_hit_type_id'], name='mtwerk_hitt_amt_hit_05dd10_idx'), models.Index(fields=['min_active'], name='mtwerk_hitt_min_act_e18222_idx')],
- },
- ),
- migrations.AddField(
- model_name='hit',
- name='hit_type',
- field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='mtwerk.hittype'),
- ),
- migrations.CreateModel(
- name='Question',
- fields=[
- ('id', models.AutoField(primary_key=True, serialize=False)),
- ('url', models.URLField()),
- ('height', models.IntegerField(default=1200)),
- ],
- options={
- 'db_table': 'mtwerk_question',
- 'unique_together': {('url', 'height')},
- },
- ),
- migrations.AddField(
- model_name='hit',
- name='question',
- field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='mtwerk.question'),
- ),
- migrations.CreateModel(
- name='Bonus',
- fields=[
- ('id', models.BigAutoField(primary_key=True, serialize=False)),
- ('payout_event_id', models.UUIDField(unique=True)),
- ('amt_worker_id', models.CharField(max_length=30, null=True)),
- ('amount', models.DecimalField(decimal_places=2, max_digits=10)),
- ('grant_time', models.DateTimeField()),
- ('reason', models.CharField(max_length=1024)),
- ('assignment', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='mtwerk.assignment')),
- ],
- options={
- 'indexes': [models.Index(fields=['amt_worker_id'], name='mtwerk_bonu_amt_wor_77d514_idx'), models.Index(fields=['grant_time'], name='mtwerk_bonu_grant_t_d43c6c_idx')],
- },
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['amt_worker_id'], name='mtwerk_assi_amt_wor_7dbe42_idx'),
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['status'], name='mtwerk_assi_status_4e01de_idx'),
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['submit_time'], name='mtwerk_assi_submit__429d3e_idx'),
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['created_at'], name='mtwerk_assi_created_9292fa_idx'),
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['modified_at'], name='mtwerk_assi_modifie_44f1d0_idx'),
- ),
- migrations.AddIndex(
- model_name='assignment',
- index=models.Index(fields=['tsid'], name='mtwerk_assi_tsid_bd796d_idx'),
- ),
- migrations.AddIndex(
- model_name='hit',
- index=models.Index(fields=['status'], name='mtwerk_hit_status_ae726f_idx'),
- ),
- migrations.AddIndex(
- model_name='hit',
- index=models.Index(fields=['review_status'], name='mtwerk_hit_review__a9f420_idx'),
- ),
- migrations.AddIndex(
- model_name='hit',
- index=models.Index(fields=['creation_time'], name='mtwerk_hit_creatio_48e0fd_idx'),
- ),
- migrations.AddIndex(
- model_name='hit',
- index=models.Index(fields=['created_at'], name='mtwerk_hit_created_f1614b_idx'),
- ),
- migrations.AddIndex(
- model_name='hit',
- index=models.Index(fields=['modified_at'], name='mtwerk_hit_modifie_45c2ed_idx'),
- ),
- ]
diff --git a/carer/mtwerk/migrations/__init__.py b/carer/mtwerk/migrations/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/carer/mtwerk/migrations/__init__.py
+++ /dev/null
diff --git a/carer/mtwerk/models.py b/carer/mtwerk/models.py
deleted file mode 100644
index 5cb026d..0000000
--- a/carer/mtwerk/models.py
+++ /dev/null
@@ -1,130 +0,0 @@
-from django.db import models
-
-
-class Question(models.Model):
- """The iFrame location and details for a HIT"""
-
- id = models.AutoField(primary_key=True)
-
- url = models.URLField(max_length=200)
- height = models.IntegerField(default=1200)
-
- class Meta:
- db_table = "mtwerk_question"
- unique_together = ["url", "height"]
-
-
-class HitType(models.Model):
- id = models.AutoField(primary_key=True)
- amt_hit_type_id = models.CharField(max_length=30, unique=True)
-
- title = models.CharField(max_length=200)
- description = models.CharField(max_length=2000)
- reward = models.DecimalField(decimal_places=2, max_digits=8)
- assignment_duration = models.PositiveIntegerField()
- auto_approval_delay = models.PositiveIntegerField()
- keywords = models.CharField(max_length=256)
-
- # If set, don't let the active HIT count drop below this amount
- min_active = models.IntegerField(null=False, default=0)
-
- class Meta:
- db_table = "mtwerk_hittype"
- indexes = [
- models.Index(fields=["amt_hit_type_id"]),
- models.Index(fields=["min_active"]),
- ]
-
-
-class Hit(models.Model):
- id = models.BigAutoField(primary_key=True)
- amt_hit_id = models.CharField(max_length=30, unique=True)
-
- hit_type = models.ForeignKey(HitType, on_delete=models.PROTECT)
-
- amt_group_id = models.CharField(max_length=30)
-
- status = models.SmallIntegerField(null=False)
- review_status = models.SmallIntegerField(null=False)
- creation_time = models.DateTimeField(null=False)
- expiration = models.DateTimeField(null=False)
-
- question = models.ForeignKey(Question, on_delete=models.PROTECT)
- created_at = models.DateTimeField(null=False, auto_now_add=True)
- modified_at = models.DateTimeField(null=False, auto_now=True)
-
- max_assignments = models.IntegerField(null=False, default=1)
- assignment_pending_count = models.IntegerField(null=False, default=0)
- assignment_available_count = models.IntegerField(null=False, default=0)
- assignment_completed_count = models.IntegerField(null=False, default=0)
-
- class Meta:
- db_table = "mtwerk_hit"
- indexes = [
- models.Index(fields=["status"]),
- models.Index(fields=["review_status"]),
- models.Index(fields=["creation_time"]),
- models.Index(fields=["created_at"]),
- models.Index(fields=["modified_at"]),
- ]
-
-
-class Assignment(models.Model):
- id = models.BigAutoField(primary_key=True)
- amt_assignment_id = models.CharField(max_length=30, unique=True)
-
- amt_worker_id = models.CharField(max_length=30, null=False)
-
- hit = models.ForeignKey(Hit, on_delete=models.PROTECT)
-
- status = models.SmallIntegerField(null=False)
-
- auto_approval_time = models.DateTimeField(null=True)
- accept_time = models.DateTimeField(null=True)
- submit_time = models.DateTimeField(null=True)
- approval_time = models.DateTimeField(null=True)
- rejection_time = models.DateTimeField(null=True)
-
- # Don't need this
- # answer = models.CharField
-
- requester_feedback = models.CharField(null=True, max_length=1024)
-
- # Internal tracking
- created_at = models.DateTimeField(null=False, auto_now_add=True)
- modified_at = models.DateTimeField(null=False, auto_now=True)
- tsid = models.UUIDField(null=True)
-
- class Meta:
- db_table = "mtwerk_assignment"
- indexes = [
- models.Index(fields=["amt_worker_id"]),
- models.Index(fields=["status"]),
- models.Index(fields=["submit_time"]),
- models.Index(fields=["created_at"]),
- models.Index(fields=["modified_at"]),
- models.Index(fields=["tsid"]),
- ]
-
-
-class Bonus(models.Model):
- """Issued bonuses to workers for supplemental payments"""
-
- id = models.BigAutoField(primary_key=True)
- payout_event_id = models.UUIDField(null=False, unique=True)
-
- amt_worker_id = models.CharField(max_length=30, null=True)
-
- # Not unique=True because maybe multiple Bonuses will be associated
- # with a single Assignment
- assignment = models.ForeignKey(Assignment, null=False, on_delete=models.PROTECT)
-
- amount = models.DecimalField(max_digits=10, decimal_places=2)
- grant_time = models.DateTimeField(null=False)
- reason = models.CharField(max_length=1024)
-
- class Meta:
- indexes = [
- models.Index(fields=["amt_worker_id"]),
- models.Index(fields=["grant_time"]),
- ]
diff --git a/carer/requirements.txt b/carer/requirements.txt
deleted file mode 100644
index f4f866e..0000000
--- a/carer/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-asgiref==3.10.0
-Django==5.2.7
-psycopg==3.2.10
-sqlparse==0.5.3
diff --git a/jb/config.py b/jb/config.py
index 954dfe8..dea6bec 100644
--- a/jb/config.py
+++ b/jb/config.py
@@ -9,21 +9,6 @@ if settings.debug:
else:
LOG_LEVEL = logging.WARNING
-# The SNS topic that 1) JB Mturk will send notifications to, 2) will make http POSTs
-# back to us (here)
-TOPIC_ARN = f"arn:aws:sns:us-east-2:{settings.aws_owner_id}:amt-jb"
-SUBSCRIPTION = {
- "SubscriptionArn": settings.aws_subscription_arn,
- "Owner": settings.aws_owner_id,
- "Protocol": "https",
- "Endpoint": f"https://jamesbillings67.com/{settings.sns_path}/",
- "TopicArn": TOPIC_ARN,
-}
-
-JB_EVENTS_STREAM = "amt_jb_events"
-CONSUMER_GROUP = "amt-jb-0"
-# We'll only have 1 consumer atm, change this if we don't
-CONSUMER_NAME = "amt-jb-0"
if settings.sentry:
import sentry_sdk
diff --git a/jb/decorators.py b/jb/decorators.py
index 9c7a31c..e275af5 100644
--- a/jb/decorators.py
+++ b/jb/decorators.py
@@ -1,20 +1,11 @@
import logging
-import boto3
-from botocore.config import Config
-from generalresearch.managers.base import Permission
from generalresearch.pg_helper import PostgresConfig
from generalresearch.redis_helper import RedisConfig
from influxdb import InfluxDBClient
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_sns import SNSClient
from jb.config import settings
-from jb.managers.amt import AMTManager
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
from jb.managers.gr_api import GRApiManager
-from jb.managers.hit import HitManager, HitQuestionManager, HitTypeManager
redis_config = RedisConfig(
dsn=settings.redis,
@@ -41,16 +32,6 @@ logging.basicConfig(
)
LOG = logging.getLogger("amtjb")
-CLIENT_CONFIG = Config(
- # connect_timeout (float or int) – The time in seconds till a timeout
- # exception is thrown when attempting to make a connection. The default
- # is 60 seconds.
- connect_timeout=1,
- # read_timeout (float or int) – The time in seconds till a timeout
- # exception is thrown when attempting to read from a connection. The
- # default is 60 seconds.
- read_timeout=2.5,
-)
gr_api_manager = GRApiManager(
base_url=str(settings.gr_api_host),
@@ -58,23 +39,6 @@ gr_api_manager = GRApiManager(
product_id=settings.product_id,
)
-# We shouldn't use this directly. Use our AMTManager wrapper
-AMT_CLIENT: MTurkClient = boto3.client(
- service_name="mturk",
- region_name="us-east-1",
- endpoint_url=str(settings.amt_endpoint),
- aws_access_key_id=settings.amt_access_id,
- aws_secret_access_key=settings.amt_secret_key,
- config=CLIENT_CONFIG,
-)
-
-SNS_CLIENT: SNSClient = boto3.client(
- service_name="sns",
- region_name="us-east-2",
- aws_access_key_id=settings.amt_access_id,
- aws_secret_access_key=settings.amt_secret_key,
- config=CLIENT_CONFIG,
-)
pg_config = PostgresConfig(
dsn=settings.amt_jb_db,
@@ -82,21 +46,6 @@ pg_config = PostgresConfig(
statement_timeout=1,
)
-AMTM = AMTManager(amt_client=AMT_CLIENT)
-
-HTM = HitTypeManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-HM = HitManager(pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE])
-HQM = HitQuestionManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-AM = AssignmentManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
-)
-
-BM = BonusManager(pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE])
-
influx_client = None
if settings.influx_db:
influx_client = InfluxDBClient.from_dsn(str(settings.influx_db))
diff --git a/jb/flow/__init__.py b/jb/flow/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/jb/flow/__init__.py
+++ /dev/null
diff --git a/jb/flow/assignment_tasks.py b/jb/flow/assignment_tasks.py
deleted file mode 100644
index b3c820a..0000000
--- a/jb/flow/assignment_tasks.py
+++ /dev/null
@@ -1,124 +0,0 @@
-from jb.decorators import LOG
-from jb.flow.monitoring import emit_assignment_event, emit_error_event
-from jb.managers.amt import (
- REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- AMTManager,
-)
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
-from jb.managers.hit import HitManager
-from jb.models.assignment import Assignment
-from jb.models.definitions import AssignmentStatus
-from jb.models.event import MTurkEvent
-
-
-def process_assignment_submitted(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- event: MTurkEvent,
-) -> None:
- """
- Reject any submitted assignments
- """
-
- #
- # Step 1: Attempt to get the Assignment out of the API
- #
- LOG.info(f"{event=}")
-
- # This is the assignment model from AMT. In the DB, we should only have
- # the AssignmentStub
- # This call is hitting AMT, not our db.
- # The API won't even return is unless it has been submitted. So if this
- # fails, we don't need to do anything (i.e. reject it).
- assignment = amtm.get_assignment_if_exists(
- amt_assignment_id=event.amt_assignment_id
- )
-
- if assignment is None:
- # It is not found in amt, either it is invalid, not yet submitted, or
- # already been approved/rejected, so we just do nothing ...
- # todo: maybe we confirm its state matches what we have in the db
- LOG.warning(f"No assignment found on AMT: {event.amt_assignment_id}")
- emit_error_event(
- event_type="assignment_not_found_in_amt",
- amt_hit_type_id=event.amt_hit_type_id,
- )
- return
-
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
-
- reject_assignment(
- amtm=amtm,
- am=am,
- hm=hm,
- amt_assignment_id=assignment.amt_assignment_id,
- msg=REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- amt_hit_type_id=hit.amt_hit_type_id,
- )
- review_hit(amtm=amtm, hm=hm, assignment=assignment)
- return
-
-
-def review_hit(amtm: AMTManager, hm: HitManager, assignment: Assignment) -> None:
- # Reviewable to Reviewing
- amtm.update_hit_review_status(amt_hit_id=assignment.amt_hit_id, revert=False)
- hit, _ = amtm.get_hit_if_exists(amt_hit_id=assignment.amt_hit_id)
-
- if hit is None:
- LOG.warning(f"Hit not found when trying to review hit: {assignment.amt_hit_id}")
- return
-
- # Update the db
- hm.update_hit(hit)
-
- return
-
-
-def reject_assignment(
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- amt_assignment_id: str,
- msg: str,
- amt_hit_type_id: str,
-) -> Assignment:
- # Reject in AMT, update db
-
- res = amtm.reject_assignment_if_possible(
- amt_assignment_id=amt_assignment_id, msg=msg
- )
- if res is None:
- # We failed to reject this assignment. This happens if the assignment
- # is already approved or already rejected.
- emit_error_event(
- event_type="failed_to_reject_assignment",
- amt_hit_type_id=amt_hit_type_id,
- )
- LOG.exception(f"Failed to reject assignment: {amt_assignment_id}")
-
- # We just rejected this assignment, get it from amazon again
- assignment = amtm.get_assignment(amt_assignment_id=amt_assignment_id)
- assert assignment.status == AssignmentStatus.Rejected
-
- # And update the db. The assignment may not actually exist in the db (if
- # a baddie intercepted it and is trying to game us). So, we might
- # need to create as assignment first ...
- stub = am.get_stub_if_exists(amt_assignment_id=assignment.amt_assignment_id)
- if stub is None:
- LOG.warning(
- f"Rejected assignment doesn't exist in DB. Creating ... : {amt_assignment_id}"
- )
- # Even if the assignment doesn't exist, the hit must ...
- hit = hm.get_from_amt_id(amt_hit_id=assignment.amt_hit_id)
- assignment.hit_id = hit.id
- am.create(assignment=assignment)
- am.reject(assignment=assignment)
- emit_assignment_event(
- status=AssignmentStatus.Rejected, amt_hit_type_id=amt_hit_type_id, reason=msg
- )
- LOG.warning(f"Rejected assignment: {amt_assignment_id}")
- return assignment
diff --git a/jb/flow/events.py b/jb/flow/events.py
deleted file mode 100644
index bee63fd..0000000
--- a/jb/flow/events.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import time
-from concurrent import futures
-from concurrent.futures import Executor, ThreadPoolExecutor
-from typing import cast
-
-import redis
-
-from jb.config import (
- CONSUMER_GROUP,
- CONSUMER_NAME,
- JB_EVENTS_STREAM,
-)
-from jb.decorators import LOG, get_redis
-from jb.flow.assignment_tasks import process_assignment_submitted
-from jb.flow.monitoring import emit_error_event
-from jb.models.event import MTurkEvent
-
-StreamMessages = list[tuple[str, list[tuple[bytes, dict[bytes, bytes]]]]]
-
-
-def process_mturk_events_task():
- executor = ThreadPoolExecutor(max_workers=5)
- create_consumer_group()
- while True:
- try:
- process_mturk_events(executor=executor)
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(1)
-
-
-def process_mturk_events(executor: Executor):
- while True:
- n = process_mturk_events_chunk(executor=executor)
- if n is None or n < 10:
- break
-
-
-def create_consumer_group():
- try:
- redis_client = get_redis()
- redis_client.xgroup_create(
- JB_EVENTS_STREAM, CONSUMER_GROUP, id="0", mkstream=True
- )
- except redis.exceptions.ResponseError as e:
- if "BUSYGROUP Consumer Group name already exists" in str(e):
- pass # group already exists
- else:
- raise
-
-
-def process_mturk_events_chunk(executor: Executor) -> int | None:
- redis_client = get_redis()
- msgs_raw = redis_client.xreadgroup(
- groupname=CONSUMER_GROUP,
- consumername=CONSUMER_NAME,
- streams={JB_EVENTS_STREAM: ">"},
- count=10,
- )
- if not msgs_raw:
- return None
-
- msgs = cast(StreamMessages, msgs_raw)[0][1] # The queue, we only have 1
-
- fs = []
- for msg in msgs:
- msg_id, data = msg
- msg_json: str = data["data"]
-
- event = MTurkEvent.model_validate_json(json_data=msg_json)
- if event.event_type == "AssignmentSubmitted":
- fs.append(
- executor.submit(process_assignment_submitted_event, event, str(msg_id))
- )
- else:
- LOG.info(f"Discarding {event}")
- redis_client.xdel(JB_EVENTS_STREAM, msg_id)
-
- futures.wait(fs, timeout=60)
- return len(msgs)
-
-
-def process_assignment_submitted_event(event: MTurkEvent, msg_id: str):
- from jb.decorators import AM, AMTM, BM, HM
-
- redis_client = get_redis()
-
- try:
- process_assignment_submitted(amtm=AMTM, am=AM, hm=HM, bm=BM, event=event)
- except Exception as e:
- LOG.exception(f"{event.amt_assignment_id=}, {e=}")
- emit_error_event(
- event_type="failed_process_assignment_submitted",
- amt_hit_type_id=event.amt_hit_type_id,
- )
-
- redis_client.xackdel(JB_EVENTS_STREAM, CONSUMER_GROUP, msg_id)
diff --git a/jb/flow/maintenance.py b/jb/flow/maintenance.py
deleted file mode 100644
index 4a2fe81..0000000
--- a/jb/flow/maintenance.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from jb.decorators import HM
-from jb.flow.monitoring import emit_hit_event
-from jb.managers.amt import AMTManager
-from jb.models.definitions import HitStatus
-
-
-def check_hit_status(
- amtm: AMTManager,
- amt_hit_id: str,
- amt_hit_type_id: str,
- reason: str | None = None,
-) -> HitStatus:
- """
- (this used to be called "process_hit")
- Request information from Amazon regarding the status of a HIT ID. Update
- the local state from that response.
- """
-
- hit_status = amtm.get_hit_status(amt_hit_id=amt_hit_id)
-
- # We're assuming that in the db this Hit is marked as Assignable, or
- # else we wouldn't have called this function.
- if hit_status != HitStatus.Assignable:
- # TODO: should update also assignment_pending_count,
- # assignment_available_count, assignment_completed_count
- HM.update_status(amt_hit_id=amt_hit_id, hit_status=hit_status)
- emit_hit_event(
- status=hit_status, amt_hit_type_id=amt_hit_type_id, reason=reason
- )
-
- return hit_status
diff --git a/jb/flow/monitoring.py b/jb/flow/monitoring.py
deleted file mode 100644
index 22e9e93..0000000
--- a/jb/flow/monitoring.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import socket
-
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk.literals import EventTypeType
-
-from jb.config import settings
-from jb.decorators import influx_client
-from jb.models.definitions import AssignmentStatus, HitStatus
-
-
-def write_hit_gauge(status: HitStatus, amt_hit_type_id: str, cnt: int) -> None:
-
- tags = {
- "host": socket.gethostname(), # could be "amt-jb-0"
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.hits",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def write_assignment_gauge(
- status: AssignmentStatus, amt_hit_type_id: str, cnt: int
-) -> None:
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
- point = {
- "measurement": "amt_jb.assignments",
- "tags": tags,
- "fields": {"count": cnt},
- }
- if influx_client:
- influx_client.write_points(points=[point])
-
-
-def emit_hit_event(
- status: HitStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. a HIT was created, Reviewable, etc. We don't have a "created"
- HitStatus, so it would just be when status=='Assignable'
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.hit_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_assignment_event(
- status: AssignmentStatus, amt_hit_type_id: str, reason: str | None = None
-) -> None:
- """
- e.g. an Assignment was accepted/approved/reject
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "status": status.value,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- if reason:
- tags["reason"] = reason
-
- point = {
- "measurement": "amt_jb.assignment_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_mturk_notification_event(
- event_type: EventTypeType, amt_hit_type_id: str
-) -> None:
- """
- e.g. a Mturk notification was received. We just put it in redis, we
- haven't processed it yet.
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.mturk_notification_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_error_event(event_type: str, amt_hit_type_id: str) -> None:
- """
- e.g. todo: structure the error_types
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "event_type": event_type,
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.error_events",
- "tags": tags,
- "fields": {"value": 1},
- }
-
- if influx_client:
- influx_client.write_points([point])
-
-
-def emit_bonus_event(amount: USDCent, amt_hit_type_id: str) -> None:
- """
- An AMT bonus was awarded
- """
- tags = {
- "host": socket.gethostname(),
- "service": "amt-jb",
- "amt_hit_type_id": amt_hit_type_id,
- "debug": settings.debug,
- }
-
- point = {
- "measurement": "amt_jb.bonus_events",
- "tags": tags,
- "fields": {"value": 1, "amount": int(amount)},
- }
-
- if influx_client:
- influx_client.write_points([point])
diff --git a/jb/flow/setup_tasks.py b/jb/flow/setup_tasks.py
deleted file mode 100644
index f5cda48..0000000
--- a/jb/flow/setup_tasks.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from jb.config import SUBSCRIPTION, TOPIC_ARN, settings
-from jb.decorators import AMT_CLIENT, SNS_CLIENT
-
-
-def initial_setup():
- # Run once for initial setup. Not on each server start or anything
- subscription = SNS_CLIENT.subscribe( # type: ignore
- TopicArn=TOPIC_ARN,
- Protocol="https",
- Endpoint=f"https://jamesbillings67.com/{settings.sns_path}/",
- ReturnSubscriptionArn=True,
- )
-
-
-def check_sns_configuration():
- SNS_CLIENT.get_topic_attributes(TopicArn=TOPIC_ARN)
-
- # check this TOPIC_ARN exists
- # (doesnt have permission, dont need this anyways)
- # res = SNS_CLIENT.list_topics()
- # arns = {x["TopicArn"] for x in res["Topics"]}
- # assert TOPIC_ARN in arns, f"SNS Topic {TOPIC_ARN} doesn't exist!"
-
- subs = SNS_CLIENT.list_subscriptions_by_topic(TopicArn=TOPIC_ARN)
- assert SUBSCRIPTION in subs["Subscriptions"]
-
- AMT_CLIENT.send_test_event_notification(
- Notification={
- "Destination": TOPIC_ARN,
- "Transport": "SNS",
- "Version": "2006-05-05",
- "EventTypes": ["AssignmentSubmitted"],
- },
- TestEventType="AssignmentSubmitted",
- )
diff --git a/jb/flow/tasks.py b/jb/flow/tasks.py
deleted file mode 100644
index c555021..0000000
--- a/jb/flow/tasks.py
+++ /dev/null
@@ -1,109 +0,0 @@
-import time
-from typing import TypedDict, cast
-
-from generalresearch.config import is_debug
-
-from jb.decorators import AMTM, HM, HQM, HTM, LOG, pg_config
-from jb.flow.maintenance import check_hit_status
-from jb.flow.monitoring import emit_hit_event, write_hit_gauge
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class HitRow(TypedDict):
- amt_hit_id: str
- amt_hit_type_id: str
-
-
-def check_stale_hits():
- # Check live hits that haven't been modified in a long time. They may not
- # be expired yet, but maybe something is wrong?
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- ORDER BY modified_at
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_stale_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="cleanup",
- )
-
-
-def check_expired_hits():
- # Check live/assignable hits that are expired (based on AMT's expiration time)
- res = pg_config.execute_sql_query(
- """
- SELECT amt_hit_id, amt_hit_type_id
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mht.id = mh.hit_type_id
- WHERE status = %(status)s
- AND expiration < now()
- LIMIT 100;""",
- params={"status": HitStatus.Assignable.value},
- )
- for hit in cast(list[HitRow], res):
- LOG.info(f"check_expired_hits: {hit["amt_hit_id"]}")
- check_hit_status(
- amtm=AMTM,
- amt_hit_id=hit["amt_hit_id"],
- amt_hit_type_id=hit["amt_hit_type_id"],
- reason="expired",
- )
-
-
-def create_hit_from_hittype(hit_type: HitType) -> Hit:
- if is_debug():
- raise Exception("Handle AMT Sandbox issues.")
-
- else:
- question = HQM.get_or_create(
- HitQuestion(height=800, url="https://jamesbillings67.com/work/")
- )
-
- hit = AMTM.create_hit_with_hit_type(hit_type=hit_type, question=question)
- HM.create(hit)
- emit_hit_event(status=hit.status, amt_hit_type_id=hit.amt_hit_type_id)
- return hit
-
-
-def refill_hits() -> None:
-
- for hit_type in HTM.filter_active():
- assert hit_type.id
- assert hit_type.amt_hit_type_id
-
- active_count = HM.get_active_count(hit_type_id=hit_type.id)
- LOG.info(
- f"HitType: {hit_type.amt_hit_type_id}, {hit_type.min_active=}, active_count={active_count}"
- )
- write_hit_gauge(
- status=HitStatus.Assignable,
- amt_hit_type_id=hit_type.amt_hit_type_id,
- cnt=active_count,
- )
- if active_count < hit_type.min_active:
- cnt_todo = hit_type.min_active - active_count
- LOG.info(f"Refilling {cnt_todo} hits")
- for _ in range(cnt_todo):
- create_hit_from_hittype(hit_type)
-
-
-def refill_hits_task():
- while True:
- try:
- check_expired_hits()
- check_stale_hits()
- refill_hits()
- except Exception as e:
- LOG.exception(e)
- finally:
- time.sleep(5 * 60)
diff --git a/jb/main.py b/jb/main.py
index b3b7470..52d1ca1 100644
--- a/jb/main.py
+++ b/jb/main.py
@@ -1,4 +1,3 @@
-from multiprocessing import Process
from typing import Any
from fastapi import FastAPI
@@ -9,7 +8,6 @@ from starlette.middleware.trustedhost import TrustedHostMiddleware
from jb.config import settings
from jb.settings import render_base_html
from jb.views.auth import auth_router
-from jb.views.common import common_router
from jb.views.wallet import wallet_router
app = FastAPI(
@@ -33,7 +31,6 @@ app.add_middleware(
)
app.add_middleware(TrustedHostMiddleware, allowed_hosts=["*"])
-app.include_router(router=common_router)
app.include_router(router=auth_router)
app.include_router(router=wallet_router)
@@ -60,11 +57,8 @@ def serve_react_app(full_path: str):
def schedule_tasks():
- from jb.flow.events import process_mturk_events_task
- from jb.flow.tasks import refill_hits_task
-
- Process(target=process_mturk_events_task).start()
- Process(target=refill_hits_task).start()
+ pass
+ # Process(target=example_task).start()
if not settings.debug:
diff --git a/jb/managers/amt.py b/jb/managers/amt.py
deleted file mode 100644
index 488bbb8..0000000
--- a/jb/managers/amt.py
+++ /dev/null
@@ -1,270 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import Any
-
-import botocore.exceptions
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_mturk.type_defs import (
- AssignmentTypeDef,
- BonusPaymentTypeDef,
- CreateHITTypeResponseTypeDef,
- CreateHITWithHITTypeResponseTypeDef,
- GetHITResponseTypeDef,
-)
-from pydantic import ValidationError
-
-from jb.config import TOPIC_ARN
-from jb.models.amt import AMTAccount
-from jb.models.assignment import Assignment
-from jb.models.bonus import Bonus
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-LOG = logging.getLogger("amtjb")
-
-REJECT_MESSAGE_UNKNOWN_ASSIGNMENT = "Unknown assignment"
-REJECT_MESSAGE_NO_WORK = "Assignment was submitted with no attempted work."
-REJECT_MESSAGE_BADDIE = "Quality has dropped below an acceptable level"
-APPROVAL_MESSAGE = "Thank you!"
-NO_WORK_APPROVAL_MESSAGE = (
- REJECT_MESSAGE_NO_WORK + " In the future, if you are not sent into a task, "
- "please return the assignment, otherwise it will be rejected!"
-)
-BONUS_MESSAGE = "Great job! Bonus for a survey complete"
-
-
-class AMTManager:
- """
- I am type annotating this more than needed (depending on the editor),
- however it's only because AMT on boto3 is not intuitive... at all.
- """
-
- def __init__(
- self,
- amt_client: MTurkClient,
- **kwargs, # type: ignore
- ):
- super().__init__(**kwargs)
- self.amt_client = amt_client
-
- def fetch_account(self) -> AMTAccount:
- res = self.amt_client.get_account_balance()
-
- return AMTAccount.model_validate(
- {
- "available_balance": res["AvailableBalance"],
- "onhold_balance": res.get("OnHoldBalance", 0),
- }
- )
-
- def get_hit_if_exists(self, amt_hit_id: str) -> tuple[Hit | None, str | None]:
- try:
- res: GetHITResponseTypeDef = self.amt_client.get_hit(HITId=amt_hit_id)
-
- except self.amt_client.exceptions.RequestError as e:
- msg = e.response.get("Error", {}).get("Message", "")
- return None, msg
-
- hit = Hit.from_amt_get_hit(res["HIT"])
- return hit, None
-
- def get_hit_status(self, amt_hit_id: str) -> HitStatus:
- res, msg = self.get_hit_if_exists(amt_hit_id=amt_hit_id)
-
- if res is None:
- if msg is None:
- return HitStatus.Unassignable
-
- if " does not exist. (" in msg:
- return HitStatus.Disposed
-
- else:
- LOG.warning(msg)
- return HitStatus.Unassignable
-
- return res.status
-
- def create_hit_type(self, hit_type: HitType) -> HitType:
- assert hit_type.amt_hit_type_id is None
-
- res: CreateHITTypeResponseTypeDef = self.amt_client.create_hit_type(
- **hit_type.to_api_request_body()
- ) # type: ignore
-
- hit_type.amt_hit_type_id = res["HITTypeId"]
-
- # TODO: Assert / Check that the SNS Notification was
- # successfully created.
- self.amt_client.update_notification_settings(
- HITTypeId=hit_type.amt_hit_type_id,
- Notification={
- "Destination": TOPIC_ARN,
- "Transport": "SNS",
- "Version": "2006-05-05",
- # you can add more events, see mypy_boto3_mturk.literals.EventTypeType
- "EventTypes": ["AssignmentSubmitted"],
- },
- Active=True,
- )
-
- return hit_type
-
- def create_hit_with_hit_type(self, hit_type: HitType, question: HitQuestion) -> Hit:
- """
- HITTypeId: str
- LifetimeInSeconds: int
- MaxAssignments: NotRequired[int]
- Question: NotRequired[str]
- UniqueRequestToken: NotRequired[str]
- """
- assert hit_type.id is not None
- assert hit_type.amt_hit_type_id is not None
-
- data = hit_type.generate_hit_amt_request(question=question)
- res: CreateHITWithHITTypeResponseTypeDef = (
- self.amt_client.create_hit_with_hit_type(**data)
- )
-
- return Hit.from_amt_create_hit(res["HIT"], hit_type=hit_type, question=question)
-
- def get_assignment(self, amt_assignment_id: str) -> Assignment:
- """
- You CANNOT get an Assignment if it has been only ACCEPTED (by the
- worker). The api is stupid, it will only show up once it is Submitted
- """
- res = self.amt_client.get_assignment(AssignmentId=amt_assignment_id)
- ass_res: AssignmentTypeDef = res["Assignment"]
- try:
- assignment = Assignment.from_amt_get_assignment(ass_res)
- except ValidationError as e:
- # Baddies have been known to submit assignments with purposely
- # malformed "answer" (xml) section, which will raise
- # a pydantic validation error. Try to parse again with no Answer.
- LOG.exception(e)
- ass_res["Answer"] = None
- # If it wasn't the Answer that caused the ValidationError, it'll raise again
- assignment = Assignment.from_amt_get_assignment(ass_res)
-
- # to be clear, this has not checked whether it exists in our db
- assert assignment.id is None
- return assignment
-
- def get_assignment_if_exists(self, amt_assignment_id: str) -> Assignment | None:
- expected_err_msg = f"Assignment {amt_assignment_id} does not exist"
-
- try:
- return self.get_assignment(amt_assignment_id=amt_assignment_id)
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- error_code = e.response["Error"]["Code"]
- error_msg = e.response["Error"]["Message"]
- if error_code == "RequestError" and expected_err_msg in error_msg:
- return None
- raise e
-
- def reject_assignment_if_possible(
- self, amt_assignment_id: str, msg: str = REJECT_MESSAGE_UNKNOWN_ASSIGNMENT
- ) -> dict[str, Any] | None:
-
- # Unclear to me when this would fail
- try:
- return self.amt_client.reject_assignment(
- AssignmentId=amt_assignment_id, RequesterFeedback=msg
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- return None
-
- def approve_assignment_if_possible(
- self,
- amt_assignment_id: str,
- msg: str = APPROVAL_MESSAGE,
- override_rejection: bool = False,
- ) -> dict[str, Any] | None:
-
- # Unclear to me when this would fail
- try:
- return self.amt_client.approve_assignment(
- AssignmentId=amt_assignment_id,
- RequesterFeedback=msg,
- OverrideRejection=override_rejection,
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(e)
- return None
-
- def update_hit_review_status(self, amt_hit_id: str, revert: bool = False) -> None:
- try:
- # Reviewable to Reviewing
- self.amt_client.update_hit_review_status(HITId=amt_hit_id, Revert=revert)
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(f"{amt_hit_id=}, {e}")
- error_msg = e.response["Error"]["Message"]
-
- if "does not exist" in error_msg:
- raise ValueError(error_msg)
-
- # elif "This HIT is currently in the state 'Reviewing'" in error_msg:
- # logging.warning(error_msg)
-
- def send_bonus(
- self,
- amt_worker_id: str,
- amount: USDCent,
- amt_assignment_id: str,
- reason: str,
- unique_request_token: str,
- ) -> dict[str, Any] | None:
- try:
- return self.amt_client.send_bonus(
- WorkerId=amt_worker_id,
- BonusAmount=str(amount.to_usd()),
- AssignmentId=amt_assignment_id,
- Reason=reason,
- UniqueRequestToken=unique_request_token,
- )
-
- except botocore.exceptions.ClientError as e:
- LOG.warning(f"{amt_worker_id=} {amt_assignment_id=}, {e}")
- return None
-
- def get_bonus(self, amt_assignment_id: str, payout_event_id: str) -> Bonus | None:
-
- res: list[BonusPaymentTypeDef] = self.amt_client.list_bonus_payments(
- AssignmentId=amt_assignment_id
- )["BonusPayments"]
-
- assert len(res) <= 1, (
- f"{amt_assignment_id=} Expected 1 or 0 bonuses, got {len(res)}"
- )
- d = res[0] if res else None
-
- if d:
- return Bonus.model_validate(
- {
- "amt_worker_id": d["WorkerId"],
- "amount": USDCent(round(float(d["BonusAmount"]) * 100)),
- "amt_assignment_id": d["AssignmentId"],
- "reason": d["Reason"],
- "grant_time": d["GrantTime"].astimezone(tz=timezone.utc),
- "payout_event_id": payout_event_id,
- }
- )
-
- return None
-
- def expire_all_hits(self) -> None:
- # Used in testing only (or in an emergency I guess)
- now = datetime.now(tz=timezone.utc)
- paginator = self.amt_client.get_paginator("list_hits")
-
- for page in paginator.paginate():
- for hit in page["HITs"]:
- if hit["HITStatus"] in ("Assignable", "Reviewable", "Reviewing"):
- self.amt_client.update_expiration_for_hit(
- HITId=hit["HITId"], ExpireAt=now
- )
diff --git a/jb/managers/assignment.py b/jb/managers/assignment.py
deleted file mode 100644
index 2740aee..0000000
--- a/jb/managers/assignment.py
+++ /dev/null
@@ -1,240 +0,0 @@
-from datetime import datetime, timezone
-
-from psycopg import sql
-from pydantic import NonNegativeInt, PositiveInt
-
-from jb.managers.base import PostgresManager
-from jb.models.assignment import Assignment, AssignmentStub
-from jb.models.definitions import AssignmentStatus
-
-
-class AssignmentManager(PostgresManager):
-
- def create_stub(self, stub: AssignmentStub) -> None:
- assert stub.id is None
- data = stub.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_assignment
- (amt_assignment_id, amt_worker_id, status,
- created_at, modified_at, hit_id)
- VALUES
- (%(amt_assignment_id)s, %(amt_worker_id)s, %(status)s,
- %(created_at)s, %(modified_at)s, %(hit_id)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- stub.id = pk
-
- def create(self, assignment: Assignment) -> None:
- # Typically this is NOT used (we'd create the stub when HIT is
- # accepted, then update it once the assignment is submitted), however
- # there may be cases where an assignment is submitted and the stub
- # was never created (probably due to error/downtime/baddie).
-
- assert assignment.id is None
- data = assignment.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_assignment
- (amt_assignment_id, amt_worker_id, status,
- created_at, modified_at, hit_id,
- auto_approval_time, accept_time, submit_time,
- approval_time, rejection_time, requester_feedback,
- tsid)
- VALUES
- (%(amt_assignment_id)s, %(amt_worker_id)s, %(status)s,
- %(created_at)s, %(modified_at)s, %(hit_id)s,
- %(auto_approval_time)s, %(accept_time)s, %(submit_time)s,
- %(approval_time)s, %(rejection_time)s, %(requester_feedback)s,
- %(tsid)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- assignment.id = pk
-
- def get_stub(self, amt_assignment_id: str) -> AssignmentStub:
- res = self.pg_config.execute_sql_query(
- """
- SELECT ma.id, ma.hit_id,
- ma.amt_assignment_id, ma.amt_worker_id,
- ma.status, ma.created_at, ma.modified_at,
- mh.amt_hit_id
- FROM mtwerk_assignment ma
- JOIN mtwerk_hit mh ON ma.hit_id = mh.id
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
- assert len(res) == 1
- return AssignmentStub.model_validate(res[0])
-
- def get_stub_if_exists(self, amt_assignment_id: str) -> AssignmentStub | None:
- try:
- return self.get_stub(amt_assignment_id=amt_assignment_id)
- except AssertionError:
- return None
-
- def get(self, amt_assignment_id: str) -> Assignment:
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT mh.amt_hit_id, ma.id, ma.amt_assignment_id,
- ma.amt_worker_id, ma.status, ma.auto_approval_time,
- ma.accept_time, ma.submit_time, ma.approval_time,
- ma.rejection_time, ma.requester_feedback,
- ma.created_at, ma.modified_at, ma.tsid, ma.hit_id
- FROM mtwerk_assignment ma
- JOIN mtwerk_hit mh ON ma.hit_id = mh.id
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
- assert len(res) == 1
- return Assignment.model_validate(res[0])
-
- def update_answer(self, assignment: Assignment) -> None:
- # We're assuming a stub already exists
- # The assignment was submitted, but we haven't made a decision yet
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "auto_approval_time": assignment.auto_approval_time,
- "tsid": assignment.tsid,
- "amt_assignment_id": assignment.amt_assignment_id,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- auto_approval_time = %(auto_approval_time)s,
- status = %(status)s,
- tsid = %(tsid)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def reject(self, assignment: Assignment) -> None:
- assert assignment.status == AssignmentStatus.Rejected
- assert assignment.rejection_time is not None
- assert assignment.approval_time is None
- assert assignment.requester_feedback is not None
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "rejection_time": assignment.rejection_time,
- "requester_feedback": assignment.requester_feedback,
- "amt_assignment_id": assignment.amt_assignment_id,
- "auto_approval_time": assignment.auto_approval_time,
- "accept_time": assignment.accept_time,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- rejection_time = %(rejection_time)s,
- status = %(status)s,
- requester_feedback = %(requester_feedback)s,
- auto_approval_time = %(auto_approval_time)s,
- accept_time = %(accept_time)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def approve(self, assignment: Assignment) -> None:
- assert assignment.status == AssignmentStatus.Approved
- # # We can override a rejection and the rejection time is not None
- # assert assignment.rejection_time is None
- assert assignment.approval_time is not None
- assert assignment.requester_feedback is not None
- now = datetime.now(tz=timezone.utc)
- data = {
- "status": assignment.status.value,
- "submit_time": assignment.submit_time,
- "approval_time": assignment.approval_time,
- "requester_feedback": assignment.requester_feedback,
- "amt_assignment_id": assignment.amt_assignment_id,
- "auto_approval_time": assignment.auto_approval_time,
- "accept_time": assignment.accept_time,
- "modified_at": now,
- }
- query = sql.SQL("""
- UPDATE mtwerk_assignment
- SET submit_time = %(submit_time)s,
- approval_time = %(approval_time)s,
- status = %(status)s,
- requester_feedback = %(requester_feedback)s,
- auto_approval_time = %(auto_approval_time)s,
- accept_time = %(accept_time)s,
- modified_at = %(modified_at)s
- WHERE amt_assignment_id = %(amt_assignment_id)s
- """)
- # We force this to fail if the assignment doesn't already exist in the db
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- assert c.rowcount == 1, f"Expected 1 row, got {c.rowcount}"
- conn.commit()
-
- def missing_tsid_count(
- self, amt_worker_id: str, lookback_hrs: PositiveInt = 24
- ) -> NonNegativeInt:
- """
- Look at this user's previous N hrs of submitted assignments.
- Count how many assignments they have submitted without a tsid.
- """
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) AS c
- FROM mtwerk_assignment
- WHERE amt_worker_id = %(amt_worker_id)s
- AND submit_time > NOW() - (%(lookback_interval)s)::interval
- AND tsid IS NULL;
- """,
- params={
- "amt_worker_id": amt_worker_id,
- "lookback_interval": f"{lookback_hrs} hour",
- },
- )
- return int(res[0]["c"]) # type: ignore
-
- def rejected_count(
- self, amt_worker_id: str, lookback_hrs: int = 24
- ) -> NonNegativeInt:
- """
- Look at this user's previous N hrs of submitted assignments.
- Count how many rejected assignments they have.
- """
- res = self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) AS c
- FROM mtwerk_assignment
- WHERE amt_worker_id = %(amt_worker_id)s
- AND submit_time > NOW() - (%(lookback_interval)s)::interval
- AND status = %(status)s;
- """,
- params={
- "amt_worker_id": amt_worker_id,
- "lookback_interval": f"{lookback_hrs} hour",
- "status": AssignmentStatus.Rejected.value,
- },
- )
- return int(res[0]["c"]) # type: ignore
diff --git a/jb/managers/bonus.py b/jb/managers/bonus.py
deleted file mode 100644
index b649103..0000000
--- a/jb/managers/bonus.py
+++ /dev/null
@@ -1,51 +0,0 @@
-from typing import Any
-
-from psycopg import sql
-
-from jb.managers.base import PostgresManager
-from jb.models.bonus import Bonus
-
-
-class BonusManager(PostgresManager):
-
- def create(self, bonus: Bonus) -> None:
- assert bonus.id is None
- data = bonus.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_bonus
- (payout_event_id, amt_worker_id, amount, grant_time, assignment_id, reason)
- VALUES (
- %(payout_event_id)s,
- %(amt_worker_id)s,
- %(amount)s,
- %(grant_time)s,
- (
- SELECT id
- FROM mtwerk_assignment
- WHERE amt_assignment_id = %(amt_assignment_id)s
- LIMIT 1
- ),
- %(reason)s
- )
- RETURNING id, assignment_id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- res = c.fetchone()
- conn.commit()
- bonus.id = res["id"] # type: ignore
- bonus.assignment_id = res["assignment_id"] # type: ignore
-
- def filter(self, amt_assignment_id: str) -> list[Bonus]:
- res: list[Any] = self.pg_config.execute_sql_query(
- """
- SELECT mb.*, ma.amt_assignment_id
- FROM mtwerk_bonus mb
- JOIN mtwerk_assignment ma ON ma.id = mb.assignment_id
- WHERE amt_assignment_id = %(amt_assignment_id)s;
- """,
- params={"amt_assignment_id": amt_assignment_id},
- )
-
- return [Bonus.from_postgres(data=x) for x in res]
diff --git a/jb/managers/hit.py b/jb/managers/hit.py
deleted file mode 100644
index bbae92b..0000000
--- a/jb/managers/hit.py
+++ /dev/null
@@ -1,327 +0,0 @@
-from datetime import datetime, timezone
-
-from psycopg import sql
-
-from jb.managers.base import PostgresManager
-from jb.models.definitions import HitStatus
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class HitQuestionManager(PostgresManager):
-
- def create(self, question: HitQuestion) -> None:
- assert question.id is None
- data = question.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_question (url, height)
- VALUES (%(url)s, %(height)s)
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- question.id = pk
-
- def get_by_id(self, question_id: int) -> HitQuestion:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_question
- WHERE id = %(question_id)s
- LIMIT 1;
- """,
- params={"question_id": question_id},
- )
- assert len(res) == 1
- return HitQuestion.model_validate(res[0])
-
- def get_by_values_if_exists(self, url: str, height: int) -> HitQuestion | None:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_question
- WHERE url = %(url)s AND height = %(height)s
- LIMIT 2;
- """,
- params={"url": url, "height": height},
- )
- assert len(res) != 2, "More than 1 result!"
- if len(res) == 0:
- return None
-
- return HitQuestion.model_validate(res[0])
-
- def get_or_create(self, question: HitQuestion) -> HitQuestion:
- res = self.get_by_values_if_exists(url=question.url, height=question.height)
- if res:
- return res
- self.create(question=question)
- return question
-
-
-class HitTypeManager(PostgresManager):
-
- def create(self, hit_type: HitType) -> None:
- assert hit_type.id is None
- assert hit_type.amt_hit_type_id is not None
-
- data = hit_type.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_hittype (
- amt_hit_type_id,
- title,
- description,
- reward,
- assignment_duration,
- auto_approval_delay,
- keywords,
- min_active
- )
- VALUES (
- %(amt_hit_type_id)s,
- %(title)s,
- %(description)s,
- %(reward)s,
- %(assignment_duration)s,
- %(auto_approval_delay)s,
- %(keywords)s,
- %(min_active)s
- )
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- hit_type.id = pk
-
- def filter_active(self) -> list[HitType]:
- res = self.pg_config.execute_sql_query("""
- SELECT *
- FROM mtwerk_hittype
- WHERE min_active > 0
- LIMIT 50
- """)
-
- if len(res) == 50:
- raise ValueError("Too many HitTypes!")
-
- return [HitType.from_postgres(i) for i in res]
-
- def get(self, amt_hit_type_id: str) -> HitType:
- res = self.pg_config.execute_sql_query(
- """
- SELECT *
- FROM mtwerk_hittype
- WHERE amt_hit_type_id = %(amt_hit_type_id)s
- """,
- params={"amt_hit_type_id": amt_hit_type_id},
- )
- assert len(res) == 1
- return HitType.from_postgres(res[0])
-
- def get_if_exists(self, amt_hit_type_id: str) -> HitType | None:
- try:
- return self.get(amt_hit_type_id=amt_hit_type_id)
- except AssertionError:
- return None
-
- def get_or_create(self, hit_type: HitType) -> HitType:
- res = self.get_if_exists(amt_hit_type_id=hit_type.amt_hit_type_id)
- if res:
- return res
-
- self.create(hit_type=hit_type)
- return self.get(amt_hit_type_id=hit_type.amt_hit_type_id)
-
- def set_min_active(self, hit_type: HitType) -> None:
- assert hit_type.id, "must be in the db first!"
- query = sql.SQL("""
- UPDATE mtwerk_hittype
- SET min_active = %(min_active)s
- WHERE id = %(id)s
- """)
- data = {"id": hit_type.id, "min_active": hit_type.min_active}
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- row_cnt = c.rowcount
-
- assert row_cnt == 1, f"Expected 1 row updated, got {row_cnt}"
-
-
-class HitManager(PostgresManager):
-
- def create(self, hit: Hit):
- assert hit.amt_hit_id is not None
- assert hit.id is None
- data = hit.to_postgres()
- query = sql.SQL("""
- INSERT INTO mtwerk_hit (
- amt_hit_id,
- hit_type_id,
- amt_group_id,
- status,
- review_status,
- creation_time,
- expiration,
- question_id,
- created_at,
- modified_at,
- max_assignments,
- assignment_pending_count,
- assignment_completed_count,
- assignment_available_count
- )
- VALUES (
- %(amt_hit_id)s,
- %(hit_type_id)s,
- %(amt_group_id)s,
- %(status)s,
- %(review_status)s,
- %(creation_time)s,
- %(expiration)s,
- %(question_id)s,
- %(created_at)s,
- %(modified_at)s,
- %(max_assignments)s,
- %(assignment_pending_count)s,
- %(assignment_completed_count)s,
- %(assignment_available_count)s
- )
- RETURNING id;
- """)
-
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- pk = c.fetchone()["id"] # type: ignore
- conn.commit()
- hit.id = pk
- return hit
-
- def update_status(self, amt_hit_id: str, hit_status: HitStatus):
- now = datetime.now(tz=timezone.utc)
- query = sql.SQL("""
- UPDATE mtwerk_hit
- SET status = %(status)s, modified_at = %(modified_at)s
- WHERE amt_hit_id = %(amt_hit_id)s;
- """)
-
- data = {
- "amt_hit_id": amt_hit_id,
- "status": hit_status.value,
- "modified_at": now,
- }
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- assert c.rowcount == 1, c.rowcount
-
- def update_hit(self, hit: Hit):
- hit.modified_at = datetime.now(tz=timezone.utc)
- # fields expected to change:
- fields = {
- "status",
- "review_status",
- "assignment_pending_count",
- "assignment_available_count",
- "assignment_completed_count",
- "amt_hit_id",
- "modified_at",
- }
- query = sql.SQL("""
- UPDATE mtwerk_hit
- SET status = %(status)s, review_status = %(review_status)s,
- assignment_pending_count = %(assignment_pending_count)s,
- assignment_available_count = %(assignment_available_count)s,
- assignment_completed_count = %(assignment_completed_count)s,
- modified_at = %(modified_at)s
- WHERE amt_hit_id = %(amt_hit_id)s
- RETURNING id;
- """)
-
- data = hit.model_dump(mode="json", include=fields)
- with self.pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(query, data)
- conn.commit()
- assert c.rowcount == 1, c.rowcount
- hit.id = c.fetchone()["id"] # type: ignore
-
- def get_from_amt_id(self, amt_hit_id: str) -> Hit:
- res = self.pg_config.execute_sql_query(
- """
- SELECT
- mh.id,
- mh.amt_hit_id,
- mh.amt_group_id,
- mh.status,
- mh.review_status,
- mh.creation_time,
- mh.expiration,
- mh.created_at,
- mh.modified_at,
- mh.assignment_available_count,
- mh.assignment_completed_count,
- mh.assignment_pending_count,
- mh.max_assignments,
- mht.amt_hit_type_id,
- mht.title,
- mht.description,
- mht.reward,
- mht.assignment_duration,
- mht.auto_approval_delay,
- mht.keywords,
- mq.id as question_id,
- mq.height,
- mq.url
- FROM mtwerk_hit mh
- JOIN mtwerk_hittype mht ON mh.hit_type_id = mht.id
- JOIN mtwerk_question mq ON mh.question_id = mq.id
- WHERE amt_hit_id = %(amt_hit_id)s
- LIMIT 2;
- """,
- params={"amt_hit_id": amt_hit_id},
- )
- assert len(res) == 1, "Incorrect number of results"
- res = res[0]
-
- question_xml = HitQuestion.model_validate(
- {"height": res.pop("height"), "url": res.pop("url")}
- ).xml
-
- res["question_id"] = res["question_id"]
- res["hit_question_xml"] = question_xml
-
- return Hit.from_postgres(res)
-
- def get_from_amt_id_if_exists(self, amt_hit_id: str) -> Hit | None:
- try:
- return self.get_from_amt_id(amt_hit_id=amt_hit_id)
-
- except (AssertionError, Exception):
- return None
-
- def get_active_count(self, hit_type_id: int) -> int:
- return self.pg_config.execute_sql_query(
- query="""
- SELECT COUNT(1) as active_count
- FROM mtwerk_hit
- WHERE status = %(status)s
- AND hit_type_id = %(hit_type_id)s;
- """,
- params={"status": HitStatus.Assignable, "hit_type_id": hit_type_id},
- )[0]["active_count"]
-
- def filter_active_ids(self, hit_type_id: int) -> set[str]:
- res = self.pg_config.execute_sql_query(
- """
- SELECT mh.amt_hit_id
- FROM mtwerk_hit mh
- WHERE hit_type_id = %(hit_type_id)s;
- """,
- params={"hit_type_id": hit_type_id},
- )
- return {x["amt_hit_id"] for x in res}
diff --git a/jb/managers/worker.py b/jb/managers/worker.py
deleted file mode 100644
index 9e7d5e3..0000000
--- a/jb/managers/worker.py
+++ /dev/null
@@ -1,14 +0,0 @@
-from mypy_boto3_mturk.type_defs import WorkerBlockTypeDef
-
-from jb.decorators import AMT_CLIENT
-
-
-class WorkerManager:
-
- @staticmethod
- def fetch_worker_blocks() -> list[WorkerBlockTypeDef]:
- p = AMT_CLIENT.get_paginator("list_worker_blocks")
- res: list[WorkerBlockTypeDef] = []
- for item in p.paginate():
- res.extend(item["WorkerBlocks"])
- return res
diff --git a/jb/models/amt.py b/jb/models/amt.py
deleted file mode 100644
index e012741..0000000
--- a/jb/models/amt.py
+++ /dev/null
@@ -1,19 +0,0 @@
-from decimal import Decimal
-
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class AMTAccount(BaseModel):
- model_config = ConfigDict(extra="ignore", validate_assignment=True)
-
- # Remaining available AWS Billing usage if you have enabled AWS Billing.
- available_balance: Decimal = Field()
- onhold_balance: Decimal = Field(default=Decimal(0))
-
- # --- Properties ---
-
- @property
- def is_healthy(self) -> bool:
- # A healthy account is one with at least $2,500 worth of
- # credit available to it
- return self.available_balance >= 2_500
diff --git a/jb/models/api_response.py b/jb/models/api_response.py
deleted file mode 100644
index 6b29e51..0000000
--- a/jb/models/api_response.py
+++ /dev/null
@@ -1,17 +0,0 @@
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from jb.models.assignment import Assignment
-from jb.models.hit import Hit
-
-
-class AssignmentResponse(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- assignment: Assignment = Field(alias="Assignment")
- hit: Hit = Field(alias="HIT")
-
- @model_validator(mode="after")
- def check_consistent_hit_id(self) -> "AssignmentResponse":
- if self.hit.id != self.assignment.hit_id:
- raise ValueError("Inconsistent Hit IDs")
- return self
diff --git a/jb/models/assignment.py b/jb/models/assignment.py
deleted file mode 100644
index fa6ccd5..0000000
--- a/jb/models/assignment.py
+++ /dev/null
@@ -1,396 +0,0 @@
-import logging
-from datetime import datetime, timezone
-from typing import Any, TypedDict
-from xml.etree import ElementTree
-
-from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
-from mypy_boto3_mturk.type_defs import AssignmentTypeDef
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- PositiveInt,
- TypeAdapter,
- ValidationError,
- model_validator,
-)
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-from jb.models.definitions import AssignmentStatus
-
-logger = logging.getLogger("amtjb")
-
-
-class AnswerDict(TypedDict):
- amt_assignment_id: str
- amt_worker_id: str
- tsid: str
-
-
-class AssignmentStub(BaseModel):
- # todo: we need an "AssignmentStub" model that just has
- # the IDs, this is used when a user accepts an assignment
- # but hasn't submitted it yet. We want to create it in the db
- # at that point.
-
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
-
- id: PositiveInt | None = Field(default=None)
- hit_id: PositiveInt | None = Field(default=None)
- amt_assignment_id: AMTBoto3ID = Field()
- amt_hit_id: AMTBoto3ID = Field()
- amt_worker_id: str = Field(min_length=3, max_length=50)
-
- status: AssignmentStatus = Field()
-
- # GRL Specific
- created_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was saved in the database",
- )
-
- modified_at: AwareDatetimeISO | None = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was updated / modified in the database",
- )
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- return d
-
-
-class Assignment(AssignmentStub):
- """
- The Assignment data structure represents a single assignment of a HIT to
- a Worker. The assignment tracks the Worker's efforts to complete the HIT,
- and contains the results for later retrieval.
-
- The Assignment data structure is used as a response element for the
- following operations:
-
- GetAssignment
- GetAssignmentsForHIT
-
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html
- """
-
- auto_approval_time: AwareDatetimeISO = Field(
- description="If results have been submitted, AutoApprovalTime is the "
- "date and time the results of the assignment results are "
- "considered Approved automatically if they have not already "
- "been explicitly approved or rejected by the Requester. "
- "This value is derived from the auto-approval delay "
- "specified by the Requester in the HIT. This value is "
- "omitted from the assignment if the Worker has not yet "
- "submitted results.",
- )
-
- accept_time: AwareDatetimeISO = Field(
- description="The date and time the Worker accepted the assignment.",
- )
-
- submit_time: AwareDatetimeISO = Field(
- description="The date and time the assignment was submitted. This value "
- "is omitted from the assignment if the Worker has not yet "
- "submitted results.",
- )
-
- approval_time: AwareDatetimeISO | None = Field(
- default=None,
- description="The date and time the Requester approved the results. This "
- "value is omitted from the assignment if the Requester has "
- "not yet approved the results.",
- )
- rejection_time: AwareDatetimeISO | None = Field(
- default=None,
- description="The date and time the Requester rejected the results.",
- )
-
- requester_feedback: str | None = Field(
- # Default: None. This field isn't returned with assignment data by
- # default. To request this field, specify a response group of
- # AssignmentFeedback. For information about response groups, see
- # Common Parameters.
- default=None,
- min_length=3,
- max_length=2_000,
- json_schema_extra={
- "help_text": "The feedback string included with the call to the "
- "ApproveAssignment operation or the RejectAssignment "
- "operation, if the Requester approved or rejected the "
- "assignment and specified feedback."
- },
- )
-
- answer_xml: str | None = Field(default=None, exclude=True)
-
- # GRL Specific
-
- tsid: UUIDStr | None = Field(default=None)
-
- # --- Validators ---
-
- @model_validator(mode="before")
- def set_tsid(cls, values: dict[str, Any]) -> dict[str, Any]:
- if values.get("tsid") is None and (answer_xml := values.get("answer_xml")):
- answer_dict = cls.parse_answer_xml(answer_xml)
- tsid = answer_dict.get("tsid")
- try:
- values["tsid"] = TypeAdapter(UUIDStr).validate_python(tsid)
- except ValidationError as e:
- # Don't break the model validation if a baddie messes with the tsid in the answer.
- logger.warning(e)
-
- values["tsid"] = None
- return values
-
- @model_validator(mode="after")
- def check_time_sequences(self) -> Self:
- if self.accept_time > self.submit_time:
- raise ValueError("Assignment times invalid")
-
- return self
-
- @model_validator(mode="after")
- def check_answers_alignment(self) -> Self:
- if self.answers_dict is None:
- return self
- if "amt_worker_id" not in self.answers_dict:
- raise ValueError("answers_dict missing amt_worker_id")
- if self.amt_worker_id != self.answers_dict["amt_worker_id"]:
- raise ValueError("Assignment answer invalid worker_id")
- if "amt_assignment_id" not in self.answers_dict:
- raise ValueError("answers_dict missing amt_assignment_id")
- if self.amt_assignment_id != self.answers_dict["amt_assignment_id"]:
- raise ValueError("Assignment answer invalid amt_assignment_id")
- if self.tsid:
- if "tsid" not in self.answers_dict:
- raise ValueError("answers_dict missing tsid")
- if self.answers_dict["tsid"] and self.tsid != self.answers_dict["tsid"]:
- raise ValueError("Assignment answer invalid tsid")
- return self
-
- # --- Properties ---
-
- @property
- def answers_dict(self) -> AnswerDict | None:
- # See https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html
- # https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMechanicalTurkRequester/Concepts_NotificationsArticle.html
- if self.answer_xml is None:
- return None
-
- return self.parse_answer_xml(self.answer_xml) # type: ignore
-
- @staticmethod
- def parse_answer_xml(answer_xml: str) -> dict[str, Any]:
- root = ElementTree.fromstring(answer_xml)
- ns = {
- "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd"
- }
- res = {}
-
- for a in root.findall("mt:Answer", ns):
- name = a.find("mt:QuestionIdentifier", ns).text # type: ignore
- value = a.find("mt:FreeText", ns).text # type: ignore
- res[name] = value or ""
-
- EXPECTED_KEYS = {"amt_assignment_id", "amt_worker_id", "tsid"}
- # We don't want validation to fail if a baddie inserts or changes url
- # params, which will result in new or missing keys. Amazon generates the xml
- # so that should always be correct
- # assert all(k in res for k in EXPECTED_KEYS), list(res.keys())
- res = {k: v for k, v in res.items() if k in EXPECTED_KEYS}
- return res
-
- @classmethod
- def from_amt_get_assignment(cls, data: AssignmentTypeDef) -> Self:
- assignment = cls(
- amt_assignment_id=data["AssignmentId"],
- amt_hit_id=data["HITId"],
- amt_worker_id=data["WorkerId"],
- status=AssignmentStatus[data["AssignmentStatus"]],
- auto_approval_time=data["AutoApprovalTime"].astimezone(tz=timezone.utc),
- accept_time=data["AcceptTime"].astimezone(tz=timezone.utc),
- submit_time=data["SubmitTime"].astimezone(tz=timezone.utc),
- approval_time=(
- data["ApprovalTime"].astimezone(tz=timezone.utc)
- if data.get("ApprovalTime")
- else None
- ),
- rejection_time=(
- data["RejectionTime"].astimezone(tz=timezone.utc)
- if data.get("RejectionTime")
- else None
- ),
- answer_xml=data["Answer"],
- requester_feedback=data.get("RequesterFeedback"),
- )
- return assignment
-
- def to_stub(self) -> AssignmentStub:
- return AssignmentStub.model_validate(
- self.model_dump(include=set(AssignmentStub.model_fields.keys()))
- )
-
- # --- Methods ---
- #
- # def refresh(self) -> Self:
- # from tasks.mtwerk.managers.assignment import AssignmentManager
- # return AssignmentManager.fetch_by_id(self)
- #
- # def reject(self, msg: str = REJECT_MESSAGE_UNKNOWN_ASSIGNMENT):
- # """
- # Save in the database that the Assignment was rejected, and also
- # Report to Amazon Mechanical Turk that this Assignment should be
- # rejected
- #
- # TODO: can this only occur when the Assignment is in a certain status?
- #
- # :return:
- # """
- # now = datetime.now(tz=None)
- #
- # MYSQLC.execute_sql_query("""
- # UPDATE `amt-jb`.`mtwerk_assignment`
- # SET submit_time = %s, rejection_time = %s, status = %s,
- # requester_feedback = %s
- # WHERE assignment_id = %s""",
- # params=[
- # now, now,
- # AssignmentStatus.Rejected.value,
- # msg, self.id],
- # commit=True)
- #
- # CLIENT.reject_assignment(
- # AssignmentId=self.id,
- # RequesterFeedback=msg)
- #
- # def approve(self, msg: str = "Approved."):
- # """
- # Report to Amazon Mechanical Turk that this Assignment should be
- # approved
- #
- # TODO: can this only occur when the Assignment is in a certain status?
- #
- # :return:
- # """
- # CLIENT.approve_assignment(
- # AssignmentId=self.id,
- # RequesterFeedback=msg)
- #
- # def submit_and_complete_request(self) -> Optional[str]:
- # """
- # This approves the Assignment and issues the Reward
- # amount (typically $.05)
- #
- # :return:
- # """
- # worker = self.worker
- # amount = DecimalUSDDollars(self.hit.reward)
- #
- # # If successful, returns the cashout id, otherwise, returns None
- # cashout: Optional[dict] = worker.cashout_request(
- # amount=amount,
- # cashout_method_id=AMT_ASSIGNMENT_CASHOUT_METHOD)
- #
- # if cashout is None or cashout.get('status') != PayoutStatus.PENDING:
- # return None
- #
- # cashout_id: str = cashout[id]
- #
- # approval: Optional[dict] = Bonus.manage_pending_cashout(
- # cashout_id=cashout_id,
- # action=PayoutStatus.APPROVED)
- #
- # if approval is None or approval['status'] != PayoutStatus.APPROVED:
- # return None
- #
- # completion: Optional[dict] = Bonus.manage_pending_cashout(
- # cashout_id=cashout_id,
- # action=PayoutStatus.COMPLETE)
- #
- # if completion is None or completion['status'] != PayoutStatus.COMPLETE:
- # return None
- #
- # return cashout_id
- #
- # # --- ORM ---
- #
- # def model_dump_mysql(self, *args, **kwargs) -> dict:
- # d = self.model_dump(mode='json', *args, **kwargs)
- #
- # d['auto_approval_time'] = self.auto_approval_time.replace(tzinfo=None)
- # d['accept_time'] = self.accept_time.replace(tzinfo=None)
- # d['submit_time'] = self.submit_time.replace(tzinfo=None)
- #
- # if self.approval_time:
- # d['approval_time'] = self.approval_time.replace(tzinfo=None)
- #
- # if self.rejection_time:
- # d['rejection_time'] = self.rejection_time.replace(tzinfo=None)
- #
- # # created is automatically added by the database
- # d['created'] = self.created.replace(tzinfo=None)
- #
- # if self.modified:
- # d['modified'] = self.modified.replace(tzinfo=None)
- #
- # d['tsid'] = self.answers.get('tsid')
- #
- # return d
- #
- # def save(self) -> bool:
- # """
- # Either INSERTS or UPDATES the Assignment instance to a Mysql
- # record.
- # """
- #
- # # We're modifying the record, so set the time to right now!
- # self.modified = datetime.now(tz=timezone.utc)
- #
- # query = """
- # INSERT `amt-jb`.`mtwerk_assignment` (
- # id, worker_id, hit_id, status,
- # auto_approval_time, accept_time, submit_time,
- # approval_time, rejection_time,
- # requester_feedback, created, modified, tsid
- # )
- # VALUES (
- # %(id)s, %(worker_id)s, %(hit_id)s, %(status)s,
- # %(auto_approval_time)s, %(accept_time)s, %(submit_time)s,
- # %(approval_time)s, %(rejection_time)s,
- # %(requester_feedback)s, %(created)s, %(modified)s, %(tsid)s
- # )
- # ON DUPLICATE KEY UPDATE
- # worker_id = %(worker_id)s,
- # hit_id = %(hit_id)s,
- # status = %(status)s,
- #
- # auto_approval_time = %(auto_approval_time)s,
- # accept_time = %(accept_time)s,
- # submit_time = %(submit_time)s,
- #
- # approval_time = %(approval_time)s,
- # rejection_time = %(rejection_time)s,
- #
- # requester_feedback = %(requester_feedback)s,
- # -- Not going to update created just incase it changed
- # -- in pydantic for some reason
- # modified = %(modified)s,
- # tsid = %(tsid)s
- # """
- #
- # try:
- # MYSQLC.execute_sql_query(query, params=self.model_dump_mysql(), commit=True)
- # return True
- #
- # except Exception as e:
- # return False
- #
-
-
-# REJECT_MESSAGE_UNKNOWN_ASSIGNMENT = "Unknown assignment"
diff --git a/jb/models/bonus.py b/jb/models/bonus.py
deleted file mode 100644
index c6da3c4..0000000
--- a/jb/models/bonus.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from typing import Any
-
-from generalresearch.currency import USDCent
-from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
-from pydantic import BaseModel, ConfigDict, Field, PositiveInt
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-
-
-class Bonus(BaseModel):
- """
- A Bonus is created (in our DB) ONLY associated with an APPROVED
- thl-payout-event, AFTER the bonus has actually been sent to
- the worker.
- We have the payout_event uuid as the unique request token to make
- sure it only gets sent once (param in the boto request).
- """
-
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
- id: PositiveInt | None = Field(default=None)
- assignment_id: PositiveInt | None = Field(default=None)
-
- amt_worker_id: str = Field(min_length=3, max_length=50)
- amt_assignment_id: AMTBoto3ID = Field()
-
- amount: USDCent = Field()
- reason: str = Field(min_length=5)
- grant_time: AwareDatetimeISO = Field()
-
- # -- GRL Specific ---
- payout_event_id: UUIDStr = Field()
- # created: Optional[AwareDatetimeISO] = Field(default=None)
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["amount"] = self.amount.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["amount"] = USDCent(round(data["amount"] * 100))
- fields = set(cls.model_fields.keys())
- data = {k: v for k, v in data.items() if k in fields}
- return cls.model_validate(data)
diff --git a/jb/models/custom_types.py b/jb/models/custom_types.py
deleted file mode 100644
index 385c1ba..0000000
--- a/jb/models/custom_types.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import re
-from typing import Annotated
-
-from pydantic import StringConstraints
-from pydantic.functional_validators import AfterValidator
-
-
-def check_valid_amt_boto3_id(v: str) -> str:
- # Test ids from amazon have 20 chars
- if not re.fullmatch(r"[A-Z0-9]{20}|[A-Z0-9]{30}", v):
- raise ValueError("Invalid AMT Boto3 ID")
- return v
-
-
-AMTBoto3ID = Annotated[
- str,
- StringConstraints(min_length=20, max_length=30),
- AfterValidator(check_valid_amt_boto3_id),
-]
diff --git a/jb/models/definitions.py b/jb/models/definitions.py
deleted file mode 100644
index 4ae7a21..0000000
--- a/jb/models/definitions.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from enum import IntEnum
-
-
-class AssignmentStatus(IntEnum):
- # boto3.mturk specific
- Submitted = 0 # same thing as Reviewable
- Approved = 1
- Rejected = 2
-
- # GRL specific
- Accepted = 3
- PreviewState = 4
- # Invalid = 5
- # NotExist = 6
-
-
-class HitStatus(IntEnum):
- """
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_HITDataStructureArticle.html
- """
-
- # Official boto3.mturk
- Assignable = 0
- Unassignable = 1
- Reviewable = 2
- Reviewing = 3
- Disposed = 4
-
- # GRL Specific
- NotExist = 5
-
-
-class HitReviewStatus(IntEnum):
- NotReviewed = 0
- MarkedForReview = 1
- ReviewedAppropriate = 2
- ReviewedInappropriate = 3
-
-
-class ReportValue(IntEnum):
- """
- The reason a user reported a task.
- """
-
- # Used to indicate the user exited the task without giving feedback
- REASON_UNKNOWN = 0
- # Task is in the wrong language/country, unanswerable question, won't proceed to
- # next question, loading forever, error message
- TECHNICAL_ERROR = 1
- # Task ended (completed or failed, and showed the user some dialog
- # indicating the task was over), but failed to redirect
- NO_REDIRECT = 2
- # Asked for full name, home address, identity on another site, cc#
- PRIVACY_INVASION = 3
- # Asked about children, employer, medical issues, drug use, STDs, etc.
- UNCOMFORTABLE_TOPICS = 4
- # Asked to install software, signup/login to external site, access webcam,
- # promise to pay using external site, etc.
- ASKED_FOR_NOT_ALLOWED_ACTION = 5
- # Task doesn't work well on a mobile device
- BAD_ON_MOBILE = 6
- # Too long, too boring, confusing, complicated, too many
- # open-ended/free-response questions
- DIDNT_LIKE = 7
diff --git a/jb/models/errors.py b/jb/models/errors.py
deleted file mode 100644
index 1fe71df..0000000
--- a/jb/models/errors.py
+++ /dev/null
@@ -1,80 +0,0 @@
-import re
-from enum import Enum
-
-from pydantic import BaseModel, ConfigDict, Field, model_validator
-
-from jb.models.response import ResponseMetadata
-
-
-class BotoRequestErrorOperation(str, Enum):
- GET_ASSIGNMENT = "GetAssignment"
- GET_HIT = "GetHIT"
-
-
-class TurkErrorCode(str, Enum):
- # Unclear: maybe when it's new?
- HIT_NOT_EXIST = "AWS.MechanicalTurk.HITDoesNotExist"
-
- # This seems to be for really old Assignments
- # Also maybe when it's only a Preview?
- # Happens 2+ years back, and also from past 24hrs
- INVALID_ASSIGNEMENT_STATE = "AWS.MechanicalTurk.InvalidAssignmentState"
-
- # If random assignmentId is used
- ASSIGNMENT_NOT_EXIST = "AWS.MechanicalTurk.AssignmentDoesNotExist"
-
-
-class BotoRequestErrorResponseErrorCodes(str, Enum):
- REQUEST_ERROR = "RequestError"
-
-
-class BotoRequestErrorResponseError(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- message: str = Field(alias="Message")
- code: BotoRequestErrorResponseErrorCodes = Field(alias="Code")
-
-
-class BotoRequestErrorResponse(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- error: BotoRequestErrorResponseError = Field(alias="Error")
- response_metadata: ResponseMetadata = Field(alias="ResponseMetadata")
- message: str = Field(alias="Message", min_length=50)
- error_code: TurkErrorCode = Field(alias="TurkErrorCode")
-
- @model_validator(mode="after")
- def check_consistent_hit_id(self) -> "BotoRequestErrorResponse":
-
- match self.error_code:
- case TurkErrorCode.HIT_NOT_EXIST:
- if not re.match(
- r"Hit [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message
- ):
- raise ValueError("Unknown message for TurkErrorCode.HIT_NOT_EXIST")
-
- case TurkErrorCode.INVALID_ASSIGNEMENT_STATE:
- if not re.match(
- r"This operation can be called with a status of: Reviewable,Approved,Rejected \(\d{13}\)",
- self.message,
- ):
- raise ValueError(
- "Unknown message for TurkErrorCode.INVALID_ASSIGNEMENT_STATE"
- )
-
- case TurkErrorCode.ASSIGNMENT_NOT_EXIST:
- if not re.match(
- r"Assignment [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message
- ):
- raise ValueError(
- "Unknown message for TurkErrorCode.ASSIGNMENT_NOT_EXIST"
- )
-
- return self
-
-
-class BotoRequestError(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- response: BotoRequestErrorResponse = Field()
- operation_name: BotoRequestErrorOperation = Field()
diff --git a/jb/models/event.py b/jb/models/event.py
deleted file mode 100644
index fb5735b..0000000
--- a/jb/models/event.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from typing import Any
-
-from generalresearch.models.custom_types import AwareDatetimeISO
-from mypy_boto3_mturk.literals import EventTypeType
-from pydantic import BaseModel, Field
-
-from jb.models.custom_types import AMTBoto3ID
-
-
-class MTurkEvent(BaseModel):
- """
- What AWS SNS will POST to our mturk_notifications endpoint (inside the request body)
- """
-
- event_type: EventTypeType = Field(
- json_schema_extra={"example": "AssignmentSubmitted"}
- )
- event_timestamp: AwareDatetimeISO = Field(
- json_schema_extra={"example": "2025-10-16T18:45:51Z"}
- )
- amt_hit_id: AMTBoto3ID = Field(
- json_schema_extra={"example": "12345678901234567890"}
- )
- amt_assignment_id: str = Field(
- max_length=64,
- json_schema_extra={"example": "1234567890123456789012345678901234567890"},
- )
- amt_hit_type_id: AMTBoto3ID = Field(
- json_schema_extra={"example": "09876543210987654321"}
- )
-
- @classmethod
- def from_sns(cls, data: dict[str, Any]):
- return cls.model_validate(
- {
- "event_type": data["EventType"],
- "event_timestamp": cls.fix_mturk_timestamp(data["EventTimestamp"]),
- "amt_hit_id": data["HITId"],
- "amt_assignment_id": data["AssignmentId"],
- "amt_hit_type_id": data["HITTypeId"],
- }
- )
-
- @staticmethod
- def fix_mturk_timestamp(ts: str) -> str:
- if ts.endswith("Z") and "." not in ts:
- ts = ts[:-1] + ".000Z"
- return ts
diff --git a/jb/models/hit.py b/jb/models/hit.py
deleted file mode 100644
index a550943..0000000
--- a/jb/models/hit.py
+++ /dev/null
@@ -1,260 +0,0 @@
-from datetime import datetime, timedelta, timezone
-from typing import Any
-from uuid import uuid4
-from xml.etree import ElementTree
-
-from generalresearch.currency import USDCent
-from generalresearch.models.custom_types import AwareDatetimeISO, HttpsUrlStr
-from mypy_boto3_mturk.type_defs import HITTypeDef
-from pydantic import (
- BaseModel,
- ConfigDict,
- Field,
- NonNegativeInt,
- PositiveInt,
-)
-from typing_extensions import Self
-
-from jb.models.custom_types import AMTBoto3ID
-from jb.models.definitions import HitReviewStatus, HitStatus
-
-
-class HitQuestion(BaseModel):
- id: PositiveInt | None = Field(default=None)
-
- url: HttpsUrlStr = Field()
- height: PositiveInt = Field(default=1_200, ge=100, le=4_000)
-
- # --- Properties ---
-
- def to_postgres(self):
- return self.model_dump(mode="json")
-
- @property
- def xml(self) -> str:
- return f"""<?xml version="1.0" encoding="UTF-8"?>
- <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">
- <ExternalURL>{self.url!s}</ExternalURL>
- <FrameHeight>{self.height}</FrameHeight>
- </ExternalQuestion>"""
-
-
-class HitTypeCommon(BaseModel):
- """
- Fields on both the HitType and Hit
- """
-
- model_config = ConfigDict(
- extra="forbid", validate_assignment=True, ser_json_timedelta="float"
- )
-
- title: str = Field(
- min_length=3,
- max_length=200,
- description="The HIT post title that appears in the listing view",
- )
- description: str = Field(
- min_length=3,
- max_length=2_000,
- description="The expand more about textarea, has a max of 2000 characters",
- )
- reward: USDCent = Field(
- description="The amount of money the Requester will pay a Worker for successfully completing the HIT."
- )
-
- assignment_duration: timedelta = Field(
- default=timedelta(minutes=90),
- description="The amount of time, in seconds, that a Worker has to complete "
- "the HIT after accepting it.",
- )
- auto_approval_delay: timedelta = Field(
- default=timedelta(days=7),
- description="The number of seconds after an assignment for the HIT has "
- "been submitted, after which the assignment is considered "
- "Approved automatically unless the Requester explicitly "
- "rejects it.",
- )
- keywords: str = Field(min_length=3, max_length=999)
-
-
-class HitType(HitTypeCommon):
- """
- https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/mturk/client/create_hit_type.html
- https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_CreateHITTypeOperation.html
- """
-
- id: PositiveInt | None = Field(default=None)
- amt_hit_type_id: AMTBoto3ID | None = Field(default=None)
-
- # --- GRL Specific ---
- min_active: NonNegativeInt = Field(default=0, le=100_000)
-
- def to_api_request_body(self) -> dict[str, Any]:
- return {
- "AutoApprovalDelayInSeconds": round(
- self.auto_approval_delay.total_seconds()
- ),
- "AssignmentDurationInSeconds": round(
- self.assignment_duration.total_seconds()
- ),
- "Reward": str(self.reward.to_usd()),
- "Title": self.title,
- "Keywords": self.keywords,
- "Description": self.description,
- }
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["reward"] = self.reward.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["reward"] = USDCent(round(data["reward"] * 100))
- return cls.model_validate(data)
-
- def generate_hit_amt_request(self, question: HitQuestion) -> dict[str, Any]:
- d = {}
- d["HITTypeId"] = self.amt_hit_type_id
- d["MaxAssignments"] = 1
- d["LifetimeInSeconds"] = round(timedelta(days=14).total_seconds())
- d["Question"] = question.xml
- d["UniqueRequestToken"] = uuid4().hex
- return d
-
-
-class Hit(HitTypeCommon):
- model_config = ConfigDict(
- extra="forbid",
- validate_assignment=True,
- )
-
- id: PositiveInt | None = Field(default=None)
- hit_type_id: PositiveInt | None = Field(default=None)
- question_id: PositiveInt | None = Field(default=None)
-
- amt_hit_id: AMTBoto3ID = Field()
- amt_hit_type_id: AMTBoto3ID = Field()
- amt_group_id: AMTBoto3ID = Field()
- hit_question_xml: str = Field()
-
- status: HitStatus = Field()
- review_status: HitReviewStatus = Field()
-
- # TODO: Check if this is actually ever going to be None. I type fixed it,
- # but I don't have anything to suggest it isn't requred. -- Max 2026-02-24
- creation_time: AwareDatetimeISO | None = Field(default=None, description="From aws")
- expiration: AwareDatetimeISO | None = Field(default=None)
-
- # GRL Specific
- created_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was saved in the database",
- )
- modified_at: AwareDatetimeISO = Field(
- default_factory=lambda: datetime.now(tz=timezone.utc),
- description="When this record was last modified",
- )
-
- # -- Hit specific
-
- qualification_requirements: list[dict[str, Any]] | None = Field(default=None)
- max_assignments: int = Field()
-
- # # this comes back as expiration. only for the request
- # lifetime: timedelta = Field(
- # default=timedelta(days=14),
- # description="An amount of time, in seconds, after which the HIT is no longer "
- # "available for users to accept.",
- # )
- assignment_pending_count: NonNegativeInt = Field()
- assignment_available_count: NonNegativeInt = Field()
- assignment_completed_count: NonNegativeInt = Field()
-
- @classmethod
- def from_amt_create_hit(
- cls, data: HITTypeDef, question: HitQuestion, hit_type: HitType
- ) -> Self:
- assert question.id is not None
- assert hit_type.id is not None
- assert hit_type.amt_hit_type_id is not None
-
- h = cls.model_validate(
- {
- "amt_hit_id": data["HITId"],
- "amt_hit_type_id": data["HITTypeId"],
- "amt_group_id": data["HITGroupId"],
- "status": HitStatus[data["HITStatus"]],
- "review_status": HitReviewStatus[data["HITReviewStatus"]],
- "creation_time": data["CreationTime"].astimezone(tz=timezone.utc),
- "expiration": data["Expiration"].astimezone(tz=timezone.utc),
- "hit_question_xml": data["Question"],
- "qualification_requirements": data["QualificationRequirements"],
- "max_assignments": data["MaxAssignments"],
- "assignment_pending_count": data["NumberOfAssignmentsPending"],
- "assignment_available_count": data["NumberOfAssignmentsAvailable"],
- "assignment_completed_count": data["NumberOfAssignmentsCompleted"],
- "description": data["Description"],
- "keywords": data["Keywords"],
- "reward": USDCent(round(float(data["Reward"]) * 100)),
- "title": data["Title"],
- "question_id": question.id,
- "hit_type_id": hit_type.id,
- }
- )
-
- return h
-
- @classmethod
- def from_amt_get_hit(cls, data: HITTypeDef) -> Self:
- h = cls.model_validate(
- {
- "amt_hit_id": data["HITId"],
- "amt_hit_type_id": data["HITTypeId"],
- "amt_group_id": data["HITGroupId"],
- "status": HitStatus[data["HITStatus"]],
- "review_status": HitReviewStatus[data["HITReviewStatus"]],
- "creation_time": data["CreationTime"].astimezone(tz=timezone.utc),
- "expiration": data["Expiration"].astimezone(tz=timezone.utc),
- "hit_question_xml": data["Question"],
- "qualification_requirements": data["QualificationRequirements"],
- "max_assignments": data["MaxAssignments"],
- "assignment_pending_count": data["NumberOfAssignmentsPending"],
- "assignment_available_count": data["NumberOfAssignmentsAvailable"],
- "assignment_completed_count": data["NumberOfAssignmentsCompleted"],
- "description": data["Description"],
- "keywords": data["Keywords"],
- "reward": USDCent(round(float(data["Reward"]) * 100)),
- "title": data["Title"],
- "question_id": None,
- "hit_type_id": None,
- }
- )
- return h
-
- def to_postgres(self):
- d = self.model_dump(mode="json")
- d["reward"] = self.reward.to_usd()
- return d
-
- @classmethod
- def from_postgres(cls, data: dict[str, Any]) -> Self:
- data["reward"] = USDCent(round(data["reward"] * 100))
- return cls.model_validate(data)
-
- @property
- def hit_question(self) -> HitQuestion:
- root = ElementTree.fromstring(self.hit_question_xml)
-
- ns = {
- "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"
- }
- res = {}
-
- lookup_table = {"ExternalURL": "url", "FrameHeight": "height"}
- for a in root.findall("mt:*", ns):
- key = lookup_table[a.tag.split("}")[1]]
- val = a.text
- res[key] = val
-
- return HitQuestion.model_validate(res, from_attributes=True)
diff --git a/jb/models/response.py b/jb/models/response.py
deleted file mode 100644
index 22985af..0000000
--- a/jb/models/response.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from pydantic import BaseModel, ConfigDict, Field
-
-
-class HTTPHeaders(BaseModel):
- request_id: str = Field(alias="x-amzn-requestid", min_length=36, max_length=36)
- content_type: str = Field(alias="content-type", min_length=26, max_length=26)
- # 'content-length': '1255',
- content_length: str = Field(alias="content-length", min_length=2)
- # 'Mon, 15 Jan 2024 23:40:32 GMT'
- date: str = Field()
-
- connection: str | None = Field(default=None) # 'close'
-
-
-class ResponseMetadata(BaseModel):
- model_config = ConfigDict(extra="forbid", validate_assignment=True)
-
- request_id: str = Field(alias="RequestId", min_length=36, max_length=36)
- status_code: int = Field(alias="HTTPStatusCode", ge=200, le=599)
- headers: HTTPHeaders = Field(alias="HTTPHeaders")
- retry_attempts: int = Field(alias="RetryAttempts", ge=0)
diff --git a/jb/settings.py b/jb/settings.py
index 3508a54..0c6be21 100644
--- a/jb/settings.py
+++ b/jb/settings.py
@@ -53,13 +53,6 @@ class Settings(GRLBaseSettings):
amt_jb_db: PostgresDsn | None = Field(default=None)
- amt_endpoint: HttpUrl | None = Field(default=None)
- amt_access_id: str | None = Field(default=None)
- amt_secret_key: str | None = Field(default=None)
-
- aws_owner_id: str | None = Field(default=None)
- aws_subscription_arn: str | None = Field(default=None)
-
# --- Pytest ---
app_name: str = "AMT JB API"
@@ -73,8 +66,6 @@ class Settings(GRLBaseSettings):
influx_db: InfluxDsn | None = Field(default=None)
- sns_path: str | None = Field(default=None)
-
session_token_ttl_seconds: int = Field(default=30 * 24 * 60 * 60, gt=0) # 30 days
session_jwt_secret: SecretStr | None = Field(default=None, min_length=32)
@@ -97,18 +88,9 @@ class Settings(GRLBaseSettings):
if not self.amt_jb_db:
raise ValueError("amt_jb_db is required")
- if not self.aws_owner_id:
- raise ValueError("aws_owner_id is required")
-
- if not self.aws_subscription_arn:
- raise ValueError("aws_subscription_arn is required")
-
if not self.product_id:
raise ValueError("product_id is required")
- if not self.sns_path:
- raise ValueError("sns_path is required")
-
if not self.session_jwt_secret:
raise ValueError("session_jwt_secret is required")
diff --git a/jb/views/common.py b/jb/views/common.py
deleted file mode 100644
index f58acda..0000000
--- a/jb/views/common.py
+++ /dev/null
@@ -1,98 +0,0 @@
-import json
-from typing import Annotated, Any
-
-import requests
-from fastapi import APIRouter, Depends, HTTPException, Request
-from fastapi.responses import HTMLResponse
-from generalresearch.redis_helper import RedisConfig
-from starlette.responses import RedirectResponse
-
-from jb.api.auth import get_authenticated_user
-from jb.config import JB_EVENTS_STREAM, settings
-from jb.decorators import get_redis_config
-from jb.flow.monitoring import emit_mturk_notification_event
-from jb.models.auth import User
-from jb.models.event import MTurkEvent
-from jb.settings import render_base_html
-
-common_router = APIRouter(prefix="", tags=["API"], include_in_schema=True)
-
-
-@common_router.get(path="/work/", response_class=HTMLResponse)
-async def work(request: Request):
- """
- HTML Page that the worker lands on in an iFrame.
- They can either be previewing the HIT, or have already accepted it.
- """
- amt_assignment_id = request.query_params.get("assignmentId", None)
- worker_id = request.query_params.get("workerId", None)
- amt_hit_id = request.query_params.get("hitId", None)
- print(f"work: {amt_assignment_id=} {worker_id=} {amt_hit_id=}")
-
- if (
- not worker_id
- or not amt_assignment_id
- or amt_assignment_id == "ASSIGNMENT_ID_NOT_AVAILABLE"
- ):
- return RedirectResponse(
- url=f"/preview/?{request.url.query}" if request.url.query else "/preview/",
- status_code=302,
- )
-
- return HTMLResponse(render_base_html())
-
-
-RedisConfigDep = Annotated[RedisConfig, Depends(get_redis_config)]
-
-
-@common_router.post(path=f"/{settings.sns_path}/", include_in_schema=False)
-async def mturk_notifications(request: Request, redis_config: RedisConfigDep):
- """
- Our SNS topic will POST to this endpoint whenever we get a new message
- """
- try:
- message = await request.json()
- except Exception:
- raise HTTPException(status_code=500, detail="No POST data")
-
- match message.get("Type"):
- case "SubscriptionConfirmation":
- subscribe_url = message["SubscribeURL"]
- print("Confirming SNS subscription...")
- requests.get(subscribe_url)
-
- case "Notification":
- msg = json.loads(message["Message"])
- print("Received MTurk event:", msg)
- enqueue_mturk_notifications(msg=msg, redis_config=redis_config)
-
- case _:
- raise HTTPException(status_code=500, detail="Invalid JSON")
-
- return {"status": "ok"}
-
-
-def enqueue_mturk_notifications(msg: dict[str, Any], redis_config: RedisConfig) -> None:
- redis_client = redis_config.create_redis_client()
-
- for evt in msg["Events"]:
- event = MTurkEvent.from_sns(evt)
- emit_mturk_notification_event(
- event_type=event.event_type, amt_hit_type_id=event.amt_hit_type_id
- )
-
- print("enqueue_mturk_notifications", event.model_dump_json())
- redis_client.xadd(JB_EVENTS_STREAM, {"data": event.model_dump_json()})
-
-
-@common_router.get(path="/work/direct/", response_class=HTMLResponse)
-async def work_direct(
- request: Request,
- user: Annotated[User, Depends(get_authenticated_user)],
-):
- """
- View for direct work (not on AMT).
- Makes sure user is authenticated.
- """
- # todo: emit event
- return HTMLResponse(render_base_html())
diff --git a/requirements.txt b/requirements.txt
index a45c1c5..ac82bed 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,111 +1,89 @@
-git+ssh://code.g-r-l.com:6611/generalresearch@v3.6.4
-aiohappyeyeballs==2.6.1
-aiohttp==3.13.0
+git+ssh://code.g-r-l.com:6611/generalresearch@v4.0.1
+aiodns==4.0.4
+aiohappyeyeballs==2.7.1
+aiohttp==3.14.5
aiosignal==1.4.0
-annotated-types==0.7.0
-anyio==4.11.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.16.0
asgi-lifespan==2.1.0
-attrs==25.4.0
-black==25.9.0
-boto3==1.40.51
-boto3-stubs==1.40.51
-botocore==1.40.51
-botocore-stubs==1.40.51
-cachetools==6.2.1
-certifi==2025.10.5
-charset-normalizer==3.4.3
-click==8.3.0
-cloudpickle==3.1.1
-dask==2025.7.0
-decorator==5.2.1
-Deprecated==1.2.18
-distributed==2025.7.0
-dnspython==2.8.0
+attrs==26.1.0
+cachetools==7.2.8
+certifi==2026.10.23
+cffi==2.1.1
+charset-normalizer==3.5.2
+click==8.5.0
+decorator==5.3.1
+Deprecated==3.0.1
+dnspython==2.9.2
+email-normalize==3.2.0
email-validator==2.3.0
-Faker==37.11.0
-fastapi==0.119.0
-filelock==3.32.5
+Faker==40.44.0
+fastapi==0.162.0
+filelock==4.2.8
frozenlist==1.8.0
-fsspec==2025.9.0
-geoip2==5.1.0
+grip-client==0.1.3
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
-idna==3.11
+idna==3.20
influxdb==5.3.2
-iniconfig==2.1.0
-Jinja2==3.1.6
-jmespath==1.0.1
-limits==5.6.0
-locket==1.0.0
-MarkupSafe==3.0.3
-maxminddb==2.8.2
-more-itertools==10.8.0
-msgpack==1.1.2
-multidict==6.7.0
-mypy-boto3-mturk==1.40.20
-mypy-boto3-sns==1.40.1
-mypy_extensions==1.1.0
-numpy==2.3.3
-packaging==25.0
-pandas==2.3.3
-pandera==0.26.1
-partd==1.4.2
-pathspec==0.12.1
-phonenumbers==9.0.38
-platformdirs==4.5.0
-pluggy==1.6.0
-propcache==0.4.1
-protobuf==6.32.1
-psutil==7.1.1
-psycopg==3.2.10
-pyarrow==25.0.1
-pycountry==24.6.1
-pydantic==2.12.0
-pydantic-extra-types==2.10.6
-pydantic-settings==2.11.0
-pydantic_core==2.41.1
-Pygments==2.19.2
-PyJWT==2.13.0
+iniconfig==2.3.1
+limits==5.8.0
+more-itertools==11.2.1
+msgpack==1.2.4
+multidict==7.2.1
+mypy_extensions==1.2.0
+numpy==2.5.4
+opentelemetry-api==1.45.2
+packaging==26.3
+pandas==3.1.1
+pandera==0.35.0
+phonenumbers==9.0.42
+pluggy==1.7.0
+prometheus_client==0.26.0
+propcache==0.5.4
+protobuf==7.36.2
+psycopg==3.3.8
+pyarrow==26.0.1
+pycares==5.2.1
+pycountry==26.2.16
+pycparser==3.12
+pydantic==2.14.2
+pydantic-extra-types==2.11.1
+pydantic-settings==2.16.2
+pydantic_core==2.50.2
+Pygments==2.21.0
+PyJWT==2.15.1
pylibmc==1.6.3
pymemcache==4.0.0
-PyMySQL==1.1.2
-pytest==8.4.2
-python-dateutil==2.9.0.post0
-python-dotenv==1.1.1
-pytokens==0.1.10
-pytz==2025.2
-PyYAML==6.0.3
-redis==6.4.0
-requests==2.32.5
+PyMySQL==1.2.3
+pytest==9.2.0
+python-dateutil==4.0.0
+python-dotenv==1.2.4
+pytz==2026.6
+redis==8.1.0
+requests==2.34.2
requests-file==3.0.1
requests-mock==1.12.1
-s3transfer==0.14.0
-scipy==1.16.2
-sentry-sdk==2.41.0
-setuptools==80.9.0
+scipy==1.18.1
+sentry-sdk==2.73.0
+setuptools==84.2.1
six==1.17.0
slackclient==2.9.4
sniffio==1.3.1
-sortedcontainers==2.4.0
-starlette==0.48.0
-tblib==3.2.0
-tldextract==5.3.2
-toolz==1.1.0
-tornado==6.5.2
-typeguard==4.4.4
-types-awscrt==0.28.1
-types-s3transfer==0.14.0
+starlette==1.10.0
+tldextract==5.4.0
+typeguard==4.6.0
typing-inspect==0.9.0
-typing-inspection==0.4.2
-typing_extensions==4.15.0
-tzdata==2025.2
-ua-parser==1.0.1
-ua-parser-builtins==0.18.0.post1
-urllib3==2.5.0
+typing-inspection==0.4.5
+typing_extensions==4.16.0
+tzdata==2026.6
+ua-parser==1.0.2
+ua-parser-builtins==202610
+urllib3==2.8.0
user-agents==2.2.0
-uvicorn==0.37.0
-wheel==0.45.1
-wrapt==1.17.3
-yarl==1.22.0
-zict==3.0.0
+uvicorn==0.54.0
+wheel==0.48.0
+wrapt==2.5.2
+yarl==1.25.1
diff --git a/tests/__init__.py b/tests/__init__.py
index 0166072..e69de29 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -1,7 +0,0 @@
-from random import choices as rand_choices
-from string import ascii_uppercase, digits
-
-
-def generate_amt_id(length: int = 30) -> str:
- chars = ascii_uppercase + digits
- return "".join(rand_choices(chars, k=length))
diff --git a/tests/conftest.py b/tests/conftest.py
index 7a74fa0..ec5073c 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,91 +1,23 @@
from __future__ import annotations
-import os
import subprocess
-import sys
-from collections.abc import Callable, Generator
-from datetime import UTC, datetime
-from pathlib import Path
+from collections.abc import Generator
from random import randint
-from typing import TYPE_CHECKING, Any
-from uuid import uuid4
+from typing import TYPE_CHECKING
import pytest
import redis
-from fastapi.testclient import TestClient
-from generalresearch.models.custom_types import InternalHostname, PostgresDict
-from generalresearch.pg_helper import PostgresConfig, PostgresDsn
from generalresearch.redis_helper import RedisConfig
-from mypy_boto3_mturk import MTurkClient
-from pydantic import TypeAdapter
-from pytest import TempPathFactory
-
-from jb.decorators import CLIENT_CONFIG, get_redis_config
-from jb.main import app
-from tests import generate_amt_id
if TYPE_CHECKING:
from jb.settings import Settings
pytest_plugins = [
- "tests.fixtures.amt",
- "tests.fixtures.flow",
"tests.fixtures.http",
- "tests.fixtures.managers",
- "tests.fixtures.models",
]
-# --- IDs and Identifiers ---
-
-
-@pytest.fixture
-def amt_hit_id() -> str:
- return generate_amt_id()
-
-
-@pytest.fixture
-def amt_hit_type_id() -> str:
- return generate_amt_id()
-
-
-@pytest.fixture
-def amt_assignment_id() -> str:
- return generate_amt_id()
-
-
-@pytest.fixture
-def amt_worker_id() -> str:
- return generate_amt_id(length=21)
-
-
-@pytest.fixture
-def amt_group_id() -> str:
- return generate_amt_id()
-
-
-@pytest.fixture
-def tsid() -> str:
- return uuid4().hex
-
-
-@pytest.fixture
-def tsid1() -> str:
- return uuid4().hex
-
-
-@pytest.fixture
-def tsid2() -> str:
- return uuid4().hex
-
-
-@pytest.fixture
-def pe_id() -> str:
- # payout event / cashout request UUID
- return uuid4().hex
-
-
# --- Settings ---
@@ -96,233 +28,6 @@ def settings() -> Settings:
return JBSettings()
-# --- Database Connectors ---
-
-
-@pytest.fixture(scope="session")
-def postgres_instance(settings: Settings) -> Generator[PostgresDsn]:
- """Create a ephemeral postgresql instance for us to use during pytest.
-
- This does not create any tables, or schema definitions within the instance.
- What this does is simply:
-
- 1. Create a database on a known, consistent, staging or unittest
- defined Postgres server.
-
- 2. Return the PostgresDsn of that table
-
- 3. On shutdown, go ahead and delete that database after the
- tests have finished.
- """
-
- msg = "Must define Postgres test settings"
- assert settings.testing_postgres, msg
- assert settings.testing_postgres_user, msg
- assert settings.testing_postgres_pass, msg
-
- db_uri, db_user, db_pass = (
- settings.testing_postgres,
- settings.testing_postgres_user,
- settings.testing_postgres_pass,
- )
-
- # Connect to default DB to create the new one
- from psycopg import connect
- from psycopg.sql import SQL, Identifier
-
- now = datetime.now(UTC)
- ts: str = now.strftime("%Y-%m-%d")
- db_name = f"unittest-{ts}-{uuid4().hex[:6]}"
-
- db_path_connect = f"postgres://{db_user}:{db_pass}@{db_uri}"
- db_path = f"{db_path_connect}/{db_name}"
-
- # The DATABASE does NOT yet exist on the Postgres SERVER, thus
- # we first must connect only to the SERVER (eg: default postgres path used)
- conn = connect(f"{db_path_connect}/postgres")
- conn.autocommit = True
- cur = conn.cursor()
- cur.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name)))
- cur.close()
- conn.close()
-
- yield PostgresDsn(db_path)
-
- # Teardown: drop the DB after the session
- conn = connect(f"{db_path_connect}/postgres")
- conn.autocommit = True
- cur = conn.cursor()
- cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name)))
- cur.close()
- conn.close()
-
-
-@pytest.fixture(scope="session")
-def django_settings_file(
- postgres_instance_dict: PostgresDict,
- tmp_path_factory: TempPathFactory,
-) -> Callable[..., tuple[str, Path]]:
-
- def _inner(extra_installed_apps: list[str] | None = None) -> tuple[str, Path]:
- installed_apps = [
- "django.contrib.postgres",
- "django.contrib.contenttypes",
- ] + (extra_installed_apps or [])
-
- settings_dir = tmp_path_factory.mktemp("django-settings")
- settings_module = "test_settings"
-
- settings_content = f"""DATABASES = {{
- "default": {{
- "ENGINE": "django.db.backends.postgresql",
- "NAME": {postgres_instance_dict["name"]!r},
- "USER": {postgres_instance_dict["username"]!r},
- "PASSWORD": {postgres_instance_dict["password"]!r},
- "HOST": {postgres_instance_dict["host"]!r},
- "PORT": {postgres_instance_dict["port"]!r},
- }}
-}}
-INSTALLED_APPS = {installed_apps!r}
-DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
-LANGUAGE_CODE = "en-us"
-TIME_ZONE = "UTC"
-USE_I18N = True
-USE_L10N = True
-USE_TZ = True
-"""
- settings_file_path = settings_dir / f"{settings_module}.py"
- settings_file_path.write_text(settings_content, encoding="utf-8")
-
- return settings_module, settings_dir
-
- return _inner
-
-
-@pytest.fixture(scope="session")
-def postgres_instance_dict(
- postgres_instance: PostgresDsn,
-) -> Generator[PostgresDict]:
- host = postgres_instance.hosts()[0]
- assert host is not None
-
- msg = "Must have full Postgres details"
- assert host["host"], msg
- assert host["username"], msg
- assert host["password"], msg
-
- assert postgres_instance.path
-
- yield PostgresDict(
- username=host["username"],
- password=host["password"],
- host=host["host"],
- name=postgres_instance.path.lstrip("/"),
- port=5432,
- )
-
-
-@pytest.fixture(scope="session")
-def postgres_instance_host(
- postgres_instance_dict: PostgresDict,
-) -> Generator[InternalHostname]:
- adapter = TypeAdapter(InternalHostname)
- value = adapter.validate_python(postgres_instance_dict["host"])
- yield value
-
-
-@pytest.fixture(scope="session")
-def django_db_factory(
- postgres_instance: PostgresDsn,
- django_settings_file: Callable[..., tuple[str, Path]],
- postgres_instance_dict: PostgresDict,
- tmp_path_factory: TempPathFactory,
-) -> Callable[..., PostgresDsn | None]:
-
- _ran = {}
-
- def _inner(
- django_project: str = "carer.mtwerk",
- ) -> PostgresDsn | None:
-
- if _ran.get(django_project, False):
- print(f"Already ran django_db_factory:{django_project}")
- return postgres_instance
- _ran[django_project] = True
-
- _cwd = None
- _manage_path = "generalresearch.thl_django.app.manage"
- _settings_module, _settings_dir = django_settings_file(
- extra_installed_apps=[
- "carer.mtwerk",
- ],
- )
-
- pythonpath = str(_settings_dir)
- if existing_pythonpath := os.environ.get("PYTHONPATH"):
- pythonpath += os.pathsep + existing_pythonpath
-
- env = {
- **os.environ,
- "DJANGO_SETTINGS_MODULE": _settings_module,
- "PYTHONPATH": pythonpath,
- }
-
- # we check right after. if we check now, we won't print if bad
- res1 = subprocess.run( # noqa: PLW1510
- [
- sys.executable,
- "-m",
- _manage_path,
- "makemigrations",
- f"--settings={_settings_module}",
- ],
- cwd=str(_cwd) if _cwd is not None else None,
- env=env,
- capture_output=True,
- text=True,
- )
-
- if res1.returncode != 0:
- print("STDOUT:", res1.stdout)
- print("STDERR:", res1.stderr)
- res1.check_returncode()
-
- res2 = subprocess.run( # noqa: PLW1510
- [
- sys.executable,
- "-m",
- _manage_path,
- "migrate",
- f"--settings={_settings_module}",
- ],
- env=env,
- cwd=str(_cwd) if _cwd is not None else None,
- capture_output=True,
- text=True,
- )
-
- if res2.returncode != 0:
- print("STDOUT:", res2.stdout)
- print("STDERR:", res2.stderr)
- res2.check_returncode()
-
- # 3. Return the Dsn so the factory gives a way to connect
- return postgres_instance
-
- return _inner
-
-
-@pytest.fixture(scope="session")
-def pg_config(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig:
- _dsn = django_db_factory()
-
- return PostgresConfig(
- dsn=_dsn,
- connect_timeout=1,
- statement_timeout=5,
- )
-
-
# --- Redis ---
@@ -360,23 +65,3 @@ def redis_config(settings: Settings, redis_config_db: str) -> Generator[RedisCon
r = redis.from_url(uri)
r.flushdb()
-
-
-# --- Connectors ---
-@pytest.fixture(scope="session")
-def amt_client(settings: Settings) -> MTurkClient:
- import boto3
-
- client = boto3.client(
- service_name="mturk",
- region_name="us-east-1",
- endpoint_url=str(settings.amt_endpoint),
- aws_access_key_id=settings.amt_access_id,
- aws_secret_access_key=settings.amt_secret_key,
- config=CLIENT_CONFIG,
- )
-
- # Confirm we're only using the Sandbox for any unittests
- assert "mturk-requester-sandbox" in client.meta.endpoint_url
-
- return client
diff --git a/tests/fixtures/amt.py b/tests/fixtures/amt.py
deleted file mode 100644
index d7cc03d..0000000
--- a/tests/fixtures/amt.py
+++ /dev/null
@@ -1,264 +0,0 @@
-import copy
-from collections.abc import Callable
-from datetime import datetime, timedelta
-from uuid import uuid4
-
-import pytest
-from dateutil.tz import tzlocal
-from mypy_boto3_mturk.type_defs import (
- CreateHITTypeResponseTypeDef,
- CreateHITWithHITTypeResponseTypeDef,
- GetAssignmentResponseTypeDef,
- GetHITResponseTypeDef,
- ResponseMetadataTypeDef,
-)
-
-from jb.managers.amt import APPROVAL_MESSAGE, NO_WORK_APPROVAL_MESSAGE
-from tests import generate_amt_id
-
-# --- Generic Boto ---
-
-
-@pytest.fixture
-def amt_response_metadata() -> ResponseMetadataTypeDef:
- req_id = str(uuid4())
- return {
- "RequestId": req_id,
- "HTTPStatusCode": 200,
- "HTTPHeaders": {
- "x-amzn-requestid": req_id,
- "content-type": "application/x-amz-json-1.1",
- "content-length": "46",
- "date": "Wed, 15 Oct 2025 02:16:16 GMT",
- },
- "RetryAttempts": 0,
- }
-
-
-# --- Assignment ---
-
-
-@pytest.fixture
-def assignment_response(
- amt_hit_id: str,
- amt_assignment_id: str,
- amt_worker_id: str,
- hit_response: GetHITResponseTypeDef,
- amt_response_metadata: ResponseMetadataTypeDef,
- tsid: str,
-) -> GetAssignmentResponseTypeDef:
-
- hit_response = hit_response["HIT"]
- local_now = datetime.now(tz=tzlocal())
-
- return {
- "Assignment": {
- "AssignmentId": amt_assignment_id,
- "WorkerId": amt_worker_id,
- "HITId": amt_hit_id,
- "AssignmentStatus": "Submitted",
- "AutoApprovalTime": local_now + timedelta(days=7),
- "AcceptTime": local_now - timedelta(minutes=10),
- "SubmitTime": local_now,
- "Deadline": local_now + timedelta(minutes=90),
- "Answer": '<?xml version="1.0" encoding="UTF-8"?>\n'
- '<QuestionFormAnswers xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd">\n '
- "<Answer>\n <QuestionIdentifier>amt_worker_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_worker_id}</FreeText>\n </Answer>\n <Answer>\n "
- " <QuestionIdentifier>amt_assignment_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_assignment_id}</FreeText>\n </Answer>\n <Answer>\n "
- f" <QuestionIdentifier>tsid</QuestionIdentifier>\n <FreeText>{tsid}</FreeText>\n "
- " </Answer>\n</QuestionFormAnswers>",
- "RequesterFeedback": "Good work",
- },
- "HIT": hit_response,
- "ResponseMetadata": amt_response_metadata,
- }
-
-
-@pytest.fixture
-def assignment_response_bad_tsid(
- assignment_response: GetAssignmentResponseTypeDef,
- amt_worker_id: str,
- amt_assignment_id: str,
-) -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response)
- res["Assignment"]["Answer"] = (
- '<?xml version="1.0" encoding="UTF-8"?>\n'
- '<QuestionFormAnswers xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd">\n '
- "<Answer>\n <QuestionIdentifier>amt_worker_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_worker_id}</FreeText>\n </Answer>\n <Answer>\n "
- " <QuestionIdentifier>amt_assignment_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_assignment_id}</FreeText>\n </Answer>\n "
- f" <Answer>\n <QuestionIdentifier>tsid</QuestionIdentifier>\n abc123 <FreeText></FreeText>\n </Answer>\n"
- f"</QuestionFormAnswers>"
- )
- return res
-
-
-@pytest.fixture
-def assignment_response_no_tsid(
- assignment_response: GetAssignmentResponseTypeDef,
- amt_worker_id: str,
- amt_assignment_id: str,
-) -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response)
- res["Assignment"]["Answer"] = (
- '<?xml version="1.0" encoding="UTF-8"?>\n'
- '<QuestionFormAnswers xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd">\n '
- "<Answer>\n <QuestionIdentifier>amt_worker_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_worker_id}</FreeText>\n </Answer>\n <Answer>\n "
- " <QuestionIdentifier>amt_assignment_id</QuestionIdentifier>\n "
- f" <FreeText>{amt_assignment_id}</FreeText>\n </Answer>\n "
- # f"<Answer>\n <QuestionIdentifier>tsid</QuestionIdentifier>\n <FreeText> {tsid}</FreeText>\n </Answer>\n"
- f"</QuestionFormAnswers>"
- )
- return res
-
-
-@pytest.fixture
-def assignment_response_approved_no_tsid(
- assignment_response_no_tsid: GetAssignmentResponseTypeDef,
-) -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response_no_tsid)
- res["Assignment"]["AssignmentStatus"] = "Approved"
- res["Assignment"]["RequesterFeedback"] = NO_WORK_APPROVAL_MESSAGE
- res["Assignment"]["ApprovalTime"] = res["Assignment"]["SubmitTime"]
- return res
-
-
-@pytest.fixture
-def assignment_response_factory_approved(
- assignment_response: GetAssignmentResponseTypeDef,
-) -> Callable[[str], GetAssignmentResponseTypeDef]:
-
- def inner(feedback: str = APPROVAL_MESSAGE) -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response)
- res["Assignment"]["AssignmentStatus"] = "Approved"
- res["Assignment"]["RequesterFeedback"] = feedback
- res["Assignment"]["ApprovalTime"] = res["Assignment"]["SubmitTime"]
- return res
-
- return inner
-
-
-@pytest.fixture
-def assignment_response_factory_rejected(
- assignment_response: GetAssignmentResponseTypeDef,
-) -> Callable[[str], GetAssignmentResponseTypeDef]:
-
- def inner(reject_reason: str = "reject reason") -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response)
- res["Assignment"]["AssignmentStatus"] = "Rejected"
- res["Assignment"]["RequesterFeedback"] = reject_reason
- res["Assignment"]["RejectionTime"] = res["Assignment"]["SubmitTime"]
- return res
-
- return inner
-
-
-@pytest.fixture
-def assignment_response_factory_rejected_no_tsid(
- assignment_response_no_tsid: GetAssignmentResponseTypeDef,
-) -> Callable[[str], GetAssignmentResponseTypeDef]:
-
- def inner(reject_reason: str = "reject reason") -> GetAssignmentResponseTypeDef:
- res = copy.deepcopy(assignment_response_no_tsid)
- res["Assignment"]["AssignmentStatus"] = "Rejected"
- res["Assignment"]["RequesterFeedback"] = reject_reason
- res["Assignment"]["RejectionTime"] = res["Assignment"]["SubmitTime"]
- return res
-
- return inner
-
-
-# --- HITType ---
-
-
-@pytest.fixture
-def create_hit_type_response(
- amt_hit_type_id: str, amt_response_metadata: ResponseMetadataTypeDef
-) -> CreateHITTypeResponseTypeDef:
- return {
- "HITTypeId": amt_hit_type_id,
- "ResponseMetadata": amt_response_metadata,
- }
-
-
-# --- HIT ---
-
-
-@pytest.fixture
-def hit_response_with_hit_type(
- amt_hit_type_id: str,
- amt_hit_id: str,
- amt_response_metadata: ResponseMetadataTypeDef,
-) -> CreateHITWithHITTypeResponseTypeDef:
- amt_group_id = generate_amt_id(length=30)
- return {
- "HIT": {
- "HITId": amt_hit_id,
- "HITTypeId": amt_hit_type_id,
- "HITGroupId": amt_group_id,
- "CreationTime": datetime(2025, 10, 14, 20, 22, tzinfo=tzlocal()),
- "Title": "Test",
- "Description": "test",
- "Question": '<?xml version="1.0" encoding="UTF-8"?>\n<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">\n <ExternalURL>https://jamesbillings67.com/work/</ExternalURL>\n <FrameHeight>1200</FrameHeight>\n </ExternalQuestion>',
- "HITStatus": "Assignable",
- "MaxAssignments": 1,
- "Reward": "0.05",
- "AutoApprovalDelayInSeconds": 2_592_000,
- "Expiration": datetime(2025, 10, 14, 20, 24, 3, tzinfo=tzlocal()),
- "AssignmentDurationInSeconds": 123,
- "QualificationRequirements": [],
- "HITReviewStatus": "NotReviewed",
- "NumberOfAssignmentsPending": 0,
- "NumberOfAssignmentsAvailable": 1,
- "NumberOfAssignmentsCompleted": 0,
- },
- "ResponseMetadata": amt_response_metadata,
- }
-
-
-@pytest.fixture
-def hit_response(
- amt_hit_type_id: str,
- amt_hit_id: str,
- amt_response_metadata: ResponseMetadataTypeDef,
-) -> GetHITResponseTypeDef:
- amt_group_id = generate_amt_id(length=30)
- return {
- "HIT": {
- "HITId": amt_hit_id,
- "HITTypeId": amt_hit_type_id,
- "HITGroupId": amt_group_id,
- "CreationTime": datetime(2025, 10, 13, 23, 0, 3, tzinfo=tzlocal()),
- "Title": "Awesome Surveys!",
- "Description": "Give us your opinion",
- "Question": '<?xml version="1.0" encoding="UTF-8"?>\n<ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd">\n <ExternalURL>https://jamesbillings67.com/work/</ExternalURL>\n <FrameHeight>1200</FrameHeight>\n </ExternalQuestion>',
- "Keywords": "market,research,amazing",
- "HITStatus": "Assignable",
- "MaxAssignments": 1,
- "Reward": "0.05",
- "AutoApprovalDelayInSeconds": 604_800,
- "Expiration": datetime(2025, 10, 27, 23, 0, 3, tzinfo=tzlocal()),
- "AssignmentDurationInSeconds": 5_400,
- "QualificationRequirements": [],
- "HITReviewStatus": "NotReviewed",
- "NumberOfAssignmentsPending": 0,
- "NumberOfAssignmentsAvailable": 1,
- "NumberOfAssignmentsCompleted": 0,
- },
- "ResponseMetadata": amt_response_metadata,
- }
-
-
-@pytest.fixture
-def hit_response_reviewing(
- hit_response: GetHITResponseTypeDef,
-) -> GetHITResponseTypeDef:
- res = copy.deepcopy(hit_response)
- res["HIT"]["NumberOfAssignmentsAvailable"] = 0
- res["HIT"]["NumberOfAssignmentsCompleted"] = 1
- res["HIT"]["HITStatus"] = "Reviewing"
- return res
diff --git a/tests/fixtures/flow.py b/tests/fixtures/flow.py
deleted file mode 100644
index 7bf8b4f..0000000
--- a/tests/fixtures/flow.py
+++ /dev/null
@@ -1,308 +0,0 @@
-from collections.abc import Callable
-from datetime import datetime, timezone
-from typing import Any
-from uuid import uuid4
-
-import pytest
-import requests
-from generalresearch.currency import USDCent
-from generalresearch.models.thl.definitions import PayoutStatus
-from generalresearch.models.thl.payout import UserPayoutEvent
-from generalresearch.models.thl.wallet.cashout_method import (
- CashoutRequestInfo,
- CashoutRequestResponse,
-)
-from generalresearch.models.thl.wallet.definitions import PayoutType
-from mypy_boto3_mturk.type_defs import (
- GetAssignmentResponseTypeDef,
- GetHITResponseTypeDef,
-)
-
-from jb.config import settings
-from jb.managers.amt import (
- APPROVAL_MESSAGE,
- BONUS_MESSAGE,
-)
-
-
-@pytest.fixture
-def approved_assignment_stubs(
- assignment_response: GetAssignmentResponseTypeDef,
- assignment_response_factory_approved: Callable[[str], GetAssignmentResponseTypeDef],
- amt_assignment_id: str,
- amt_hit_id: str,
- hit_response_reviewing: GetHITResponseTypeDef,
-) -> Callable[..., list[dict[str, Any]]]:
-
- # These are the AMT_CLIENT stubs/mocks that need to be set when running
- # process_assignment_submitted() which will result in an approved
- # assignment and sent bonus
- def _inner(
- feedback: str = APPROVAL_MESSAGE,
- override_response: str | None = None,
- override_approve_response: str | None = None,
- ) -> list[dict[str, Any]]:
-
- response = override_response or assignment_response
- approve_response = (
- override_approve_response or assignment_response_factory_approved(feedback)
- )
-
- return [
- {
- "operation": "get_assignment",
- "response": response,
- "expected_params": {"AssignmentId": amt_assignment_id},
- },
- {
- "operation": "approve_assignment",
- "response": {},
- "expected_params": {
- "AssignmentId": amt_assignment_id,
- "RequesterFeedback": feedback,
- "OverrideRejection": False,
- },
- },
- {
- "operation": "get_assignment",
- "response": approve_response,
- "expected_params": {"AssignmentId": amt_assignment_id},
- },
- {
- "operation": "update_hit_review_status",
- "response": {},
- "expected_params": {"HITId": amt_hit_id, "Revert": False},
- },
- {
- "operation": "get_hit",
- "response": hit_response_reviewing,
- "expected_params": {"HITId": amt_hit_id},
- },
- ]
-
- return _inner
-
-
-@pytest.fixture
-def approved_assignment_stubs_w_bonus(
- approved_assignment_stubs: Callable[..., list[dict[str, Any]]],
- amt_worker_id: str,
- amt_assignment_id: str,
- pe_id: str,
-) -> list[dict[str, Any]]:
-
- now = datetime.now(tz=timezone.utc)
- stubs = approved_assignment_stubs().copy()
- stubs.append(
- {
- "operation": "send_bonus",
- "response": {},
- "expected_params": {
- "WorkerId": amt_worker_id,
- "BonusAmount": "0.07",
- "AssignmentId": amt_assignment_id,
- "Reason": BONUS_MESSAGE,
- "UniqueRequestToken": pe_id,
- },
- }
- )
- stubs.append(
- {
- "operation": "list_bonus_payments",
- "response": {
- "BonusPayments": [
- {
- "WorkerId": amt_worker_id,
- "BonusAmount": "0.07",
- "AssignmentId": amt_assignment_id,
- "Reason": BONUS_MESSAGE,
- "GrantTime": now,
- }
- ]
- },
- "expected_params": {"AssignmentId": amt_assignment_id},
- }
- )
- return stubs
-
-
-@pytest.fixture
-def rejected_assignment_stubs(
- assignment_response: GetAssignmentResponseTypeDef,
- assignment_response_factory_rejected: Callable[[str], GetAssignmentResponseTypeDef],
- amt_assignment_id: str,
- amt_hit_id: str,
- hit_response_reviewing: GetHITResponseTypeDef,
-) -> Callable[..., list[dict[str, Any]]]:
-
- # These are the AMT_CLIENT stubs/mocks that need to be set when running
- # process_assignment_submitted() which will result in a rejected
- # assignment
- def _inner(
- reject_reason: str,
- override_response: str | None = None,
- override_reject_response: str | None = None,
- ) -> list[dict[str, Any]]:
-
- response = override_response or assignment_response
- reject_response = (
- override_reject_response
- or assignment_response_factory_rejected(reject_reason)
- )
-
- return [
- {
- "operation": "get_assignment",
- "response": response,
- "expected_params": {"AssignmentId": amt_assignment_id},
- },
- {
- "operation": "reject_assignment",
- "response": {},
- "expected_params": {
- "AssignmentId": amt_assignment_id,
- "RequesterFeedback": reject_reason,
- },
- },
- {
- "operation": "get_assignment",
- "response": reject_response,
- "expected_params": {"AssignmentId": amt_assignment_id},
- },
- {
- "operation": "update_hit_review_status",
- "response": {},
- "expected_params": {"HITId": amt_hit_id, "Revert": False},
- },
- {
- "operation": "get_hit",
- "response": hit_response_reviewing,
- "expected_params": {"HITId": amt_hit_id},
- },
- ]
-
- return _inner
-
-
-@pytest.fixture
-def mock_thl_responses(
- monkeypatch: pytest.MonkeyPatch, amt_worker_id: str, tsid: str, pe_id: str
-) -> Callable[..., None]:
-
- original_get = requests.get
- original_post = requests.post
-
- class MockThlCashoutRequestResponse:
- def json(self):
- return CashoutRequestResponse(
- status="success",
- cashout=CashoutRequestInfo(
- id=pe_id,
- description="amt something",
- status=PayoutStatus.PENDING,
- ),
- ).model_dump(mode="json")
-
- def _inner(
- user_blocked: bool = False,
- status_finished: bool = True,
- status_complete: bool = False,
- wallet_redeemable_amount: int = 10,
- ):
-
- def mock_get(url, *args, **kwargs): # type: ignore
- profile_url = f"{settings.fsb_host}{settings.product_id}/user/{amt_worker_id}/profile/"
- status_url = f"{settings.fsb_host}{settings.product_id}/status/{tsid}/"
- cashout_request_url = f"{settings.fsb_host}{settings.product_id}/cashout/"
- wallet_url = f"{settings.fsb_host}{settings.product_id}/wallet/"
- if url == profile_url:
-
- class MockThlProfileResponse:
- def json(self):
- return {
- "user_profile": {
- "user": {
- "product_id": settings.product_id,
- "product_user_id": amt_worker_id,
- "blocked": user_blocked,
- }
- }
- }
-
- return MockThlProfileResponse()
-
- elif url == wallet_url:
-
- class MockThlWalletResponse:
- def json(self) -> dict[str, Any]:
- return {
- "wallet": {
- "amount": wallet_redeemable_amount,
- "redeemable_amount": wallet_redeemable_amount,
- }
- }
-
- return MockThlWalletResponse()
-
- elif url == status_url:
-
- class MockThlStatusResponse:
- def json(self) -> dict[str, Any]:
- return {
- "tsid": tsid,
- "product_id": str(settings.product_id),
- "product_user_id": amt_worker_id,
- "started": "2020-06-02T00:30:35.036398Z",
- "finished": (
- "2020-06-02T00:31:35.036398Z"
- if status_finished
- else None
- ),
- "status": 3 if status_complete else 2,
- "payout": 10 if status_complete else 0,
- "user_payout": 10 if status_complete else 0,
- "status_code_1": "BUYER_FAIL",
- "status_code_2": None,
- }
-
- return MockThlStatusResponse()
-
- elif url == cashout_request_url:
- return MockThlCashoutRequestResponse()
-
- else:
- raise ValueError(f"unhandled call: {url=} {args=} {kwargs=}")
-
- return original_get(url, *args, **kwargs)
-
- def mock_post(url, *args, **kwargs): # type: ignore
- cashout_request_url = f"{settings.fsb_host}{settings.product_id}/cashout/"
- manage_cashout_request_url = f"{settings.fsb_host}{settings.fsb_host_private_route}/thl/manage_cashout/"
-
- if url == cashout_request_url:
- return MockThlCashoutRequestResponse()
-
- elif url == manage_cashout_request_url:
- json = kwargs["json"]
- print(json)
- payout_id = json["payout_id"]
- new_status = json["new_status"]
-
- class MockThlManageCashoutResponse:
- def json(self):
- return UserPayoutEvent(
- uuid=payout_id,
- status=new_status,
- amount=USDCent(5),
- debit_account_uuid=uuid4().hex,
- cashout_method_uuid=uuid4().hex,
- payout_type=PayoutType.AMT,
- ).model_dump(mode="json")
-
- return MockThlManageCashoutResponse()
- return original_post(url, *args, **kwargs)
-
- monkeypatch.setattr(requests, "get", mock_get)
- monkeypatch.setattr(requests, "post", mock_post)
-
- return _inner
diff --git a/tests/fixtures/http.py b/tests/fixtures/http.py
index e38c853..0f88fb8 100644
--- a/tests/fixtures/http.py
+++ b/tests/fixtures/http.py
@@ -1,7 +1,6 @@
-import json
import secrets
from collections.abc import AsyncGenerator
-from typing import Any
+from uuid import uuid4
import httpx
import pytest
@@ -10,12 +9,8 @@ from asgi_lifespan import LifespanManager
from generalresearch.redis_helper import RedisConfig
from httpx import ASGITransport, AsyncClient
-from jb.config import JB_EVENTS_STREAM, settings
from jb.decorators import get_redis_config
from jb.main import app
-from jb.models.assignment import AssignmentStub
-from jb.models.hit import Hit
-from tests import generate_amt_id
@pytest.fixture(scope="session")
@@ -64,43 +59,16 @@ def mock_requests():
yield m
-def generate_hex_id(length: int = 40) -> str:
- # length is number of hex chars, so we need length//2 bytes
- return secrets.token_hex(length // 2)
+@pytest.fixture
+def email() -> str:
+ return "unittest@generalresearch.com"
@pytest.fixture
-def mturk_event_body_record(
- hit_record: Hit, assignment_stub_record: AssignmentStub
-) -> dict[str, Any]:
- return {
- "Type": "Notification",
- "Message": json.dumps(
- {
- "Events": [
- {
- "EventType": "AssignmentSubmitted",
- "EventTimestamp": "2025-10-16T18:45:51.000000Z",
- "HITId": hit_record.amt_hit_id,
- "AssignmentId": assignment_stub_record.amt_assignment_id,
- "HITTypeId": hit_record.amt_hit_type_id,
- }
- ],
- "EventDocId": generate_hex_id(),
- "SourceAccount": settings.aws_owner_id,
- "CustomerId": generate_amt_id(length=14),
- "EventDocVersion": "2006-05-05",
- }
- ),
- }
+def amt_worker_id() -> str:
+ return uuid4().hex
-@pytest.fixture()
-def clean_mturk_events_redis_stream(redis_config: RedisConfig):
- redis_client = redis_config.create_redis_client()
-
- redis_client.xtrim(JB_EVENTS_STREAM, maxlen=0)
- assert redis_client.xlen(JB_EVENTS_STREAM) == 0
- yield
- redis_client.xtrim(JB_EVENTS_STREAM, maxlen=0)
- assert redis_client.xlen(JB_EVENTS_STREAM) == 0
+def generate_hex_id(length: int = 40) -> str:
+ # length is number of hex chars, so we need length//2 bytes
+ return secrets.token_hex(length // 2)
diff --git a/tests/fixtures/managers.py b/tests/fixtures/managers.py
deleted file mode 100644
index 8f87e5d..0000000
--- a/tests/fixtures/managers.py
+++ /dev/null
@@ -1,88 +0,0 @@
-from typing import TYPE_CHECKING
-
-import pytest
-from generalresearch.managers.base import Permission
-from generalresearch.pg_helper import PostgresConfig
-from mypy_boto3_mturk import MTurkClient
-
-if TYPE_CHECKING:
- from jb.managers.amt import AMTManager
- from jb.managers.assignment import AssignmentManager
- from jb.managers.bonus import BonusManager
- from jb.managers.hit import HitManager, HitQuestionManager, HitTypeManager
-
-
-# --- Managers ---
-@pytest.fixture(scope="session")
-def amtm(amt_client: MTurkClient) -> "AMTManager":
- from jb.managers.amt import AMTManager
-
- amtm = AMTManager(amt_client=amt_client)
- assert "mturk-requester-sandbox" in amtm.amt_client.meta.endpoint_url
-
- return amtm
-
-
-@pytest.fixture(scope="session")
-def hqm(pg_config: PostgresConfig) -> "HitQuestionManager":
- assert (
- pg_config.dsn.path and "/unittest-" in pg_config.dsn.path
- ), "pg_config must point to a unittest database (dsn path must contain '/unittest-')"
-
- from jb.managers.hit import HitQuestionManager
-
- return HitQuestionManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
- )
-
-
-@pytest.fixture(scope="session")
-def htm(pg_config: PostgresConfig) -> "HitTypeManager":
- assert (
- pg_config.dsn.path and "/unittest-" in pg_config.dsn.path
- ), "pg_config must point to a unittest database (dsn path must contain '/unittest-')"
-
- from jb.managers.hit import HitTypeManager
-
- return HitTypeManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
- )
-
-
-@pytest.fixture(scope="session")
-def hm(pg_config: PostgresConfig) -> "HitManager":
- assert (
- pg_config.dsn.path and "/unittest-" in pg_config.dsn.path
- ), "pg_config must point to a unittest database (dsn path must contain '/unittest-')"
-
- from jb.managers.hit import HitManager
-
- return HitManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
- )
-
-
-@pytest.fixture(scope="session")
-def am(pg_config: PostgresConfig) -> "AssignmentManager":
- assert (
- pg_config.dsn.path and "/unittest-" in pg_config.dsn.path
- ), "pg_config must point to a unittest database (dsn path must contain '/unittest-')"
-
- from jb.managers.assignment import AssignmentManager
-
- return AssignmentManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
- )
-
-
-@pytest.fixture(scope="session")
-def bm(pg_config: PostgresConfig) -> "BonusManager":
- assert (
- pg_config.dsn.path and "/unittest-" in pg_config.dsn.path
- ), "pg_config must point to a unittest database (dsn path must contain '/unittest-')"
-
- from jb.managers.bonus import BonusManager
-
- return BonusManager(
- pg_config=pg_config, permissions=[Permission.READ, Permission.CREATE]
- )
diff --git a/tests/fixtures/models.py b/tests/fixtures/models.py
deleted file mode 100644
index 157daec..0000000
--- a/tests/fixtures/models.py
+++ /dev/null
@@ -1,296 +0,0 @@
-from collections.abc import Callable, Generator
-from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
-
-import pytest
-from generalresearch.currency import USDCent
-from generalresearch.pg_helper import PostgresConfig
-from psycopg.errors import ForeignKeyViolation
-
-from jb.managers.amt import AMTManager
-from jb.models.assignment import Assignment, AssignmentStub
-from jb.models.definitions import AssignmentStatus, HitReviewStatus, HitStatus
-from jb.models.event import MTurkEvent
-from jb.models.hit import Hit, HitQuestion, HitType
-from tests import generate_amt_id
-
-if TYPE_CHECKING:
- from jb.managers.assignment import AssignmentManager
- from jb.managers.hit import HitManager, HitQuestionManager, HitTypeManager
-
-
-# --- MTurk Event ---
-
-
-@pytest.fixture
-def mturk_event(
- amt_assignment_id: str, amt_hit_id: str, amt_hit_type_id: str
-) -> MTurkEvent:
- now = datetime.now(tz=timezone.utc)
- return MTurkEvent(
- event_type="AssignmentSubmitted",
- event_timestamp=now,
- amt_assignment_id=amt_assignment_id,
- amt_hit_type_id=amt_hit_type_id,
- amt_hit_id=amt_hit_id,
- )
-
-
-# --- Question ---
-
-
-@pytest.fixture
-def question() -> HitQuestion:
- return HitQuestion(url="https://jamesbillings67.com/work/", height=1200)
-
-
-@pytest.fixture
-def question_record(hqm: "HitQuestionManager", question: HitQuestion) -> HitQuestion:
- return hqm.get_or_create(question)
-
-
-# --- HITType ---
-
-
-@pytest.fixture
-def hit_type() -> HitType:
- return HitType(
- title="Awesome Surveys!",
- description="Give us your opinion",
- reward=USDCent(5),
- keywords="market,research,amazing",
- min_active=10,
- )
-
-
-@pytest.fixture
-def hit_type_record(
- pg_config: PostgresConfig, htm: "HitTypeManager", hit_type: HitType
-) -> Generator[HitType, None, None]:
-
- hit_type.amt_hit_type_id = generate_amt_id()
-
- ht = htm.get_or_create(hit_type)
-
- yield ht
-
- try:
- with pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute("DELETE FROM mtwerk_hittype WHERE id=%s", (ht.id,))
- conn.commit()
-
- except ForeignKeyViolation:
- pass # DB gets dropped anyway, don't care
-
-
-@pytest.fixture
-def hit_type_record_with_amt_id(
- pg_config: PostgresConfig, htm: "HitTypeManager", hit_type: HitType
-) -> Generator[HitType, None, None]:
- # This is a real hit type I've previously registered with amt (sandbox).
- # It will always exist
- hit_type.amt_hit_type_id = "3217B3DC4P5YW9DRV9R3X8O56V041J"
-
- # Get or create our db
- ht = htm.get_or_create(hit_type)
-
- yield ht
-
- try:
- with pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute("DELETE FROM mtwerk_hittype WHERE id=%s", (ht.id,))
- conn.commit()
-
- except ForeignKeyViolation:
- pass # DB gets dropped anyway, don't care
-
-
-# --- HIT ---
-
-
-@pytest.fixture
-def hit(
- amt_hit_id: str, amt_hit_type_id: str, amt_group_id: str, question: HitQuestion
-) -> Hit:
- now = datetime.now(tz=timezone.utc)
-
- return Hit.model_validate(
- dict(
- amt_hit_id=amt_hit_id,
- amt_hit_type_id=amt_hit_type_id,
- amt_group_id=amt_group_id,
- status=HitStatus.Assignable,
- review_status=HitReviewStatus.NotReviewed,
- creation_time=now,
- expiration=now + timedelta(days=3),
- hit_question_xml=question.xml,
- qualification_requirements=[],
- max_assignments=1,
- assignment_pending_count=0,
- assignment_available_count=1,
- assignment_completed_count=0,
- description="Description",
- keywords="Keywords",
- reward=USDCent(5),
- title="Title",
- question_id=question.id,
- hit_type_id=None,
- )
- )
-
-
-@pytest.fixture
-def hit_record(
- pg_config: PostgresConfig,
- hm: "HitManager",
- question_record: HitQuestion,
- hit_type_record: HitType,
- hit: Hit,
- amt_hit_id: str,
-) -> Generator[Hit, None, None]:
- """
- Returns a hit that exists in our db, but does not in amazon (the amt ids
- are random). The mtwerk_hittype and mtwerk_question records will also
- exist (in the db)
- """
-
- hit.hit_type_id = hit_type_record.id
- hit.amt_hit_id = amt_hit_id
- hit.question_id = question_record.id
-
- hm.create(hit)
-
- yield hit
-
- try:
- with pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute("DELETE FROM mtwerk_hit WHERE id=%s", (hit.id,))
- conn.commit()
-
- except ForeignKeyViolation:
- pass # DB gets dropped anyway, don't care
-
-
-@pytest.fixture
-def hit_in_amt(
- amtm: AMTManager,
- hm: "HitManager",
- question_record: HitQuestion,
- hit_type_record_with_amt_id: HitType,
-) -> Hit:
- # Actually create a new HIT in amt (sandbox)
- hit = amtm.create_hit_with_hit_type(
- hit_type=hit_type_record_with_amt_id, question=question_record
- )
-
- # Create it in the DB
- hm.create(hit)
- return hit
-
-
-# --- Assignment ---
-
-
-@pytest.fixture
-def assignment_stub(
- hit: Hit, amt_assignment_id: str, amt_worker_id: str
-) -> AssignmentStub:
- now = datetime.now(tz=timezone.utc)
- return AssignmentStub(
- amt_assignment_id=amt_assignment_id,
- amt_hit_id=hit.amt_hit_id,
- amt_worker_id=amt_worker_id,
- status=AssignmentStatus.Submitted,
- modified_at=now,
- created_at=now,
- )
-
-
-@pytest.fixture
-def assignment_stub_record(
- pg_config: PostgresConfig,
- am: "AssignmentManager",
- hit_record: Hit,
- assignment_stub: AssignmentStub,
-) -> Generator[AssignmentStub, None, None]:
- """
- Returns an AssignmentStub that exists in our db, but does not in
- amazon (the amt ids are random). The mtwerk_hit, mtwerk_hittype, and
- mtwerk_question records will also exist (in the db)
- """
- assignment_stub.hit_id = hit_record.id
- am.create_stub(stub=assignment_stub)
-
- yield assignment_stub
-
- try:
- with pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute(
- "DELETE FROM mtwerk_assignment WHERE id=%s", (assignment_stub.id,)
- )
- conn.commit()
-
- except ForeignKeyViolation:
- pass # DB gets dropped anyway, don't care
-
-
-@pytest.fixture
-def assignment(assignment_factory: Callable[..., Assignment]) -> Assignment:
- return assignment_factory()
-
-
-@pytest.fixture
-def assignment_record(
- pg_config: PostgresConfig,
- hit_record: Hit,
- assignment_record_factory: Callable[..., Assignment],
-) -> Generator[Assignment, None, None]:
- assignment = assignment_record_factory(hit_id=hit_record.id)
-
- yield assignment
-
- try:
- with pg_config.make_connection() as conn, conn.cursor() as c:
- c.execute("DELETE FROM mtwerk_assignment WHERE id=%s", (assignment.id,))
- conn.commit()
-
- except ForeignKeyViolation:
- pass # DB gets dropped anyway, don't care
-
-
-@pytest.fixture
-def assignment_factory(hit_record: Hit) -> Callable[[str | None], Assignment]:
-
- def _inner(amt_worker_id: str | None = None) -> Assignment:
- now = datetime.now(tz=timezone.utc)
- amt_assignment_id = generate_amt_id()
- amt_worker_id = amt_worker_id or generate_amt_id()
-
- return Assignment(
- amt_assignment_id=amt_assignment_id,
- amt_hit_id=hit_record.amt_hit_id,
- amt_worker_id=amt_worker_id,
- status=AssignmentStatus.Submitted,
- modified_at=now,
- created_at=now,
- accept_time=now,
- auto_approval_time=now,
- submit_time=now,
- )
-
- return _inner
-
-
-@pytest.fixture
-def assignment_record_factory(
- am: "AssignmentManager", assignment_factory: Callable[..., Assignment]
-) -> Callable[..., Assignment]:
-
- def _inner(hit_id: int, amt_worker_id: str | None = None) -> Assignment:
- a = assignment_factory(amt_worker_id=amt_worker_id)
- a.hit_id = hit_id
- am.create_stub(a)
- am.update_answer(a)
- return a
-
- return _inner
diff --git a/tests/flow/__init__.py b/tests/flow/__init__.py
deleted file mode 100644
index 3c88c7e..0000000
--- a/tests/flow/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# def test_a_hit():
-# hit_type = HitType(
-# title="Sandbox Test 2026-02-04",
-# description="test description",
-# reward=USDCent(5),
-# keywords="a,b,c",
-# min_active=10,
-# )
-# AMTManager.create_hit_type(hit_type=hit_type)
-# HTM.create(hit_type)
-
-# hit_type = HTM.get(amt_hit_type_id=hit_type.amt_hit_type_id)
-# hit = create_hit_from_hittype(hit_type)
-
-# hit_type = HTM.get(amt_hit_type_id="3QMCR5SRRVWV0O3UDSC64VSB8KL4GL")
-# hit = create_hit_from_hittype(hit_type)
diff --git a/tests/flow/test_tasks.py b/tests/flow/test_tasks.py
deleted file mode 100644
index f28a6aa..0000000
--- a/tests/flow/test_tasks.py
+++ /dev/null
@@ -1,352 +0,0 @@
-import logging
-from collections.abc import Callable
-from contextlib import contextmanager
-from typing import Any
-
-import pytest
-from botocore.stub import Stubber
-from generalresearch.currency import USDCent
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_mturk.type_defs import (
- GetAssignmentResponseTypeDef,
-)
-
-from jb.flow.assignment_tasks import process_assignment_submitted
-from jb.managers.amt import (
- APPROVAL_MESSAGE,
- NO_WORK_APPROVAL_MESSAGE,
- REJECT_MESSAGE_BADDIE,
- REJECT_MESSAGE_NO_WORK,
- REJECT_MESSAGE_UNKNOWN_ASSIGNMENT,
- AMTManager,
-)
-from jb.managers.assignment import AssignmentManager
-from jb.managers.bonus import BonusManager
-from jb.managers.hit import HitManager
-from jb.models.assignment import Assignment, AssignmentStub
-from jb.models.definitions import AssignmentStatus
-from jb.models.event import MTurkEvent
-from jb.models.hit import Hit
-
-
-@contextmanager
-def amt_stub_context(amt_client: MTurkClient, responses: list[dict[str, Any]]):
-
- # ty chatgpt for this
- with Stubber(amt_client) as stub:
- for r in responses:
- stub.add_response(
- r["operation"],
- r.get("response", {}),
- r.get("expected_params", {}),
- )
- yield stub
-
-
-class TestHITTasks:
-
- def test_fake_get_assignment(
- self,
- amt_client: MTurkClient,
- amtm: AMTManager,
- amt_assignment_id: str,
- amt_worker_id: str,
- assignment_response: GetAssignmentResponseTypeDef,
- ):
- # Testing just that this boto stubber works (we fake a response
- # using the real boto client)
- fake_response = assignment_response.copy()
-
- with Stubber(amt_client) as stub:
- expected_params = {"AssignmentId": amt_assignment_id}
- stub.add_response("get_assignment", fake_response, expected_params)
-
- assignment = amtm.get_assignment_if_exists(
- amt_assignment_id=amt_assignment_id
- )
- assert assignment is not None
- assert assignment.amt_assignment_id == amt_assignment_id
- assert assignment.amt_worker_id == amt_worker_id
-
- # Optionally, ensure all queued responses were used:
- stub.assert_no_pending_responses()
-
-
-class TestProcessAssignmentSubmitted:
-
- def test_no_assignment_in_db(
- self,
- amtm: AMTManager,
- hm: HitManager,
- am: AssignmentManager,
- bm: BonusManager,
- amt_client: MTurkClient,
- hit_record: Hit,
- mturk_event: MTurkEvent,
- amt_assignment_id: str,
- caplog: pytest.LogCaptureFixture,
- rejected_assignment_stubs: Callable[..., list[dict[str, Any]]],
- ):
-
- # These records are auto cleaned up, so we need to explicitly create
- # a HIT record in the DB so the process_assignment_submitted task
- # doesn't error when we try to process the Request
- _ = hit_record
-
- # An assignment is submitted. The hit exists in the DB. The amt
- # assignment id is valid, but the assignment stub is not in our
- # db. Reject it and write the assignment to the db.
-
- amt_stubs = rejected_assignment_stubs(
- reject_reason=REJECT_MESSAGE_UNKNOWN_ASSIGNMENT
- )
-
- with amt_stub_context(amt_client, amt_stubs) as stub, caplog.at_level(
- logging.WARNING
- ):
- process_assignment_submitted(
- amtm=amtm, hm=hm, am=am, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- # assert f"No assignment found in DB: {amt_assignment_id}" in caplog.text
- assert "Rejected assignment doesn't exist in DB. Creating ... " in caplog.text
- assert "Rejected assignment: " in caplog.text
- stub.assert_no_pending_responses()
-
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Rejected
- assert ass.requester_feedback == REJECT_MESSAGE_UNKNOWN_ASSIGNMENT
-
- def test_assignment_in_db_user_doesnt_exist(
- self,
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- amt_client: MTurkClient,
- mturk_event: MTurkEvent,
- amt_assignment_id: str,
- assignment_stub_record: AssignmentStub,
- caplog: pytest.LogCaptureFixture,
- mock_thl_responses: Callable[..., None],
- rejected_assignment_stubs: Callable[..., list[dict[str, Any]]],
- ):
- # An assignment is submitted. The hit and AssignmentStub exist in the
- # DB. We think we're going to approve the Assignment, but the
- # user-profile / check blocked call on THL shows the user doesn't
- # exist (same thing would happen if the user does exist and is
- # blocked). So we reject.
-
- # We need this to make the assignment stub in the db
- _ = assignment_stub_record
-
- amt_stubs = rejected_assignment_stubs(reject_reason=REJECT_MESSAGE_BADDIE)
- mock_thl_responses(user_blocked=True)
-
- with amt_stub_context(amt_client, amt_stubs) as stub, caplog.at_level(
- logging.WARNING
- ):
- process_assignment_submitted(
- amtm=amtm, am=am, hm=hm, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- # assert f"No assignment found in DB: {amt_assignment_id}" not in caplog.text
- assert "blocked or not exists" in caplog.text
- assert "Rejected assignment: " in caplog.text
-
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Rejected
- assert ass.requester_feedback == REJECT_MESSAGE_BADDIE
-
- def test_no_work_w_warning(
- self,
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- mturk_event: MTurkEvent,
- amt_client: MTurkClient,
- amt_assignment_id: str,
- assignment_stub_record: AssignmentStub,
- caplog: pytest.LogCaptureFixture,
- mock_thl_responses: Callable[..., None],
- approved_assignment_stubs: Callable[..., list[dict[str, Any]]],
- assignment_response_approved_no_tsid: GetAssignmentResponseTypeDef,
- assignment_response_no_tsid: GetAssignmentResponseTypeDef,
- ):
- # An Assignment is submitted. The hit and AssignmentStub exist in
- # the DB. The assignment has no tsid.
- # We APPROVE this assignment b/c we are very nice and give users a
- # couple chances, with an explanation, before rejecting.
-
- # We need this to make the assignment stub in the db
- _ = assignment_stub_record
-
- # Simulate that the AMT.get_assignment call returns the assignment,
- # but the answers XML has no tsid.
- amt_stubs = approved_assignment_stubs(
- feedback=NO_WORK_APPROVAL_MESSAGE,
- override_response=assignment_response_no_tsid,
- override_approve_response=assignment_response_approved_no_tsid,
- )
-
- mock_thl_responses(user_blocked=False)
-
- with amt_stub_context(amt_client, amt_stubs) as stub, caplog.at_level(
- logging.WARNING
- ):
- process_assignment_submitted(
- amtm=amtm, hm=hm, am=am, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- assert f"No assignment found in DB: {amt_assignment_id}" not in caplog.text
- assert "Assignment submitted with no tsid" in caplog.text
- assert "Approved assignment: " in caplog.text
-
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Approved
- assert ass.requester_feedback == NO_WORK_APPROVAL_MESSAGE
- assert am.missing_tsid_count(amt_worker_id=ass.amt_worker_id) == 1
-
- def test_no_work_no_warning(
- self,
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- mturk_event: MTurkEvent,
- amt_client: MTurkClient,
- amt_assignment_id: str,
- assignment_stub_record: AssignmentStub,
- caplog: pytest.LogCaptureFixture,
- mock_thl_responses: Callable[..., None],
- rejected_assignment_stubs: Callable[..., list[dict[str, Any]]],
- assignment_response_factory_rejected_no_tsid: Callable[
- ..., GetAssignmentResponseTypeDef
- ],
- assignment_response_no_tsid: GetAssignmentResponseTypeDef,
- assignment_record_factory: Callable[..., Assignment],
- hit_record: Hit, # used in assignment_factory
- amt_worker_id: str,
- ):
- # An assignment is submitted. The hit and assignment stub exist in the DB.
- # The assignment has no tsid.
-
- # Going to create and submit 3 assignments w no work
- # (all on the same hit, which we don't do in JB for real,
- # but doesn't matter here)
- _a1 = assignment_record_factory(
- hit_id=hit_record.id, amt_worker_id=amt_worker_id
- )
- _a2 = assignment_record_factory(
- hit_id=hit_record.id, amt_worker_id=amt_worker_id
- )
- _a3 = assignment_record_factory(
- hit_id=hit_record.id, amt_worker_id=amt_worker_id
- )
-
- assert am.missing_tsid_count(amt_worker_id=amt_worker_id) == 3
- # So now, we'll reject, b/c they've already gotten 3 warnings
-
- _ = assignment_stub_record # we need this to make the assignment stub in the db
-
- # Simulate that the AMT.get_assignment call returns the assignment, but the answers xml
- # has no tsid.
- amt_stubs = rejected_assignment_stubs(
- reject_reason=REJECT_MESSAGE_NO_WORK,
- override_response=assignment_response_no_tsid,
- override_reject_response=assignment_response_factory_rejected_no_tsid(
- REJECT_MESSAGE_NO_WORK
- ),
- )
-
- mock_thl_responses(user_blocked=False)
-
- with amt_stub_context(amt_client, amt_stubs) as stub, caplog.at_level(
- logging.WARNING
- ):
- process_assignment_submitted(
- amtm=amtm, hm=hm, am=am, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- assert f"No assignment found in DB: {amt_assignment_id}" not in caplog.text
- assert "Assignment submitted with no tsid" in caplog.text
- assert "Rejected assignment: " in caplog.text
-
- # It will exist in the db since we can validate the model.
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Rejected
- assert ass.requester_feedback == REJECT_MESSAGE_NO_WORK
-
- def test_assignment_submitted_no_bonus(
- self,
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- amt_client: MTurkClient,
- mturk_event: MTurkEvent,
- amt_assignment_id: str,
- assignment_stub_record: Assignment,
- caplog: pytest.LogCaptureFixture,
- mock_thl_responses: Callable[..., None],
- approved_assignment_stubs: Callable[..., list[dict[str, Any]]],
- ):
-
- _ = assignment_stub_record # we need this to make the assignment stub in the db
-
- # The "send bonus" stuff will still run, even if the user didn't get
- # a complete, because all we do is check the user's wallet balance (if
- # an assignment is approved) and they may have money in their wallet
- # from a prev event or bribe
- #
- # So mock the wallet balance as 1cent, so no bonus will be triggered
-
- mock_thl_responses(status_complete=False, wallet_redeemable_amount=1)
- with amt_stub_context(
- amt_client, approved_assignment_stubs()
- ) as stub, caplog.at_level(logging.WARNING):
- process_assignment_submitted(
- amtm=amtm, hm=hm, am=am, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Approved
- assert ass.requester_feedback == APPROVAL_MESSAGE
-
- def test_assignment_submitted_w_bonus(
- self,
- amtm: AMTManager,
- am: AssignmentManager,
- hm: HitManager,
- bm: BonusManager,
- amt_client: MTurkClient,
- mturk_event: MTurkEvent,
- amt_assignment_id: str,
- assignment_stub_record: Assignment,
- caplog: pytest.LogCaptureFixture,
- mock_thl_responses: Callable[..., None],
- approved_assignment_stubs_w_bonus: list[dict[str, Any]],
- ):
- _ = assignment_stub_record # we need this to make the assignment stub in the db
- mock_thl_responses(status_complete=True, wallet_redeemable_amount=10)
-
- with amt_stub_context(
- amt_client, approved_assignment_stubs_w_bonus
- ) as stub, caplog.at_level(logging.WARNING):
- process_assignment_submitted(
- amtm=amtm, hm=hm, am=am, bm=bm, event=mturk_event
- )
- stub.assert_no_pending_responses()
-
- ass = am.get(amt_assignment_id=amt_assignment_id)
- assert ass.status == AssignmentStatus.Approved
- assert ass.requester_feedback == APPROVAL_MESSAGE
-
- bonus = bm.filter(amt_assignment_id=amt_assignment_id)[0]
- assert bonus.amount == USDCent(7)
diff --git a/tests/http/test_auth.py b/tests/http/test_auth.py
index b44a36a..40f0366 100644
--- a/tests/http/test_auth.py
+++ b/tests/http/test_auth.py
@@ -1,4 +1,5 @@
from urllib.parse import parse_qs, urlparse
+from uuid import uuid4
import pytest
from httpx import AsyncClient
@@ -35,9 +36,6 @@ class FakeGRApiManager:
return user
-@pytest.fixture
-def email() -> str:
- return "unittest@generalresearch.com"
@pytest.fixture
@@ -58,20 +56,20 @@ class TestAuth:
):
client = httpxclient
- res = await client.post("/auth/magic-link/request", json={"email": email})
+ res = await client.post("/auth/magic-link/request/", json={"email": email})
d = res.json()
assert res.status_code == 200
assert d["magic_link"]
token = parse_qs(urlparse(d["magic_link"]).query)["token"][0]
- url = "/auth/magic-link/exchange"
+ url = "/auth/magic-link/exchange/"
body = {"token": token}
res = await client.post(url, json=body)
assert res.status_code == 204
assert client.cookies.get(SESSION_COOKIE_NAME)
- res = await client.get("/auth/session")
+ res = await client.get("/auth/session/")
assert res.status_code == 200
assert res.json()["email"] == email
@@ -87,7 +85,7 @@ class TestAuth:
fake_gr_api_manager.add_amt_user(amt_worker_id)
res = await client.post(
- "/auth/link-amt/request",
+ "/auth/link-amt/request/",
json={"email": email, "amt_worker_id": amt_worker_id},
)
d = res.json()
@@ -110,7 +108,7 @@ class TestAuth:
assert transitioned_amt_worker_id == amt_worker_id
assert amt_worker_id not in fake_gr_api_manager.amt_worker_ids
- res = await client.get("/auth/session")
+ res = await client.get("/auth/session/")
assert res.status_code == 200
assert res.json()["email"] == email
diff --git a/tests/http/test_magic.py b/tests/http/test_magic.py
index f645677..278a194 100644
--- a/tests/http/test_magic.py
+++ b/tests/http/test_magic.py
@@ -1,5 +1,6 @@
from uuid import uuid4
+import pytest
from generalresearch.redis_helper import RedisConfig
from jb.api.magic_token import (
@@ -11,8 +12,12 @@ from jb.api.magic_token import (
from jb.models.auth import AmtAccountLink
-class TestViewFunctions:
+@pytest.fixture
+def amt_worker_id() -> str:
+ return uuid4().hex
+
+class TestViewFunctions:
def test_create_amt_account_link_token(
self, redis_config: RedisConfig, amt_worker_id: str
):
diff --git a/tests/http/test_notifications.py b/tests/http/test_notifications.py
deleted file mode 100644
index 3df2423..0000000
--- a/tests/http/test_notifications.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import json
-from typing import Any
-from uuid import uuid4
-
-import pytest
-from generalresearch.redis_helper import RedisConfig
-from httpx import AsyncClient
-
-from jb.config import JB_EVENTS_STREAM, settings
-from jb.models.assignment import AssignmentStub
-from jb.models.event import MTurkEvent
-from jb.models.hit import Hit
-
-
-class TestNotifications:
-
- @pytest.mark.anyio
- async def test_no_post_keys(
- self,
- httpxclient: AsyncClient,
- ):
- client = httpxclient
-
- res = await client.post(url=f"/{settings.sns_path}/", json={})
- assert res.status_code == 500
- assert res.json() == {"detail": "Invalid JSON"}
-
- @pytest.mark.anyio
- async def test_no_post_data(
- self,
- httpxclient: AsyncClient,
- ):
- client = httpxclient
-
- res = await client.post(url=f"/{settings.sns_path}/")
- assert res.status_code == 500
- assert res.json() == {"detail": "No POST data"}
-
- @pytest.mark.anyio
- async def test_invalid_post_data(
- self,
- httpxclient: AsyncClient,
- ):
- client = httpxclient
-
- res = await client.post(
- url=f"/{settings.sns_path}/", json={uuid4().hex: uuid4().hex}
- )
- assert res.status_code == 500
- assert res.json() == {"detail": "Invalid JSON"}
-
- @pytest.mark.anyio
- async def test_mturk_notifications(
- self,
- redis_config: RedisConfig,
- httpxclient: AsyncClient,
- hit_record: Hit,
- assignment_stub_record: AssignmentStub,
- mturk_event_body_record: dict[str, Any],
- ):
- client = httpxclient
- redis_client = redis_config.create_redis_client()
-
- json_msg = json.loads(mturk_event_body_record["Message"])
- # Assert the mturk event is owned by the correct account
- assert json_msg["SourceAccount"] == settings.aws_owner_id
-
- # Assert the HIT and Assignment are "connected"
- assert assignment_stub_record.hit_id == hit_record.id
- assert assignment_stub_record.amt_hit_id == hit_record.amt_hit_id
-
- # Assert the event body and the Assignment/HIT are "connected"
- assert json_msg["Events"][0]["HITId"] == hit_record.amt_hit_id
- assert json_msg["Events"][0]["HITId"] == assignment_stub_record.amt_hit_id
- assert (
- json_msg["Events"][0]["AssignmentId"]
- == assignment_stub_record.amt_assignment_id
- )
-
- # Confirm the stream is empty
- assert redis_client.xlen(JB_EVENTS_STREAM) == 0
-
- res = await client.post(
- url=f"/{settings.sns_path}/", json=mturk_event_body_record
- )
- res.raise_for_status()
-
- # Now that we POSTed, confirm the stream has 1 event in it
- # Confirm the stream is empty
- assert redis_client.xlen(JB_EVENTS_STREAM) == 1
-
- # AMT SNS needs to receive a 200 response to stop retrying the notification
- assert res.status_code == 200
- assert res.json() == {"status": "ok"}
-
- # Check that the event was enqueued in Redis
- msg_res = redis_client.xread(streams={JB_EVENTS_STREAM: 0}, count=1, block=100)
- msg_res = msg_res[0][1][0]
- msg_id, msg = msg_res
- redis_client.xdel(JB_EVENTS_STREAM, msg_id)
-
- # After running xdel, we can confirm the stream is empty
- assert redis_client.xlen(JB_EVENTS_STREAM) == 0
-
- msg_json = msg["data"]
- event = MTurkEvent.model_validate_json(msg_json)
-
- # Confirm that the event that we got from redis is what was POSTED by
- # AMT SNS. We're using the fixture() so the values are the same as
- # what was in the example_mturk_event_body
- assert event.event_type == "AssignmentSubmitted"
-
- assert event.amt_assignment_id == assignment_stub_record.amt_assignment_id
- assert event.amt_hit_id == hit_record.amt_hit_id
- assert event.amt_hit_type_id == hit_record.amt_hit_type_id
diff --git a/tests/http/test_preview.py b/tests/http/test_preview.py
deleted file mode 100644
index 39a6f5b..0000000
--- a/tests/http/test_preview.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# There are two types of "preview" - one is where we navigate direclty
-# to it and one is where we redirect possibly
-
-import pytest
-from httpx import AsyncClient
-
-from jb.models.assignment import AssignmentStub
-from jb.models.hit import Hit
-
-
-class TestPreview:
-
- @pytest.mark.anyio
- async def test_preview_direct(self, httpxclient: AsyncClient):
- client = httpxclient
- res = await client.get("/preview/")
-
- assert res.status_code == 200
- assert res.headers["content-type"] == "text/html; charset=utf-8"
- assert res.num_bytes_downloaded == 507
-
- assert "James Billings loves you." in res.text
- assert "https://cdn.jamesbillings67.com/james-has-style.css" in res.text
- assert "https://cdn.jamesbillings67.com/james-billings.js" in res.text
-
- @pytest.mark.anyio
- async def test_preview_redirect_from_work_random_str(
- self, httpxclient: AsyncClient, amt_hit_id: str, amt_assignment_id: str
- ):
- client = httpxclient
-
- """
- The redirect occurs regardless of any parameter validation. This is
- because the query params will be used for record lookup on the work
- page itself.
- """
-
- params = {
- "workerId": None,
- "assignmentId": amt_assignment_id,
- "hitId": amt_hit_id,
- }
- res = await client.get("/work/", params=params)
-
- assert res.status_code == 302
- assert "/preview/" in res.headers["location"]
-
- @pytest.mark.anyio
- async def test_preview_redirect_from_work_records(
- self,
- httpxclient: AsyncClient,
- hit_record: Hit,
- assignment_stub_record: AssignmentStub,
- ):
- client = httpxclient
-
- """
- The redirect occurs regardless of any parameter validation. This is
- because the query params will be used for record lookup on the work
- page itself.
- """
-
- params = {
- "workerId": None,
- "assignmentId": assignment_stub_record.amt_assignment_id,
- "hitId": hit_record.amt_hit_id,
- }
- res = await client.get("/work/", params=params)
-
- assert res.status_code == 302
- assert "/preview/" in res.headers["location"]
diff --git a/tests/http/test_wallet.py b/tests/http/test_wallet.py
new file mode 100644
index 0000000..4ba5ba6
--- /dev/null
+++ b/tests/http/test_wallet.py
@@ -0,0 +1,96 @@
+from urllib.parse import parse_qs, urlparse
+from uuid import uuid4
+
+import pytest
+from generalresearch.redis_helper import RedisConfig
+from httpx import AsyncClient
+
+from jb.api.auth import SESSION_COOKIE_NAME, create_session
+from jb.api.cashout_token import CASHOUT_TOKEN_PREFIX
+from jb.api.magic_token import redis_token_key
+from jb.dependencies import get_gr_api_manager
+from jb.main import app
+from jb.models.auth import User
+from jb.models.wallet import PendingCashout
+
+
+class FakeGRApiManager:
+ def __init__(self, user: User):
+ self.user = user
+
+ def get_user(self, product_user_id: str) -> User:
+ assert product_user_id == self.user.product_user_id
+ return self.user
+
+
+@pytest.fixture
+def authenticated_client(httpxclient: AsyncClient, email: str):
+ user = User(email=email)
+ app.dependency_overrides[get_gr_api_manager] = lambda: FakeGRApiManager(user)
+ httpxclient.cookies.set(
+ SESSION_COOKIE_NAME, create_session(user.product_user_id), path="/"
+ )
+ yield httpxclient, user
+ httpxclient.cookies.delete(SESSION_COOKIE_NAME, path="/")
+ app.dependency_overrides.pop(get_gr_api_manager, None)
+
+
+class TestCashout:
+ @pytest.mark.anyio
+ async def test_request_requires_authentication(self, httpxclient: AsyncClient):
+ httpxclient.cookies.delete(SESSION_COOKIE_NAME, path="/")
+
+ response = await httpxclient.post(
+ "/wallet/cashout/request/", json={"amount": 500}
+ )
+
+ assert response.status_code == 401
+
+ @pytest.mark.anyio
+ @pytest.mark.skip("needs a cashout method mocked. this wont work")
+ async def test_request_binds_user_and_amount_to_single_use_token(
+ self,
+ authenticated_client: tuple[AsyncClient, User],
+ redis_config: RedisConfig,
+ ):
+ client, user = authenticated_client
+ cashout_method_id = uuid4().hex
+
+ response = await client.post(
+ "/wallet/cashout/request/",
+ json={"amount": 500, "cashout_method_id": cashout_method_id},
+ )
+
+ assert response.status_code == 200, response.text
+ token = parse_qs(urlparse(response.json()["confirmation_link"]).query)["token"][
+ 0
+ ]
+ stored = redis_config.create_redis_client().get(
+ redis_token_key(token, CASHOUT_TOKEN_PREFIX)
+ )
+ assert PendingCashout.model_validate_json(stored) == PendingCashout(
+ product_user_id=user.product_user_id,
+ amount=500,
+ cashout_method_id=cashout_method_id,
+ )
+
+ response = await client.post("/wallet/cashout/confirm/", json={"token": token})
+ assert response.status_code == 204
+
+ response = await client.post("/wallet/cashout/confirm/", json={"token": token})
+ assert response.status_code == 401
+
+ @pytest.mark.anyio
+ @pytest.mark.parametrize("amount", [0, -1, 1000])
+ async def test_request_rejects_invalid_amount(
+ self,
+ authenticated_client: tuple[AsyncClient, User],
+ amount: int,
+ ):
+ client, _ = authenticated_client
+
+ response = await client.post(
+ "/wallet/cashout/request/", json={"amount": amount}
+ )
+
+ assert response.status_code == 422
diff --git a/tests/http/test_work.py b/tests/http/test_work.py
deleted file mode 100644
index 9eee15a..0000000
--- a/tests/http/test_work.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import pytest
-from httpx import AsyncClient
-
-from jb.managers.assignment import AssignmentManager
-from jb.models.assignment import AssignmentStub
-from jb.models.hit import Hit
-
-
-class TestWork:
-
- @pytest.mark.anyio
- async def test_work(
- self,
- httpxclient: AsyncClient,
- hit_record: Hit,
- amt_assignment_id: str,
- amt_worker_id: str,
- ):
-
- assert isinstance(hit_record.id, int)
-
- params = {
- "workerId": amt_worker_id,
- "assignmentId": amt_assignment_id,
- "hitId": hit_record.amt_hit_id,
- }
- res = await httpxclient.get("/work/", params=params)
- assert res.status_code == 200
-
- @pytest.mark.anyio
- async def test_work_no_hit_record(
- self,
- httpxclient: AsyncClient,
- hit: Hit,
- amt_assignment_id: str,
- amt_worker_id: str,
- ):
- # Because no AssignmentStub record is created, and we're just using
- # random strings as IDs, we should also confirm that the Hit record
- # is not a saved record.
- assert hit.id is None
-
- params = {
- "workerId": amt_worker_id,
- "assignmentId": amt_assignment_id,
- "hitId": hit.amt_hit_id,
- }
- res = await httpxclient.get("/work/", params=params)
-
- # This either results a 302 redirect to the Preview page,
- # or a 200. In previous tests, it expected a 500 but is
- # unclear what that behavior was intended for, but does
- # not seem to be the expected response anyway.
- assert res.status_code == 200
-
- @pytest.mark.anyio
- async def test_work_assignment_stub_existing(
- self,
- httpxclient: AsyncClient,
- am: AssignmentManager,
- hit: Hit,
- assignment_stub_record: AssignmentStub,
- amt_assignment_id: str,
- amt_worker_id: str,
- ):
-
- # Because the AssignmentStub is created with a reference to the Hit,
- # the Hit is actually a "Hit Record" (with a primary key), so it's
- # saved in the database
- assert isinstance(hit.id, int)
-
- # Confirm that it exists in the database before the call
- res = am.get_stub_if_exists(amt_assignment_id=amt_assignment_id)
- assert isinstance(res, AssignmentStub)
- assert isinstance(res.id, int)
-
- params = {
- "workerId": amt_worker_id,
- "assignmentId": assignment_stub_record.amt_assignment_id,
- "hitId": hit.amt_hit_id,
- }
- res = await httpxclient.get("/work/", params=params)
- assert res.status_code == 200
-
- # Confirm that it exists in the database
- res = am.get_stub_if_exists(amt_assignment_id=amt_assignment_id)
- assert isinstance(res, AssignmentStub)
- assert isinstance(res.id, int)
-
- @pytest.mark.anyio
- async def test_work_assignment_stub_created(
- self,
- httpxclient: AsyncClient,
- am: AssignmentManager,
- hit_record: Hit,
- assignment_stub: AssignmentStub,
- amt_assignment_id: str,
- amt_worker_id: str,
- ):
-
- # Confirm that it exists in the database before the call
- res = am.get_stub_if_exists(amt_assignment_id=amt_assignment_id)
- assert res is None
-
- params = {
- "workerId": amt_worker_id,
- "assignmentId": assignment_stub.amt_assignment_id,
- "hitId": hit_record.amt_hit_id,
- }
- res = await httpxclient.get("/work/", params=params)
- assert res.status_code == 200
-
- # Confirm that it exists in the database
- res = am.get_stub_if_exists(amt_assignment_id=amt_assignment_id)
- # assert isinstance(res, AssignmentStub)
- # assert isinstance(res.id, int)
-
- # As of Sep 10th, 2026 - I don't see any logic where the /work/
- # would go ahead and create the Assignment Stub. Maybe it was moved
- # somewhere else, but it would continue to be None as the /work/
- # page only returns back the template or a redirect.. - Max
- assert res is None
diff --git a/tests/managers/__init__.py b/tests/managers/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/tests/managers/__init__.py
+++ /dev/null
diff --git a/tests/managers/test_amt.py b/tests/managers/test_amt.py
deleted file mode 100644
index 6a944a8..0000000
--- a/tests/managers/test_amt.py
+++ /dev/null
@@ -1,132 +0,0 @@
-from mypy_boto3_mturk import MTurkClient
-from mypy_boto3_mturk.type_defs import (
- GetAccountBalanceResponseTypeDef,
- ListHITsResponseTypeDef,
-)
-
-from jb.managers.amt import AMTManager
-from jb.managers.hit import HitManager, HitTypeManager
-from jb.models.hit import HitQuestion, HitType
-
-# from jb.decorators import HM
-# from jb.flow.tasks import refill_hits, check_stale_hits, check_expired_hits
-
-
-# def test_refill_hits(
-# set_hit_types_in_db_min_active_0,
-# hit_type_record, expire_all_hits, amt_manager
-# ):
-
-# assert HM.get_active_count(hit_type_in_db.id) == 0
-# assert hit_type_in_db.min_active > 0
-# refill_hits()
-# assert HM.get_active_count(hit_type_in_db.id) == hit_type_in_db.min_active
-
-# amt_hit_ids = HM.filter_active_ids(hit_type_id=hit_type_in_db.id)
-# amt_hit_id = list(amt_hit_ids)[0]
-# hit, _ = amt_manager.get_hit_if_exists(amt_hit_id=amt_hit_id)
-# assert hit
-
-
-# def test_check_stale_hits():
-# # todo: I'd have to create some purposely stale hits.
-# # just make sure it runs for now
-# check_stale_hits()
-
-
-# def test_check_expired_hits():
-# # todo: I'd have to create some purposely expired hits.
-# # just make sure it runs for now
-# check_expired_hits()
-
-
-class TestMTurkClient:
-
- def test_sandbox(self, amt_client: MTurkClient):
- assert amt_client is not None
- assert "mturk-requester-sandbox" in amt_client.meta.endpoint_url
-
- balance: GetAccountBalanceResponseTypeDef = amt_client.get_account_balance()
- assert balance["AvailableBalance"] == "10000.00"
-
- # def test_create_hit()
-
- # def test_delete_hit()
-
- # def test_get_assignment(self, amt_client: MTurkClient):
-
- # def test_get_hit
-
- def test_list_hits(self, amt_client: MTurkClient, amtm: AMTManager):
- res: ListHITsResponseTypeDef = amt_client.list_hits()
- assert isinstance(res, dict)
- assert isinstance(res.get("HITs"), list)
-
- for hit in res["HITs"]:
- assert isinstance(hit, dict)
-
- assert isinstance(hit.get("HITId"), str)
- assert isinstance(hit.get("HITTypeId"), str)
- assert isinstance(hit.get("HITGroupId"), str)
- assert isinstance(hit.get("Title"), str)
-
- # botocore.errorfactory.RequestError: An error occurred
- # (RequestError) when calling the DeleteHIT operation: This
- # HIT is currently in the state 'Assignable'. This operation
- # can be called with a status of: Reviewing, Reviewable
- # (1772146028380 s)
-
- if hit["HITStatus"] in ["Reviewing", "Reviewable"]:
- print(
- f"Deleting HIT with id {hit['HITId']} and status {hit['HITStatus']}"
- )
- amt_client.delete_hit(HITId=hit["HITId"])
-
- elif hit["HITStatus"] == "Assignable":
- # I don't know... just let expire I guess. To changes it's
- # status you need to do manage Assignments (I think)
-
- # amt_client.update_hit_review_status(HITId=hit["HITId"], Revert=True)
- pass
-
- amtm.expire_all_hits()
-
-
-class TestAMTManager:
-
- def test_create_hit_type(self, amtm: AMTManager, hit_type: HitType):
- assert hit_type.amt_hit_type_id is None
-
- amtm.create_hit_type(hit_type=hit_type)
- assert hit_type.amt_hit_type_id is not None
-
- def test_create_hit_with_hit_type(
- self,
- amtm: AMTManager,
- htm: HitTypeManager,
- hm: HitManager,
- hit_type_record_with_amt_id: HitType,
- question_record: HitQuestion,
- ):
-
- sandbox_amt_hit_type_id = hit_type_record_with_amt_id.amt_hit_type_id
- assert isinstance(sandbox_amt_hit_type_id, str)
-
- res = [
- x
- for x in htm.filter_active()
- if x.amt_hit_type_id == sandbox_amt_hit_type_id
- ]
-
- assert len(res) == 1
- sandbox_hit_type = res[0]
-
- hit = amtm.create_hit_with_hit_type(
- hit_type=sandbox_hit_type, question=question_record
- )
-
- assert hit.amt_hit_id is not None
- assert hit.id is None
-
- hm.create(hit)
- assert hit.id is not None
diff --git a/tests/managers/test_hit.py b/tests/managers/test_hit.py
deleted file mode 100644
index 7227e4d..0000000
--- a/tests/managers/test_hit.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from jb.managers.hit import HitManager, HitTypeManager
-from jb.models.hit import Hit, HitQuestion, HitType
-
-
-class TestHitQuestionManager:
-
- def test_base(self, question_record: HitQuestion):
- assert isinstance(question_record, HitQuestion)
- assert isinstance(question_record.id, int)
-
-
-class TestHitTypeManager:
-
- def test_create(self, htm: HitTypeManager, hit_type_record: HitType):
-
- assert isinstance(hit_type_record.id, int)
- assert isinstance(hit_type_record.amt_hit_type_id, str)
-
- count1 = len(htm.filter_active())
- # assert count1 == 1
-
- hit_type_record.min_active = 0
- htm.set_min_active(hit_type=hit_type_record)
-
- count2 = len(htm.filter_active())
- assert count1 - 1 == count2
-
-
-class TestHitManager:
-
- # def test_create
-
- # def update_status
-
- # def update_hit
-
- # def get_from_amt_id
-
- # get_from_amt_id_if_exists
-
- def test_get_active_count(self, hm: HitManager, hit_record: Hit):
-
- count = hm.get_active_count(hit_type_id=999_999_999)
- assert isinstance(count, int)
- assert count == 0
-
- count = hm.get_active_count(hit_type_id=hit_record.hit_type_id)
- assert count == 1
-
- # filter_active_ids
diff --git a/tests/models/__init__.py b/tests/models/__init__.py
deleted file mode 100644
index e69de29..0000000
--- a/tests/models/__init__.py
+++ /dev/null
diff --git a/tests/models/test_assignment.py b/tests/models/test_assignment.py
deleted file mode 100644
index ecaafe9..0000000
--- a/tests/models/test_assignment.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from mypy_boto3_mturk.type_defs import (
- GetAssignmentResponseTypeDef,
-)
-
-from jb.models.assignment import Assignment, AssignmentStub
-
-
-class TestAssignmentStub:
-
- def test_init(self, assignment_stub: AssignmentStub):
- assert isinstance(assignment_stub, AssignmentStub)
- assert assignment_stub.id is None
-
- def test_record(self, assignment_stub_record: AssignmentStub):
- assert isinstance(assignment_stub_record, AssignmentStub)
- assert assignment_stub_record.id is not None
-
-
-class TestAssignment:
-
- def test_get_assignment(self, assignment_record: Assignment):
- assert isinstance(assignment_record, Assignment)
- assert assignment_record.id is not None
-
-
-class TestAssignmentResponse:
-
- def test_get_assignment_no_tsid(
- self, assignment_response_no_tsid: GetAssignmentResponseTypeDef
- ):
- assignment = Assignment.from_amt_get_assignment(
- assignment_response_no_tsid["Assignment"]
- )
- assert assignment.tsid is None
-
- def test_get_assignment_bad_tsid(
- self, assignment_response_bad_tsid: GetAssignmentResponseTypeDef
- ):
- assignment = Assignment.from_amt_get_assignment(
- assignment_response_bad_tsid["Assignment"]
- )
- assert assignment.tsid is None
diff --git a/tests/models/test_event.py b/tests/models/test_event.py
deleted file mode 100644
index a4c591d..0000000
--- a/tests/models/test_event.py
+++ /dev/null
@@ -1,12 +0,0 @@
-import pytest
-
-from jb.models.event import MTurkEvent
-
-
-class TestMTurkEvent:
-
- @pytest.mark.anyio
- def test_init(self, mturk_event: MTurkEvent):
- assert isinstance(mturk_event, MTurkEvent)
- assert isinstance(mturk_event.amt_assignment_id, str)
- assert isinstance(mturk_event.amt_hit_type_id, str)
diff --git a/tests/models/test_hit.py b/tests/models/test_hit.py
deleted file mode 100644
index aa48f00..0000000
--- a/tests/models/test_hit.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import pytest
-
-from jb.models.hit import Hit
-
-
-class TestAssignment:
-
- @pytest.mark.anyio
- def test_get_assignment(self, hit_record: Hit):
- assert isinstance(hit_record, Hit)
- assert hit_record.id is not None
diff --git a/tests/test_postgres.py b/tests/test_postgres.py
deleted file mode 100644
index 244c132..0000000
--- a/tests/test_postgres.py
+++ /dev/null
@@ -1,69 +0,0 @@
-import socket
-import subprocess
-from collections.abc import Callable
-from typing import TYPE_CHECKING
-
-from generalresearch.pg_helper import PostgresConfig
-from pydantic import PostgresDsn
-
-if TYPE_CHECKING:
- from generalresearch.models.custom_types import InternalHostname, PostgresDict
-
-
-def is_port_open(host: InternalHostname, port: int = 5432, timeout: int = 3):
- try:
- with socket.create_connection((host, port), timeout=timeout):
- return True
- except (TimeoutError, ConnectionRefusedError, OSError):
- return False
-
-
-def can_ping(host: InternalHostname):
- return (
- subprocess.call(
- ["ping", "-c", "1", str(host)],
- stdout=subprocess.DEVNULL,
- stderr=subprocess.DEVNULL,
- )
- == 0
- )
-
-
-class TestPostgresDSN:
-
- def test_ping(self, postgres_instance_host: InternalHostname):
- assert can_ping(host=postgres_instance_host)
-
- def test_port(self, postgres_instance_host: InternalHostname):
- assert is_port_open(host=postgres_instance_host)
-
- def test_conn(self, postgres_instance: PostgresDsn):
- config = PostgresConfig(
- dsn=postgres_instance,
- connect_timeout=1,
- statement_timeout=1,
- )
- res = config.execute_sql_query(query="SELECT 1;")
- assert len(res) == 1
-
-
-class TestPostgresDjangoCreation:
-
- def test_ping(self, postgres_instance_dict: PostgresDict):
- assert can_ping(host=postgres_instance_dict["host"])
-
- def test_django_creation(
- self,
- django_db_factory: Callable[..., None],
- ):
- dsn = django_db_factory()
- assert isinstance(dsn, PostgresDsn)
-
- def test_django_tables(self, pg_config: PostgresConfig):
- res = pg_config.execute_sql_query(query="""
- SELECT COUNT(*)
- FROM information_schema.tables
- WHERE table_schema = 'public';
- """)
- assert len(res) == 1
- assert res[0]["count"] == 7