diff options
| author | stuppie | 2026-11-04 12:11:03 -0600 |
|---|---|---|
| committer | stuppie | 2026-11-04 12:11:03 -0600 |
| commit | 671a7b1b02503cfa5c9aecd40c27a0c2729a285b (patch) | |
| tree | 6cb1bd7c33e3e1d243ee5ae74decc57faaa5742d /jb/models | |
| parent | e99ec6b7e38113a2237dc1ff402d8f4337c651d5 (diff) | |
| download | amt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.tar.gz amt-jb-671a7b1b02503cfa5c9aecd40c27a0c2729a285b.zip | |
strip out AMT
Diffstat (limited to 'jb/models')
| -rw-r--r-- | jb/models/amt.py | 19 | ||||
| -rw-r--r-- | jb/models/api_response.py | 17 | ||||
| -rw-r--r-- | jb/models/assignment.py | 396 | ||||
| -rw-r--r-- | jb/models/bonus.py | 48 | ||||
| -rw-r--r-- | jb/models/custom_types.py | 19 | ||||
| -rw-r--r-- | jb/models/definitions.py | 64 | ||||
| -rw-r--r-- | jb/models/errors.py | 80 | ||||
| -rw-r--r-- | jb/models/event.py | 48 | ||||
| -rw-r--r-- | jb/models/hit.py | 260 | ||||
| -rw-r--r-- | jb/models/response.py | 21 |
10 files changed, 0 insertions, 972 deletions
diff --git a/jb/models/amt.py b/jb/models/amt.py deleted file mode 100644 index e012741..0000000 --- a/jb/models/amt.py +++ /dev/null @@ -1,19 +0,0 @@ -from decimal import Decimal - -from pydantic import BaseModel, ConfigDict, Field - - -class AMTAccount(BaseModel): - model_config = ConfigDict(extra="ignore", validate_assignment=True) - - # Remaining available AWS Billing usage if you have enabled AWS Billing. - available_balance: Decimal = Field() - onhold_balance: Decimal = Field(default=Decimal(0)) - - # --- Properties --- - - @property - def is_healthy(self) -> bool: - # A healthy account is one with at least $2,500 worth of - # credit available to it - return self.available_balance >= 2_500 diff --git a/jb/models/api_response.py b/jb/models/api_response.py deleted file mode 100644 index 6b29e51..0000000 --- a/jb/models/api_response.py +++ /dev/null @@ -1,17 +0,0 @@ -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from jb.models.assignment import Assignment -from jb.models.hit import Hit - - -class AssignmentResponse(BaseModel): - model_config = ConfigDict(extra="forbid", validate_assignment=True) - - assignment: Assignment = Field(alias="Assignment") - hit: Hit = Field(alias="HIT") - - @model_validator(mode="after") - def check_consistent_hit_id(self) -> "AssignmentResponse": - if self.hit.id != self.assignment.hit_id: - raise ValueError("Inconsistent Hit IDs") - return self diff --git a/jb/models/assignment.py b/jb/models/assignment.py deleted file mode 100644 index fa6ccd5..0000000 --- a/jb/models/assignment.py +++ /dev/null @@ -1,396 +0,0 @@ -import logging -from datetime import datetime, timezone -from typing import Any, TypedDict -from xml.etree import ElementTree - -from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr -from mypy_boto3_mturk.type_defs import AssignmentTypeDef -from pydantic import ( - BaseModel, - ConfigDict, - Field, - PositiveInt, - TypeAdapter, - ValidationError, - model_validator, -) -from typing_extensions import Self - -from jb.models.custom_types import AMTBoto3ID -from jb.models.definitions import AssignmentStatus - -logger = logging.getLogger("amtjb") - - -class AnswerDict(TypedDict): - amt_assignment_id: str - amt_worker_id: str - tsid: str - - -class AssignmentStub(BaseModel): - # todo: we need an "AssignmentStub" model that just has - # the IDs, this is used when a user accepts an assignment - # but hasn't submitted it yet. We want to create it in the db - # at that point. - - model_config = ConfigDict( - extra="forbid", - validate_assignment=True, - ) - - id: PositiveInt | None = Field(default=None) - hit_id: PositiveInt | None = Field(default=None) - amt_assignment_id: AMTBoto3ID = Field() - amt_hit_id: AMTBoto3ID = Field() - amt_worker_id: str = Field(min_length=3, max_length=50) - - status: AssignmentStatus = Field() - - # GRL Specific - created_at: AwareDatetimeISO = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When this record was saved in the database", - ) - - modified_at: AwareDatetimeISO | None = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When this record was updated / modified in the database", - ) - - def to_postgres(self): - d = self.model_dump(mode="json") - return d - - -class Assignment(AssignmentStub): - """ - The Assignment data structure represents a single assignment of a HIT to - a Worker. The assignment tracks the Worker's efforts to complete the HIT, - and contains the results for later retrieval. - - The Assignment data structure is used as a response element for the - following operations: - - GetAssignment - GetAssignmentsForHIT - - https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html - """ - - auto_approval_time: AwareDatetimeISO = Field( - description="If results have been submitted, AutoApprovalTime is the " - "date and time the results of the assignment results are " - "considered Approved automatically if they have not already " - "been explicitly approved or rejected by the Requester. " - "This value is derived from the auto-approval delay " - "specified by the Requester in the HIT. This value is " - "omitted from the assignment if the Worker has not yet " - "submitted results.", - ) - - accept_time: AwareDatetimeISO = Field( - description="The date and time the Worker accepted the assignment.", - ) - - submit_time: AwareDatetimeISO = Field( - description="The date and time the assignment was submitted. This value " - "is omitted from the assignment if the Worker has not yet " - "submitted results.", - ) - - approval_time: AwareDatetimeISO | None = Field( - default=None, - description="The date and time the Requester approved the results. This " - "value is omitted from the assignment if the Requester has " - "not yet approved the results.", - ) - rejection_time: AwareDatetimeISO | None = Field( - default=None, - description="The date and time the Requester rejected the results.", - ) - - requester_feedback: str | None = Field( - # Default: None. This field isn't returned with assignment data by - # default. To request this field, specify a response group of - # AssignmentFeedback. For information about response groups, see - # Common Parameters. - default=None, - min_length=3, - max_length=2_000, - json_schema_extra={ - "help_text": "The feedback string included with the call to the " - "ApproveAssignment operation or the RejectAssignment " - "operation, if the Requester approved or rejected the " - "assignment and specified feedback." - }, - ) - - answer_xml: str | None = Field(default=None, exclude=True) - - # GRL Specific - - tsid: UUIDStr | None = Field(default=None) - - # --- Validators --- - - @model_validator(mode="before") - def set_tsid(cls, values: dict[str, Any]) -> dict[str, Any]: - if values.get("tsid") is None and (answer_xml := values.get("answer_xml")): - answer_dict = cls.parse_answer_xml(answer_xml) - tsid = answer_dict.get("tsid") - try: - values["tsid"] = TypeAdapter(UUIDStr).validate_python(tsid) - except ValidationError as e: - # Don't break the model validation if a baddie messes with the tsid in the answer. - logger.warning(e) - - values["tsid"] = None - return values - - @model_validator(mode="after") - def check_time_sequences(self) -> Self: - if self.accept_time > self.submit_time: - raise ValueError("Assignment times invalid") - - return self - - @model_validator(mode="after") - def check_answers_alignment(self) -> Self: - if self.answers_dict is None: - return self - if "amt_worker_id" not in self.answers_dict: - raise ValueError("answers_dict missing amt_worker_id") - if self.amt_worker_id != self.answers_dict["amt_worker_id"]: - raise ValueError("Assignment answer invalid worker_id") - if "amt_assignment_id" not in self.answers_dict: - raise ValueError("answers_dict missing amt_assignment_id") - if self.amt_assignment_id != self.answers_dict["amt_assignment_id"]: - raise ValueError("Assignment answer invalid amt_assignment_id") - if self.tsid: - if "tsid" not in self.answers_dict: - raise ValueError("answers_dict missing tsid") - if self.answers_dict["tsid"] and self.tsid != self.answers_dict["tsid"]: - raise ValueError("Assignment answer invalid tsid") - return self - - # --- Properties --- - - @property - def answers_dict(self) -> AnswerDict | None: - # See https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_AssignmentDataStructureArticle.html - # https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMechanicalTurkRequester/Concepts_NotificationsArticle.html - if self.answer_xml is None: - return None - - return self.parse_answer_xml(self.answer_xml) # type: ignore - - @staticmethod - def parse_answer_xml(answer_xml: str) -> dict[str, Any]: - root = ElementTree.fromstring(answer_xml) - ns = { - "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionFormAnswers.xsd" - } - res = {} - - for a in root.findall("mt:Answer", ns): - name = a.find("mt:QuestionIdentifier", ns).text # type: ignore - value = a.find("mt:FreeText", ns).text # type: ignore - res[name] = value or "" - - EXPECTED_KEYS = {"amt_assignment_id", "amt_worker_id", "tsid"} - # We don't want validation to fail if a baddie inserts or changes url - # params, which will result in new or missing keys. Amazon generates the xml - # so that should always be correct - # assert all(k in res for k in EXPECTED_KEYS), list(res.keys()) - res = {k: v for k, v in res.items() if k in EXPECTED_KEYS} - return res - - @classmethod - def from_amt_get_assignment(cls, data: AssignmentTypeDef) -> Self: - assignment = cls( - amt_assignment_id=data["AssignmentId"], - amt_hit_id=data["HITId"], - amt_worker_id=data["WorkerId"], - status=AssignmentStatus[data["AssignmentStatus"]], - auto_approval_time=data["AutoApprovalTime"].astimezone(tz=timezone.utc), - accept_time=data["AcceptTime"].astimezone(tz=timezone.utc), - submit_time=data["SubmitTime"].astimezone(tz=timezone.utc), - approval_time=( - data["ApprovalTime"].astimezone(tz=timezone.utc) - if data.get("ApprovalTime") - else None - ), - rejection_time=( - data["RejectionTime"].astimezone(tz=timezone.utc) - if data.get("RejectionTime") - else None - ), - answer_xml=data["Answer"], - requester_feedback=data.get("RequesterFeedback"), - ) - return assignment - - def to_stub(self) -> AssignmentStub: - return AssignmentStub.model_validate( - self.model_dump(include=set(AssignmentStub.model_fields.keys())) - ) - - # --- Methods --- - # - # def refresh(self) -> Self: - # from tasks.mtwerk.managers.assignment import AssignmentManager - # return AssignmentManager.fetch_by_id(self) - # - # def reject(self, msg: str = REJECT_MESSAGE_UNKNOWN_ASSIGNMENT): - # """ - # Save in the database that the Assignment was rejected, and also - # Report to Amazon Mechanical Turk that this Assignment should be - # rejected - # - # TODO: can this only occur when the Assignment is in a certain status? - # - # :return: - # """ - # now = datetime.now(tz=None) - # - # MYSQLC.execute_sql_query(""" - # UPDATE `amt-jb`.`mtwerk_assignment` - # SET submit_time = %s, rejection_time = %s, status = %s, - # requester_feedback = %s - # WHERE assignment_id = %s""", - # params=[ - # now, now, - # AssignmentStatus.Rejected.value, - # msg, self.id], - # commit=True) - # - # CLIENT.reject_assignment( - # AssignmentId=self.id, - # RequesterFeedback=msg) - # - # def approve(self, msg: str = "Approved."): - # """ - # Report to Amazon Mechanical Turk that this Assignment should be - # approved - # - # TODO: can this only occur when the Assignment is in a certain status? - # - # :return: - # """ - # CLIENT.approve_assignment( - # AssignmentId=self.id, - # RequesterFeedback=msg) - # - # def submit_and_complete_request(self) -> Optional[str]: - # """ - # This approves the Assignment and issues the Reward - # amount (typically $.05) - # - # :return: - # """ - # worker = self.worker - # amount = DecimalUSDDollars(self.hit.reward) - # - # # If successful, returns the cashout id, otherwise, returns None - # cashout: Optional[dict] = worker.cashout_request( - # amount=amount, - # cashout_method_id=AMT_ASSIGNMENT_CASHOUT_METHOD) - # - # if cashout is None or cashout.get('status') != PayoutStatus.PENDING: - # return None - # - # cashout_id: str = cashout[id] - # - # approval: Optional[dict] = Bonus.manage_pending_cashout( - # cashout_id=cashout_id, - # action=PayoutStatus.APPROVED) - # - # if approval is None or approval['status'] != PayoutStatus.APPROVED: - # return None - # - # completion: Optional[dict] = Bonus.manage_pending_cashout( - # cashout_id=cashout_id, - # action=PayoutStatus.COMPLETE) - # - # if completion is None or completion['status'] != PayoutStatus.COMPLETE: - # return None - # - # return cashout_id - # - # # --- ORM --- - # - # def model_dump_mysql(self, *args, **kwargs) -> dict: - # d = self.model_dump(mode='json', *args, **kwargs) - # - # d['auto_approval_time'] = self.auto_approval_time.replace(tzinfo=None) - # d['accept_time'] = self.accept_time.replace(tzinfo=None) - # d['submit_time'] = self.submit_time.replace(tzinfo=None) - # - # if self.approval_time: - # d['approval_time'] = self.approval_time.replace(tzinfo=None) - # - # if self.rejection_time: - # d['rejection_time'] = self.rejection_time.replace(tzinfo=None) - # - # # created is automatically added by the database - # d['created'] = self.created.replace(tzinfo=None) - # - # if self.modified: - # d['modified'] = self.modified.replace(tzinfo=None) - # - # d['tsid'] = self.answers.get('tsid') - # - # return d - # - # def save(self) -> bool: - # """ - # Either INSERTS or UPDATES the Assignment instance to a Mysql - # record. - # """ - # - # # We're modifying the record, so set the time to right now! - # self.modified = datetime.now(tz=timezone.utc) - # - # query = """ - # INSERT `amt-jb`.`mtwerk_assignment` ( - # id, worker_id, hit_id, status, - # auto_approval_time, accept_time, submit_time, - # approval_time, rejection_time, - # requester_feedback, created, modified, tsid - # ) - # VALUES ( - # %(id)s, %(worker_id)s, %(hit_id)s, %(status)s, - # %(auto_approval_time)s, %(accept_time)s, %(submit_time)s, - # %(approval_time)s, %(rejection_time)s, - # %(requester_feedback)s, %(created)s, %(modified)s, %(tsid)s - # ) - # ON DUPLICATE KEY UPDATE - # worker_id = %(worker_id)s, - # hit_id = %(hit_id)s, - # status = %(status)s, - # - # auto_approval_time = %(auto_approval_time)s, - # accept_time = %(accept_time)s, - # submit_time = %(submit_time)s, - # - # approval_time = %(approval_time)s, - # rejection_time = %(rejection_time)s, - # - # requester_feedback = %(requester_feedback)s, - # -- Not going to update created just incase it changed - # -- in pydantic for some reason - # modified = %(modified)s, - # tsid = %(tsid)s - # """ - # - # try: - # MYSQLC.execute_sql_query(query, params=self.model_dump_mysql(), commit=True) - # return True - # - # except Exception as e: - # return False - # - - -# REJECT_MESSAGE_UNKNOWN_ASSIGNMENT = "Unknown assignment" diff --git a/jb/models/bonus.py b/jb/models/bonus.py deleted file mode 100644 index c6da3c4..0000000 --- a/jb/models/bonus.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Any - -from generalresearch.currency import USDCent -from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr -from pydantic import BaseModel, ConfigDict, Field, PositiveInt -from typing_extensions import Self - -from jb.models.custom_types import AMTBoto3ID - - -class Bonus(BaseModel): - """ - A Bonus is created (in our DB) ONLY associated with an APPROVED - thl-payout-event, AFTER the bonus has actually been sent to - the worker. - We have the payout_event uuid as the unique request token to make - sure it only gets sent once (param in the boto request). - """ - - model_config = ConfigDict( - extra="forbid", - validate_assignment=True, - ) - id: PositiveInt | None = Field(default=None) - assignment_id: PositiveInt | None = Field(default=None) - - amt_worker_id: str = Field(min_length=3, max_length=50) - amt_assignment_id: AMTBoto3ID = Field() - - amount: USDCent = Field() - reason: str = Field(min_length=5) - grant_time: AwareDatetimeISO = Field() - - # -- GRL Specific --- - payout_event_id: UUIDStr = Field() - # created: Optional[AwareDatetimeISO] = Field(default=None) - - def to_postgres(self): - d = self.model_dump(mode="json") - d["amount"] = self.amount.to_usd() - return d - - @classmethod - def from_postgres(cls, data: dict[str, Any]) -> Self: - data["amount"] = USDCent(round(data["amount"] * 100)) - fields = set(cls.model_fields.keys()) - data = {k: v for k, v in data.items() if k in fields} - return cls.model_validate(data) diff --git a/jb/models/custom_types.py b/jb/models/custom_types.py deleted file mode 100644 index 385c1ba..0000000 --- a/jb/models/custom_types.py +++ /dev/null @@ -1,19 +0,0 @@ -import re -from typing import Annotated - -from pydantic import StringConstraints -from pydantic.functional_validators import AfterValidator - - -def check_valid_amt_boto3_id(v: str) -> str: - # Test ids from amazon have 20 chars - if not re.fullmatch(r"[A-Z0-9]{20}|[A-Z0-9]{30}", v): - raise ValueError("Invalid AMT Boto3 ID") - return v - - -AMTBoto3ID = Annotated[ - str, - StringConstraints(min_length=20, max_length=30), - AfterValidator(check_valid_amt_boto3_id), -] diff --git a/jb/models/definitions.py b/jb/models/definitions.py deleted file mode 100644 index 4ae7a21..0000000 --- a/jb/models/definitions.py +++ /dev/null @@ -1,64 +0,0 @@ -from enum import IntEnum - - -class AssignmentStatus(IntEnum): - # boto3.mturk specific - Submitted = 0 # same thing as Reviewable - Approved = 1 - Rejected = 2 - - # GRL specific - Accepted = 3 - PreviewState = 4 - # Invalid = 5 - # NotExist = 6 - - -class HitStatus(IntEnum): - """ - https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_HITDataStructureArticle.html - """ - - # Official boto3.mturk - Assignable = 0 - Unassignable = 1 - Reviewable = 2 - Reviewing = 3 - Disposed = 4 - - # GRL Specific - NotExist = 5 - - -class HitReviewStatus(IntEnum): - NotReviewed = 0 - MarkedForReview = 1 - ReviewedAppropriate = 2 - ReviewedInappropriate = 3 - - -class ReportValue(IntEnum): - """ - The reason a user reported a task. - """ - - # Used to indicate the user exited the task without giving feedback - REASON_UNKNOWN = 0 - # Task is in the wrong language/country, unanswerable question, won't proceed to - # next question, loading forever, error message - TECHNICAL_ERROR = 1 - # Task ended (completed or failed, and showed the user some dialog - # indicating the task was over), but failed to redirect - NO_REDIRECT = 2 - # Asked for full name, home address, identity on another site, cc# - PRIVACY_INVASION = 3 - # Asked about children, employer, medical issues, drug use, STDs, etc. - UNCOMFORTABLE_TOPICS = 4 - # Asked to install software, signup/login to external site, access webcam, - # promise to pay using external site, etc. - ASKED_FOR_NOT_ALLOWED_ACTION = 5 - # Task doesn't work well on a mobile device - BAD_ON_MOBILE = 6 - # Too long, too boring, confusing, complicated, too many - # open-ended/free-response questions - DIDNT_LIKE = 7 diff --git a/jb/models/errors.py b/jb/models/errors.py deleted file mode 100644 index 1fe71df..0000000 --- a/jb/models/errors.py +++ /dev/null @@ -1,80 +0,0 @@ -import re -from enum import Enum - -from pydantic import BaseModel, ConfigDict, Field, model_validator - -from jb.models.response import ResponseMetadata - - -class BotoRequestErrorOperation(str, Enum): - GET_ASSIGNMENT = "GetAssignment" - GET_HIT = "GetHIT" - - -class TurkErrorCode(str, Enum): - # Unclear: maybe when it's new? - HIT_NOT_EXIST = "AWS.MechanicalTurk.HITDoesNotExist" - - # This seems to be for really old Assignments - # Also maybe when it's only a Preview? - # Happens 2+ years back, and also from past 24hrs - INVALID_ASSIGNEMENT_STATE = "AWS.MechanicalTurk.InvalidAssignmentState" - - # If random assignmentId is used - ASSIGNMENT_NOT_EXIST = "AWS.MechanicalTurk.AssignmentDoesNotExist" - - -class BotoRequestErrorResponseErrorCodes(str, Enum): - REQUEST_ERROR = "RequestError" - - -class BotoRequestErrorResponseError(BaseModel): - model_config = ConfigDict(extra="forbid", validate_assignment=True) - - message: str = Field(alias="Message") - code: BotoRequestErrorResponseErrorCodes = Field(alias="Code") - - -class BotoRequestErrorResponse(BaseModel): - model_config = ConfigDict(extra="forbid", validate_assignment=True) - - error: BotoRequestErrorResponseError = Field(alias="Error") - response_metadata: ResponseMetadata = Field(alias="ResponseMetadata") - message: str = Field(alias="Message", min_length=50) - error_code: TurkErrorCode = Field(alias="TurkErrorCode") - - @model_validator(mode="after") - def check_consistent_hit_id(self) -> "BotoRequestErrorResponse": - - match self.error_code: - case TurkErrorCode.HIT_NOT_EXIST: - if not re.match( - r"Hit [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message - ): - raise ValueError("Unknown message for TurkErrorCode.HIT_NOT_EXIST") - - case TurkErrorCode.INVALID_ASSIGNEMENT_STATE: - if not re.match( - r"This operation can be called with a status of: Reviewable,Approved,Rejected \(\d{13}\)", - self.message, - ): - raise ValueError( - "Unknown message for TurkErrorCode.INVALID_ASSIGNEMENT_STATE" - ) - - case TurkErrorCode.ASSIGNMENT_NOT_EXIST: - if not re.match( - r"Assignment [A-Z0-9]{30} does not exist. \(\d{13}\)", self.message - ): - raise ValueError( - "Unknown message for TurkErrorCode.ASSIGNMENT_NOT_EXIST" - ) - - return self - - -class BotoRequestError(BaseModel): - model_config = ConfigDict(extra="forbid", validate_assignment=True) - - response: BotoRequestErrorResponse = Field() - operation_name: BotoRequestErrorOperation = Field() diff --git a/jb/models/event.py b/jb/models/event.py deleted file mode 100644 index fb5735b..0000000 --- a/jb/models/event.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Any - -from generalresearch.models.custom_types import AwareDatetimeISO -from mypy_boto3_mturk.literals import EventTypeType -from pydantic import BaseModel, Field - -from jb.models.custom_types import AMTBoto3ID - - -class MTurkEvent(BaseModel): - """ - What AWS SNS will POST to our mturk_notifications endpoint (inside the request body) - """ - - event_type: EventTypeType = Field( - json_schema_extra={"example": "AssignmentSubmitted"} - ) - event_timestamp: AwareDatetimeISO = Field( - json_schema_extra={"example": "2025-10-16T18:45:51Z"} - ) - amt_hit_id: AMTBoto3ID = Field( - json_schema_extra={"example": "12345678901234567890"} - ) - amt_assignment_id: str = Field( - max_length=64, - json_schema_extra={"example": "1234567890123456789012345678901234567890"}, - ) - amt_hit_type_id: AMTBoto3ID = Field( - json_schema_extra={"example": "09876543210987654321"} - ) - - @classmethod - def from_sns(cls, data: dict[str, Any]): - return cls.model_validate( - { - "event_type": data["EventType"], - "event_timestamp": cls.fix_mturk_timestamp(data["EventTimestamp"]), - "amt_hit_id": data["HITId"], - "amt_assignment_id": data["AssignmentId"], - "amt_hit_type_id": data["HITTypeId"], - } - ) - - @staticmethod - def fix_mturk_timestamp(ts: str) -> str: - if ts.endswith("Z") and "." not in ts: - ts = ts[:-1] + ".000Z" - return ts diff --git a/jb/models/hit.py b/jb/models/hit.py deleted file mode 100644 index a550943..0000000 --- a/jb/models/hit.py +++ /dev/null @@ -1,260 +0,0 @@ -from datetime import datetime, timedelta, timezone -from typing import Any -from uuid import uuid4 -from xml.etree import ElementTree - -from generalresearch.currency import USDCent -from generalresearch.models.custom_types import AwareDatetimeISO, HttpsUrlStr -from mypy_boto3_mturk.type_defs import HITTypeDef -from pydantic import ( - BaseModel, - ConfigDict, - Field, - NonNegativeInt, - PositiveInt, -) -from typing_extensions import Self - -from jb.models.custom_types import AMTBoto3ID -from jb.models.definitions import HitReviewStatus, HitStatus - - -class HitQuestion(BaseModel): - id: PositiveInt | None = Field(default=None) - - url: HttpsUrlStr = Field() - height: PositiveInt = Field(default=1_200, ge=100, le=4_000) - - # --- Properties --- - - def to_postgres(self): - return self.model_dump(mode="json") - - @property - def xml(self) -> str: - return f"""<?xml version="1.0" encoding="UTF-8"?> - <ExternalQuestion xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd"> - <ExternalURL>{self.url!s}</ExternalURL> - <FrameHeight>{self.height}</FrameHeight> - </ExternalQuestion>""" - - -class HitTypeCommon(BaseModel): - """ - Fields on both the HitType and Hit - """ - - model_config = ConfigDict( - extra="forbid", validate_assignment=True, ser_json_timedelta="float" - ) - - title: str = Field( - min_length=3, - max_length=200, - description="The HIT post title that appears in the listing view", - ) - description: str = Field( - min_length=3, - max_length=2_000, - description="The expand more about textarea, has a max of 2000 characters", - ) - reward: USDCent = Field( - description="The amount of money the Requester will pay a Worker for successfully completing the HIT." - ) - - assignment_duration: timedelta = Field( - default=timedelta(minutes=90), - description="The amount of time, in seconds, that a Worker has to complete " - "the HIT after accepting it.", - ) - auto_approval_delay: timedelta = Field( - default=timedelta(days=7), - description="The number of seconds after an assignment for the HIT has " - "been submitted, after which the assignment is considered " - "Approved automatically unless the Requester explicitly " - "rejects it.", - ) - keywords: str = Field(min_length=3, max_length=999) - - -class HitType(HitTypeCommon): - """ - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/mturk/client/create_hit_type.html - https://docs.aws.amazon.com/AWSMechTurk/latest/AWSMturkAPI/ApiReference_CreateHITTypeOperation.html - """ - - id: PositiveInt | None = Field(default=None) - amt_hit_type_id: AMTBoto3ID | None = Field(default=None) - - # --- GRL Specific --- - min_active: NonNegativeInt = Field(default=0, le=100_000) - - def to_api_request_body(self) -> dict[str, Any]: - return { - "AutoApprovalDelayInSeconds": round( - self.auto_approval_delay.total_seconds() - ), - "AssignmentDurationInSeconds": round( - self.assignment_duration.total_seconds() - ), - "Reward": str(self.reward.to_usd()), - "Title": self.title, - "Keywords": self.keywords, - "Description": self.description, - } - - def to_postgres(self): - d = self.model_dump(mode="json") - d["reward"] = self.reward.to_usd() - return d - - @classmethod - def from_postgres(cls, data: dict[str, Any]) -> Self: - data["reward"] = USDCent(round(data["reward"] * 100)) - return cls.model_validate(data) - - def generate_hit_amt_request(self, question: HitQuestion) -> dict[str, Any]: - d = {} - d["HITTypeId"] = self.amt_hit_type_id - d["MaxAssignments"] = 1 - d["LifetimeInSeconds"] = round(timedelta(days=14).total_seconds()) - d["Question"] = question.xml - d["UniqueRequestToken"] = uuid4().hex - return d - - -class Hit(HitTypeCommon): - model_config = ConfigDict( - extra="forbid", - validate_assignment=True, - ) - - id: PositiveInt | None = Field(default=None) - hit_type_id: PositiveInt | None = Field(default=None) - question_id: PositiveInt | None = Field(default=None) - - amt_hit_id: AMTBoto3ID = Field() - amt_hit_type_id: AMTBoto3ID = Field() - amt_group_id: AMTBoto3ID = Field() - hit_question_xml: str = Field() - - status: HitStatus = Field() - review_status: HitReviewStatus = Field() - - # TODO: Check if this is actually ever going to be None. I type fixed it, - # but I don't have anything to suggest it isn't requred. -- Max 2026-02-24 - creation_time: AwareDatetimeISO | None = Field(default=None, description="From aws") - expiration: AwareDatetimeISO | None = Field(default=None) - - # GRL Specific - created_at: AwareDatetimeISO = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When this record was saved in the database", - ) - modified_at: AwareDatetimeISO = Field( - default_factory=lambda: datetime.now(tz=timezone.utc), - description="When this record was last modified", - ) - - # -- Hit specific - - qualification_requirements: list[dict[str, Any]] | None = Field(default=None) - max_assignments: int = Field() - - # # this comes back as expiration. only for the request - # lifetime: timedelta = Field( - # default=timedelta(days=14), - # description="An amount of time, in seconds, after which the HIT is no longer " - # "available for users to accept.", - # ) - assignment_pending_count: NonNegativeInt = Field() - assignment_available_count: NonNegativeInt = Field() - assignment_completed_count: NonNegativeInt = Field() - - @classmethod - def from_amt_create_hit( - cls, data: HITTypeDef, question: HitQuestion, hit_type: HitType - ) -> Self: - assert question.id is not None - assert hit_type.id is not None - assert hit_type.amt_hit_type_id is not None - - h = cls.model_validate( - { - "amt_hit_id": data["HITId"], - "amt_hit_type_id": data["HITTypeId"], - "amt_group_id": data["HITGroupId"], - "status": HitStatus[data["HITStatus"]], - "review_status": HitReviewStatus[data["HITReviewStatus"]], - "creation_time": data["CreationTime"].astimezone(tz=timezone.utc), - "expiration": data["Expiration"].astimezone(tz=timezone.utc), - "hit_question_xml": data["Question"], - "qualification_requirements": data["QualificationRequirements"], - "max_assignments": data["MaxAssignments"], - "assignment_pending_count": data["NumberOfAssignmentsPending"], - "assignment_available_count": data["NumberOfAssignmentsAvailable"], - "assignment_completed_count": data["NumberOfAssignmentsCompleted"], - "description": data["Description"], - "keywords": data["Keywords"], - "reward": USDCent(round(float(data["Reward"]) * 100)), - "title": data["Title"], - "question_id": question.id, - "hit_type_id": hit_type.id, - } - ) - - return h - - @classmethod - def from_amt_get_hit(cls, data: HITTypeDef) -> Self: - h = cls.model_validate( - { - "amt_hit_id": data["HITId"], - "amt_hit_type_id": data["HITTypeId"], - "amt_group_id": data["HITGroupId"], - "status": HitStatus[data["HITStatus"]], - "review_status": HitReviewStatus[data["HITReviewStatus"]], - "creation_time": data["CreationTime"].astimezone(tz=timezone.utc), - "expiration": data["Expiration"].astimezone(tz=timezone.utc), - "hit_question_xml": data["Question"], - "qualification_requirements": data["QualificationRequirements"], - "max_assignments": data["MaxAssignments"], - "assignment_pending_count": data["NumberOfAssignmentsPending"], - "assignment_available_count": data["NumberOfAssignmentsAvailable"], - "assignment_completed_count": data["NumberOfAssignmentsCompleted"], - "description": data["Description"], - "keywords": data["Keywords"], - "reward": USDCent(round(float(data["Reward"]) * 100)), - "title": data["Title"], - "question_id": None, - "hit_type_id": None, - } - ) - return h - - def to_postgres(self): - d = self.model_dump(mode="json") - d["reward"] = self.reward.to_usd() - return d - - @classmethod - def from_postgres(cls, data: dict[str, Any]) -> Self: - data["reward"] = USDCent(round(data["reward"] * 100)) - return cls.model_validate(data) - - @property - def hit_question(self) -> HitQuestion: - root = ElementTree.fromstring(self.hit_question_xml) - - ns = { - "mt": "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd" - } - res = {} - - lookup_table = {"ExternalURL": "url", "FrameHeight": "height"} - for a in root.findall("mt:*", ns): - key = lookup_table[a.tag.split("}")[1]] - val = a.text - res[key] = val - - return HitQuestion.model_validate(res, from_attributes=True) diff --git a/jb/models/response.py b/jb/models/response.py deleted file mode 100644 index 22985af..0000000 --- a/jb/models/response.py +++ /dev/null @@ -1,21 +0,0 @@ -from pydantic import BaseModel, ConfigDict, Field - - -class HTTPHeaders(BaseModel): - request_id: str = Field(alias="x-amzn-requestid", min_length=36, max_length=36) - content_type: str = Field(alias="content-type", min_length=26, max_length=26) - # 'content-length': '1255', - content_length: str = Field(alias="content-length", min_length=2) - # 'Mon, 15 Jan 2024 23:40:32 GMT' - date: str = Field() - - connection: str | None = Field(default=None) # 'close' - - -class ResponseMetadata(BaseModel): - model_config = ConfigDict(extra="forbid", validate_assignment=True) - - request_id: str = Field(alias="RequestId", min_length=36, max_length=36) - status_code: int = Field(alias="HTTPStatusCode", ge=200, le=599) - headers: HTTPHeaders = Field(alias="HTTPHeaders") - retry_attempts: int = Field(alias="RetryAttempts", ge=0) |
