From e278f65adc0cec52f354ab1dffaaf3507e49245b Mon Sep 17 00:00:00 2001 From: stuppie Date: Wed, 9 Sep 2026 13:53:57 -0600 Subject: get user txs incuding UserLedgerTransactionAttemptCredit, query by wallet uuid or name --- .../managers/thl/ledger_manager/ledger.py | 18 ++++++++++ .../managers/thl/ledger_manager/thl_ledger.py | 37 +++++++++++++++++-- generalresearch/models/thl/ledger.py | 42 ++++++++++++++++------ 3 files changed, 83 insertions(+), 14 deletions(-) diff --git a/generalresearch/managers/thl/ledger_manager/ledger.py b/generalresearch/managers/thl/ledger_manager/ledger.py index 3a02cdf..d027bd5 100644 --- a/generalresearch/managers/thl/ledger_manager/ledger.py +++ b/generalresearch/managers/thl/ledger_manager/ledger.py @@ -869,6 +869,24 @@ class LedgerAccountManager(LedgerManagerBasePostgres): ) return res[0] if len(res) == 1 else None + def get_account_by_uuid( + self, account_uuid: UUIDStr, raise_on_error: bool = True + ) -> LedgerAccount | None: + check_valid_uuid(account_uuid) + res = self.pg_config.execute_sql_query( + query=""" + SELECT + uuid, display_name, qualified_name, account_type, + normal_balance, reference_type, reference_uuid, currency + FROM ledger_account + WHERE uuid = %s; + """, + params=[account_uuid], + ) + if raise_on_error and len(res) != 1: + raise LedgerAccountDoesntExistError + return LedgerAccount.model_validate(res[0]) if len(res) == 1 else None + def get_account_many_( self, qualified_names: list[str], raise_on_error: bool = True ) -> list[dict[str, Any]]: diff --git a/generalresearch/managers/thl/ledger_manager/thl_ledger.py b/generalresearch/managers/thl/ledger_manager/thl_ledger.py index 2fef50f..5210189 100644 --- a/generalresearch/managers/thl/ledger_manager/thl_ledger.py +++ b/generalresearch/managers/thl/ledger_manager/thl_ledger.py @@ -2234,12 +2234,39 @@ class ThlLedgerManager(LedgerManager): page: int = 1, size: int = 50, order_by: str | None = "created,tag", + account_uuid: UUIDStr | None = None, + qualified_name: str | None = None, ) -> UserLedgerTransactions: + """Return user-facing transactions for one of the user's ledger accounts. + + The account can be selected by UUID or qualified name. If neither is + supplied, this defaults to the user's primary wallet. + """ + assert user.product_id is not None user.prefetch_product(self.pg_config) - user_account = self.get_account_or_create_user_wallet(user) + assert account_uuid is None or qualified_name is None, ( + "Pass either account_uuid or qualified_name, not both" + ) + + if account_uuid is not None: + user_account = self.get_account_by_uuid(account_uuid) + elif qualified_name is not None: + user_account = self.get_account(qualified_name) + else: + user_account = self.get_account_or_create_user_wallet(user) + + assert user_account is not None + assert user_account.reference_type == "user", ( + "Account must be a user ledger account" + ) + assert user_account.reference_uuid == user.uuid, "Account must be owned by user" exclude_txs_before = None - if user.product_id == JAMES_BILLINGS_BPID: + if ( + user.product_id == JAMES_BILLINGS_BPID + and user_account.account_type == AccountType.USER_WALLET + and user_account.currency == self.currency.value + ): time_start = ( max([JAMES_BILLINGS_TX_CUTOFF, time_start]) if time_start is not None @@ -2263,7 +2290,11 @@ class ThlLedgerManager(LedgerManager): user_account=user_account, txs=txs, product_id=user.product_id, - payout_format=user.product.payout_config.payout_format, + payout_format=( + user.product.payout_config.payout_format + if user_account.currency == self.currency.value + else None + ), summary=summary, page=page, size=size, diff --git a/generalresearch/models/thl/ledger.py b/generalresearch/models/thl/ledger.py index 613d16e..4160e98 100644 --- a/generalresearch/models/thl/ledger.py +++ b/generalresearch/models/thl/ledger.py @@ -352,7 +352,7 @@ class LedgerTransaction(BaseModel): return d def to_user_tx( - self, user_account: LedgerAccount, product_id: str, payout_format: str + self, user_account: LedgerAccount, product_id: str, payout_format: str | None ): from generalresearch.models.thl.wallet.definitions import PayoutType @@ -386,10 +386,15 @@ class LedgerTransaction(BaseModel): raise ValueError(payout_type) return UserLedgerTransactionUserPayout.model_validate(d) elif d["tx_type"] == TransactionType.BP_PAYMENT.value: + assert len(debits) == 1 or len(credits) == 1 + d["amount"] = credits[0].amount if credits else debits[0].amount * -1 + d["tsid"] = self.metadata.get("thl_session") + return UserLedgerTransactionTaskComplete.model_validate(d) + elif d["tx_type"] == TransactionType.USER_ATTEMPT_CREDIT.value: assert len(credits) == 1 d["amount"] = credits[0].amount d["tsid"] = self.metadata.get("thl_session") - return UserLedgerTransactionTaskComplete.model_validate(d) + return UserLedgerTransactionAttemptCredit.model_validate(d) elif d["tx_type"] == TransactionType.USER_BONUS.value: assert len(credits) == 1 d["amount"] = credits[0].amount @@ -546,11 +551,6 @@ class UserLedgerTransactionTaskComplete(UserLedgerTransaction): def create_url(self, product_id: str) -> str | None: return f"https://fsb.generalresearch.com/{product_id}/status/{self.tsid}/" - @model_validator(mode="after") - def validate_amount(self): - assert self.amount >= 0, f"UserLedgerTransactionTaskComplete: {self.amount=}" - return self - class UserLedgerTransactionTaskAdjustment(UserLedgerTransaction): model_config = ConfigDict( @@ -578,11 +578,28 @@ class UserLedgerTransactionTaskAdjustment(UserLedgerTransaction): return f"https://fsb.generalresearch.com/{product_id}/status/{self.tsid}/" +class UserLedgerTransactionAttemptCredit(UserLedgerTransaction): + tx_type: Literal[TransactionType.USER_ATTEMPT_CREDIT] = Field( + default=TransactionType.USER_ATTEMPT_CREDIT + ) + description: str = Field(default="Attempt Credit", max_length=255) + tsid: UUIDStr + + def create_url(self, product_id: str) -> str | None: + return f"https://fsb.generalresearch.com/{product_id}/status/{self.tsid}/" + + @model_validator(mode="after") + def validate_amount(self): + assert self.amount > 0, f"UserLedgerTransactionAttemptCredit: {self.amount=}" + return self + + UserLedgerTransactionType = Annotated[ UserLedgerTransactionUserPayout | UserLedgerTransactionUserBonus | UserLedgerTransactionTaskAdjustment - | UserLedgerTransactionTaskComplete, + | UserLedgerTransactionTaskComplete + | UserLedgerTransactionAttemptCredit, Field(discriminator="tx_type"), ] @@ -611,6 +628,9 @@ class UserLedgerTransactionTypesSummary(BaseModel): user_bonus: UserLedgerTransactionTypeSummary = Field( default_factory=UserLedgerTransactionTypeSummary ) + user_attempt_credit: UserLedgerTransactionTypeSummary = Field( + default_factory=UserLedgerTransactionTypeSummary + ) user_payout_request: UserLedgerTransactionTypeSummary = Field( default_factory=UserLedgerTransactionTypeSummary ) @@ -623,8 +643,8 @@ class UserLedgerTransactions(Page): transactions: list[UserLedgerTransactionType] = Field(default_factory=list) # The summary is w.r.t an optional time-filter. The transactions are - # paginated so the counts won't necesarily match. In other words, the - # summary is across all transaction in all pages, not this the transactions + # paginated so the counts won't necessarily match. In other words, the + # summary is across all transactions in all pages, not this the transactions # in this page. summary: UserLedgerTransactionTypesSummary = Field() @@ -634,7 +654,7 @@ class UserLedgerTransactions(Page): user_account: LedgerAccount, txs: list[LedgerTransaction], product_id: str, - payout_format: str, + payout_format: PayoutFormatType | None, summary: UserLedgerTransactionTypesSummary, page: int, size: int, -- cgit v1.2.3 From 0144d37b3e9c38e5acecc1f9f1750b56b3de9749 Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 11 Sep 2026 12:15:24 -0600 Subject: wip: moving cashout method stuff over from grpc methods --- generalresearch/managers/thl/cashout_method.py | 76 ++++++- generalresearch/managers/thl/wallet/tango.py | 231 +++++++++++---------- generalresearch/managers/thl/wallet/tango_admin.py | 193 +++++++++++++++++ generalresearch/managers/thl/wallet/tango_tasks.py | 127 +++++++++++ .../models/thl/wallet/cashout_method.py | 4 +- generalresearch/models/thl/wallet/definitions.py | 2 + 6 files changed, 512 insertions(+), 121 deletions(-) create mode 100644 generalresearch/managers/thl/wallet/tango_admin.py create mode 100644 generalresearch/managers/thl/wallet/tango_tasks.py diff --git a/generalresearch/managers/thl/cashout_method.py b/generalresearch/managers/thl/cashout_method.py index 910d37b..dad1ef1 100644 --- a/generalresearch/managers/thl/cashout_method.py +++ b/generalresearch/managers/thl/cashout_method.py @@ -8,9 +8,14 @@ from uuid import UUID, uuid4 from pydantic import NonNegativeInt +from generalresearch.currency import USDCent from generalresearch.managers.base import PostgresManager from generalresearch.models.thl.user_ref import UserRef -from generalresearch.models.thl.wallet.definitions import PayoutType +from generalresearch.models.thl.wallet.definitions import ( + SUPPORTED_CURRENCIES, + Currency, + PayoutType, +) if TYPE_CHECKING: from generalresearch.models.thl.user import User @@ -86,7 +91,7 @@ class CashoutMethodManager(PostgresManager): name="Cash in Mail", description="USPS delivery of cash", id=uuid4().hex, - currency="USD", + currency=Currency.USD, image_url="https://www.shutterstock.com/shutterstock/photos/2175413929/display_1500/stock-vector-opened" "-envelope-with-money-dollar-bills-salary-earning-and-savings-concept-d-web-vector-2175413929.jpg", min_value=500, # $5.00 @@ -131,7 +136,7 @@ class CashoutMethodManager(PostgresManager): name="PayPal", description="Cashout via PayPal", id=uuid4().hex, - currency="USD", + currency=Currency.USD, image_url="https://cdn.mmfwcl.com/images/brands/p439786-1200w-326ppi.png", min_value=100, # $1.00 max_value=25_000, # $250.00 @@ -280,8 +285,70 @@ class CashoutMethodManager(PostgresManager): if (x.type == PayoutType.AMT and product.user_wallet_config.amt) or (x.type != PayoutType.AMT) ] + # This is b/c there might be some Tango cards in here for currency we don't support + cms = [ + x + for x in cms + if x.original_currency is None + or x.original_currency in SUPPORTED_CURRENCIES + ] return cms + def get_user_cashout_methods( + self, + user: User, + country_iso: str, + usd_exchange_rate: dict[Currency, float], + ): + """ + Get the cashout methods allowed for this user. Does not check financial stuff at all. + This checks the user's country for filtering purposes. + Gets cashoutmethods from accounting_cashoutmethod, along with a consistent UUID. + If BP has a min_cashout, modifies the cashout method's min_value based on the BP's min cashout. + :return: Dict with keys: the UIID, values: Dict with keys: 'id', 'provider', 'ext_id', 'data', + where 'data' is provider-specific JSON data containing details about the cashout method. + + The available cashout methods is based off the user's latest IP address -> country, and + also (in the future) a risk assessment (maybe riskier user aren't allowed certain + methods). Also, a user may have different minimums based off "stuff". + # country_iso = get_user_latest_country(user_iph_manager, user) or "us" + If a user requests their cashout methods before they ever enter a survey, we won't have + saved their IP address, and this will fail. I think this call should just default to US. + assert country_iso, "unknown country from IP address" + """ + + # BP can set their own min in USD or equivalent + user.prefetch_product(pg_config=self.pg_config) + product = user.product + min_value_usd = product.user_wallet_config.min_cashout or 0 + min_value = USDCent(round(min_value_usd * 100)) + + cms = self.get_cashout_methods(user=user) + + for x in cms: + # assets in non-USD need to be converted to USD here + if x.original_currency is not None: + if x.original_currency == Currency.USD: + x.usd_exchange_rate = 1.0 + else: + x.usd_exchange_rate = usd_exchange_rate[x.original_currency] + # If the user has foreign cards available, we need to show their min_value in USD + x.min_value_usd = USDCent(round(x.min_value * x.usd_exchange_rate)) + x.max_value_usd = USDCent(round(x.max_value * x.usd_exchange_rate)) + # Adjust min_value for BP + x.min_value = max(x.min_value_usd, min_value) + + cms = [ + cm + for cm in cms + if ( + cm.type == PayoutType.TANGO and country_iso.lower() in cm.data.countries + ) + or cm.type != PayoutType.TANGO + ] + + return {x.id: x for x in cms} + @staticmethod def format_from_db(x: dict[str, Any], user: User | None = None) -> CashoutMethod: x["id"] = UUID(x["id"]).hex @@ -302,5 +369,6 @@ class CashoutMethodManager(PostgresManager): x["data"]["type"] = x["type"] if user and x["type"] in {PayoutType.PAYPAL, PayoutType.CASH_IN_MAIL}: x["user"] = user.to_user_ref() - + x["original_currency"] = x.get("currency") or Currency.USD + x["currency"] = Currency.USD return CashoutMethod.model_validate(x) diff --git a/generalresearch/managers/thl/wallet/tango.py b/generalresearch/managers/thl/wallet/tango.py index 038f67f..aa780c3 100644 --- a/generalresearch/managers/thl/wallet/tango.py +++ b/generalresearch/managers/thl/wallet/tango.py @@ -1,127 +1,128 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any - -from generalresearch.config import ( - is_debug, +from datetime import timedelta +from decimal import Decimal +from threading import Lock +from typing import Any + +from cachetools import TTLCache, cachedmethod + +from generalresearch.currency import USDCent +from generalresearch.managers.thl.cashout_method import CashoutMethodManager +from generalresearch.managers.thl.tango_api import TangoClient +from generalresearch.models.thl.wallet.definitions import ( + CURRENCY_FORMATTER, + Currency, + PayoutType, ) -from generalresearch.managers.thl.tango_api import TangoOrderRequest -from generalresearch.models.thl.definitions import PayoutStatus - -if TYPE_CHECKING: - from generalresearch.managers.thl.ledger_manager.thl_ledger import ( - ThlLedgerManager, - ) - from generalresearch.managers.thl.payout import PayoutEventManager - from generalresearch.managers.thl.tango_api import TangoClient - from generalresearch.models.thl.payout import UserPayoutEvent - from generalresearch.models.thl.user import User - - -def complete_tango_order( - user: User, - payout_event: UserPayoutEvent, - payout_event_manager: PayoutEventManager, - ledger_manager: ThlLedgerManager, - tango_client: TangoClient, -): - """ - We approved the Tango card redemption. Actually request the card. - (Note: we're skipping the PENDING -> APPROVED -> COMPLETE order for tango. - When a tango request gets APPROVED, we COMPLETE it (or FAIL!) in the - same step) - """ - assert payout_event.status in { - PayoutStatus.PENDING, - PayoutStatus.FAILED, - }, "attempting to manage payout that is not pending (or you can retry a failed order)" - request = payout_event.request_data - ref_id = request["externalRefID"] - # amount_usd = Decimal(payout_event.request_data["amount_usd"]) - # Note: tango uses the ref_id to uniquify orders, so locking is not actually needed as long - # as the ref_id is the same. - try: - order = create_tango_order( - request_data=payout_event.request_data, - ref_id=ref_id, - tango_client=tango_client, +class TangoManager: + def __init__( + self, + tango_client: TangoClient, + tango_account_id: str, + tango_customer_id: str, + cashout_method_manager: CashoutMethodManager, + ) -> None: + self.tango_client = tango_client + self.tango_account_id = tango_account_id + self.tango_customer_id = tango_customer_id + self.cashout_method_manager = cashout_method_manager + self.supported_currencies = frozenset({currency.value for currency in Currency}) + self._exchange_rate_cache = TTLCache( + maxsize=1, ttl=timedelta(minutes=30).total_seconds() ) + self._name_cache = TTLCache( + maxsize=1000, ttl=timedelta(minutes=30).total_seconds() + ) + self._exchange_rate_lock = Lock() + self._name_lock = Lock() - except AssertionError: - # todo: its possible the order went through, but something else was wrong - # we should try to retrieve the order by its ref_id and confirm it really - # failed... - payout_event_manager.update(payout_event, status=PayoutStatus.FAILED) - return payout_event - - # update TangoPayoutEvent with the order data - payout_event_manager.update( - payout_event, - status=order["status"], - ext_ref_id=order["referenceOrderID"], - order_data=order, + @cachedmethod( + cache=lambda self: self._exchange_rate_cache, + lock=lambda self: self._exchange_rate_lock, ) - - ledger_manager.create_tx_user_payout_complete(user, payout_event=payout_event) - - return payout_event - - -def create_tango_order( - request_data: dict[str, Any], ref_id: str, tango_client: TangoClient -) -> dict[str, Any]: - """ - Create a tango gift card order. - Throws exception if anything is not right. - - https://integration-www.tangocard.com/raas_api_console/v2/ - - https://www.apimatic.io/apidocs/tangocard/v/2_3_4#/python - - :param utid: Card identifier - :param amount: requested card value in USD - :param ref_id: TangoPayoutEvent.uuid - :return: - """ - # make sure we don't create more than one tango order for a single PayoutEvent - assert tango_client.get_order_if_exists(ref_id) is None - amount = request_data["amount"] - request_data.pop("amount_usd", None) - request_data.pop("description", None) - - if is_debug(): + def get_exchange_rates(self) -> dict[Currency, float]: + """Return supported foreign-currency-to-USD Tango exchange rates.""" + rates = self.tango_client.get_exchange_rates()["exchangeRates"] return { - "status": "COMPLETE", - "referenceOrderID": "test", - "reward": { - "credentials": { - "Security Code": "XXXX-XXXX", - "Redemption URL": "https://codes.rewardcodes.com/r2/1/XXXX", - }, - "credentialList": [ - { - "type": "text", - "label": "Security Code", - "value": "XXXX-XXXX", - }, - { - "type": "url", - "label": "Redemption URL", - "value": "https://codes.rewardcodes.com/r2/1/XXXX", - }, - ], - "redemptionInstructions": "do your thang fam", - }, + Currency(rate["baseCurrency"]): rate["baseFx"] + for rate in rates + if rate["rewardCurrency"] == "USD" + and rate["baseCurrency"] in self.supported_currencies } - request = TangoOrderRequest.model_validate(request_data) - order = tango_client.create_order(request) - - amount_f: float = float(amount) - assert order["status"] == "COMPLETE" - assert abs(order["amountCharged"]["total"] - amount_f) < 0.0200001 - assert order["amountCharged"]["currencyCode"] == "USD" - if order["denomination"]["currencyCode"] == "USD": - assert order["denomination"]["value"] == amount_f + def get_order_detail(self, tango_order_id: str) -> dict[str, Any]: + return self.tango_client.get_order(tango_order_id) + + def make_request( + self, amount_usd: Decimal, cashout_method: Any, external_ref_id: str + ) -> dict[str, Any]: + """Build the data needed to place a Tango order.""" + assert type(amount_usd) is Decimal + utid = cashout_method.data.utid + amount: Decimal | float = amount_usd + currency = cashout_method.original_currency + currency_code = getattr(currency, "value", currency) + if currency_code and currency_code != "USD": + amount = round(float(amount) / self.get_exchange_rates()[currency_code], 2) + return { + "accountIdentifier": self.tango_account_id, + "customerIdentifier": self.tango_customer_id, + "utid": utid, + "amount": str(amount), + "amount_usd": str(amount_usd), + "campaign": "300large", + "sendEmail": False, + "externalRefID": external_ref_id, + "description": self.get_name(utid), + } - return order + @cachedmethod( + cache=lambda self: self._name_cache, + lock=lambda self: self._name_lock, + ) + def get_name(self, utid: str) -> str: + methods = self.cashout_method_manager.filter( + ext_id=utid, payout_types=[PayoutType.TANGO], is_live=None + ) + cashout_method = next(iter(methods), None) + return cashout_method.name if cashout_method else "Tango Gift Card" + + def get_expected_redemption_value( + self, cashout_method_id: str, amount: USDCent + ) -> tuple[int, Currency]: + """ + Convert USD cents to a Tango card's smallest currency unit. + for e.g.: A user wants a variable value CAD visa card. He redeems $10 USD (amount=1000), + this function returns the amount that will be redeemed in CAD. + :param cashout_method_id: ID of tango card. expected to be non USD. If USD, just returns the amount. + :param amount: amount the user is redeeming from their wallet in USD integer cents. + :return: amount that will be redeemed through tango on their foreign card, in the card's + (specified by the UTID) foreign currency (in integer units of the lowest denomination) + # example CAD visa: U121653 + """ + assert type(amount) is USDCent + res = self.cashout_method_manager.filter(uuid=cashout_method_id, is_live=True) + if not res: + raise ValueError(f"no cashout method found for {cashout_method_id!r}") + cashout_method = res[0] + assert cashout_method.type == PayoutType.TANGO + assert cashout_method.original_currency is not None + + currency = cashout_method.original_currency + if currency == Currency.USD: + return int(amount), currency + + foreign_amount = round(int(amount) / self.get_exchange_rates()[currency]) + return foreign_amount, currency + + def format_currency(self, amount: int, currency: Currency): + return CURRENCY_FORMATTER[currency](amount) + + def clear_caches(self) -> None: + with self._exchange_rate_lock: + self._exchange_rate_cache.clear() + with self._name_lock: + self._name_cache.clear() diff --git a/generalresearch/managers/thl/wallet/tango_admin.py b/generalresearch/managers/thl/wallet/tango_admin.py new file mode 100644 index 0000000..579b21f --- /dev/null +++ b/generalresearch/managers/thl/wallet/tango_admin.py @@ -0,0 +1,193 @@ +import re + +from generalresearch.managers.base import PostgresManager +from generalresearch.managers.thl.tango_api import TangoClient +from generalresearch.models.thl.wallet.definitions import ( + CURRENCY_MAX_VALUE, +) + + +class TangoAdmin(PostgresManager): + def __init__(self, *, tango_client: TangoClient, tango_account_id: str, **kwargs): + super().__init__(**kwargs) + self.tango_client = tango_client + self.tango_account_id = tango_account_id + + def get_account_balance(self) -> float: + account = self.tango_client.get_account( + account_identifier=self.tango_account_id + ) + return account["currentBalance"] + + @staticmethod + def is_available_tango_card(item: dict): + """ + We're supporting cards that meet the following conditions or checks: + - see below + - non USD card support variable value only (todo: handling conversion of fixed amounts) + """ + from generalresearch.models.thl.wallet.definitions import SUPPORTED_CURRENCIES + + reward_types = ["cash equivalent", "gift card"] # these are the only options + value_types = ["VARIABLE_VALUE", "FIXED_VALUE"] # these are the only options + ignore_cards = { + "U557938", # Southwest® Gift Card $25, doesn't work...? + } + return bool( + item["rewardType"] in reward_types + and item["status"] == "active" + and item["valueType"] in value_types + and item["utid"] not in ignore_cards + and item["currencyCode"] in SUPPORTED_CURRENCIES + and ( + item["currencyCode"] == "USD" + or item["currencyCode"] != "USD" + and item["valueType"] == "VARIABLE_VALUE" + ) + ) + + @staticmethod + def format_tango_card(item: dict, brand: dict): + """ + For non-USD, we support variable value only. We aren't doing any currency conversion here, for + them. To handle the $250USD max value, I'm setting a hard limit in each currency. These are + for our supported, stable currencies only, so this shouldn't be a problem.... + """ + tango_card_info = { + # 'currency': 'USD', + "provider": "TANGO", + "cashout_method_type": "GIFT_CARD", + } + brand_dict = { + "image_url": brand["imageUrls"]["1200w-326ppi"], + "description": brand["description"], + "disclaimer": brand["disclaimer"], + "terms": brand["terms"], + } + value_type_map = {"VARIABLE_VALUE": "variable", "FIXED_VALUE": "fixed"} + item["rewardName"] = re.sub(r"\s+", " ", item["rewardName"]) + # if this is a fixed value, non USD card, we need to save the original value before + # we convert it to USD, b/c we need that value to make the request + if item["valueType"] == "FIXED_VALUE" and item["currencyCode"] != "USD": + raise NotImplementedError("FIXED VALUE non-USD not supported") + if item["valueType"] == "VARIABLE_VALUE" and item["currencyCode"] != "USD": + pass + if item["valueType"] == "FIXED_VALUE": + item["minValue"] = item["faceValue"] + item["maxValue"] = item["faceValue"] + + card = { + "name": item["rewardName"], + "utid": item["utid"], + "type": item["rewardType"], + # 'min_value_usd': round(item.min_value, 2), + # 'max_value_usd': round(min(item.max_value, MAX_VALUE[item.currency_code]), 2), + "min_value": round(100 * float(item["minValue"])), + "max_value": min( + round(100 * float(item["maxValue"])), + CURRENCY_MAX_VALUE[item["currencyCode"]] * 100, + ), + "countries": item["countries"], + "value_type": value_type_map[item["valueType"]], + # This is the currency the reward is based in. We'll need this to convert the request + # to USD when the user requests it. + "currency": item["currencyCode"], + } + card.update(brand_dict) + card.update(tango_card_info) + return card + + def get_tango_cards(self) -> list: + """ + This is called to populate the accounting_cashoutmethod table ONLY. It is not LIVE. + For non-USD cards, the $ fields are in foreign currency because this is stored in + our DB for lookup purposes and has to be converted when requested. + """ + # usd_exchange_rate = get_tango_exchange_rates() + cards: list[dict] = [] + catalog = self.tango_client.get_catalog() + brands = [b for b in catalog["brands"] if b["status"] == "active"] + for brand in brands: + for item in brand["items"]: + if self.is_available_tango_card(item): + print( + item["currencyCode"], + item["rewardName"], + item["valueType"], + ( + f"{item['minValue']}-{item['maxValue']}" + if "minValue" in item + else item["faceValue"] + ), + ) + cards.append(self.format_tango_card(item, brand)) + return cards + +# +# def update_tango_cashout_method_db(): +# # Synchronizes Tango cashout methods/gift cards with the accounting_cashoutmethod table. +# # Should only need to be run sporadically. +# # Note: Nothing calls this at the moment. It was run once manually. +# now = datetime.utcnow() +# tango_cards = get_tango_cards() +# provider = "TANGO" +# +# db_res = THL_WEB_RW.execute_sql_query( +# f""" +# SELECT `id`, `provider`, `ext_id` FROM `{THL_WEB_RW.db}`.`accounting_cashoutmethod`""" +# ) +# existing_ids = {(provider, x["ext_id"]): x["id"] for x in db_res} +# +# fields = ["id", "last_updated", "is_live", "provider", "ext_id", "name", "data"] +# values = [ +# [ +# existing_ids.get((provider, x["utid"]), uuid.uuid4().hex), +# now, +# True, +# provider, +# x["utid"], +# x["name"], +# json.dumps(x), +# ] +# for x in tango_cards +# ] +# +# THL_WEB_RW.bulk_update("accounting_cashoutmethod", fields, values) +# +# +# def check_and_refill_balance() -> None: +# # dont do this more often than once a week!!! +# REFILL_AMOUNT = 5000 +# LOW_BALANCE = 1000 +# balance = get_account_balance() +# +# if balance < LOW_BALANCE: +# SC.chat_postMessage( +# channel=settings.SLACK_CHANNEL, +# text=f"Refilling Tango account. current balance: {balance}", +# ) +# depost_request = { +# "accountIdentifier": settings.TANGO_ACCOUNT_ID, +# "amount": REFILL_AMOUNT, +# "creditCardToken": settings.TANGO_CREDITCARD_TOKEN, +# "customerIdentifier": settings.TANGO_CUSTOMER_ID, +# } +# response = None +# try: +# response = api_client.fund.add_funds(depost_request) +# assert response.status == "SUCCESS" +# response = json.loads(APIHelper.json_serialize(response)) +# except Exception as e: +# SC.chat_postMessage( +# channel=settings.SLACK_CHANNEL, +# text=f"Tango refill error: {e}. Response: {response}", +# ) +# SC.chat_postMessage( +# channel=settings.SLACK_CHANNEL, +# text=f"Tango refill success!: Response: {response}", +# ) +# elif balance < (LOW_BALANCE * 1.5): +# SC.chat_postMessage( +# channel=settings.SLACK_CHANNEL, +# text=f"Tango balance low. current balance: {balance}", +# ) diff --git a/generalresearch/managers/thl/wallet/tango_tasks.py b/generalresearch/managers/thl/wallet/tango_tasks.py new file mode 100644 index 0000000..038f67f --- /dev/null +++ b/generalresearch/managers/thl/wallet/tango_tasks.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from generalresearch.config import ( + is_debug, +) +from generalresearch.managers.thl.tango_api import TangoOrderRequest +from generalresearch.models.thl.definitions import PayoutStatus + +if TYPE_CHECKING: + from generalresearch.managers.thl.ledger_manager.thl_ledger import ( + ThlLedgerManager, + ) + from generalresearch.managers.thl.payout import PayoutEventManager + from generalresearch.managers.thl.tango_api import TangoClient + from generalresearch.models.thl.payout import UserPayoutEvent + from generalresearch.models.thl.user import User + + +def complete_tango_order( + user: User, + payout_event: UserPayoutEvent, + payout_event_manager: PayoutEventManager, + ledger_manager: ThlLedgerManager, + tango_client: TangoClient, +): + """ + We approved the Tango card redemption. Actually request the card. + + (Note: we're skipping the PENDING -> APPROVED -> COMPLETE order for tango. + When a tango request gets APPROVED, we COMPLETE it (or FAIL!) in the + same step) + """ + assert payout_event.status in { + PayoutStatus.PENDING, + PayoutStatus.FAILED, + }, "attempting to manage payout that is not pending (or you can retry a failed order)" + request = payout_event.request_data + ref_id = request["externalRefID"] + # amount_usd = Decimal(payout_event.request_data["amount_usd"]) + + # Note: tango uses the ref_id to uniquify orders, so locking is not actually needed as long + # as the ref_id is the same. + try: + order = create_tango_order( + request_data=payout_event.request_data, + ref_id=ref_id, + tango_client=tango_client, + ) + + except AssertionError: + # todo: its possible the order went through, but something else was wrong + # we should try to retrieve the order by its ref_id and confirm it really + # failed... + payout_event_manager.update(payout_event, status=PayoutStatus.FAILED) + return payout_event + + # update TangoPayoutEvent with the order data + payout_event_manager.update( + payout_event, + status=order["status"], + ext_ref_id=order["referenceOrderID"], + order_data=order, + ) + + ledger_manager.create_tx_user_payout_complete(user, payout_event=payout_event) + + return payout_event + + +def create_tango_order( + request_data: dict[str, Any], ref_id: str, tango_client: TangoClient +) -> dict[str, Any]: + """ + Create a tango gift card order. + Throws exception if anything is not right. + - https://integration-www.tangocard.com/raas_api_console/v2/ + - https://www.apimatic.io/apidocs/tangocard/v/2_3_4#/python + + :param utid: Card identifier + :param amount: requested card value in USD + :param ref_id: TangoPayoutEvent.uuid + :return: + """ + # make sure we don't create more than one tango order for a single PayoutEvent + assert tango_client.get_order_if_exists(ref_id) is None + amount = request_data["amount"] + request_data.pop("amount_usd", None) + request_data.pop("description", None) + + if is_debug(): + return { + "status": "COMPLETE", + "referenceOrderID": "test", + "reward": { + "credentials": { + "Security Code": "XXXX-XXXX", + "Redemption URL": "https://codes.rewardcodes.com/r2/1/XXXX", + }, + "credentialList": [ + { + "type": "text", + "label": "Security Code", + "value": "XXXX-XXXX", + }, + { + "type": "url", + "label": "Redemption URL", + "value": "https://codes.rewardcodes.com/r2/1/XXXX", + }, + ], + "redemptionInstructions": "do your thang fam", + }, + } + + request = TangoOrderRequest.model_validate(request_data) + order = tango_client.create_order(request) + + amount_f: float = float(amount) + assert order["status"] == "COMPLETE" + assert abs(order["amountCharged"]["total"] - amount_f) < 0.0200001 + assert order["amountCharged"]["currencyCode"] == "USD" + if order["denomination"]["currencyCode"] == "USD": + assert order["denomination"]["value"] == amount_f + + return order diff --git a/generalresearch/models/thl/wallet/cashout_method.py b/generalresearch/models/thl/wallet/cashout_method.py index 6ad0bf2..1e96818 100644 --- a/generalresearch/models/thl/wallet/cashout_method.py +++ b/generalresearch/models/thl/wallet/cashout_method.py @@ -58,8 +58,8 @@ class CashoutMethodBase(BaseModel): id: UUIDStr = Field(description="Unique ID for this cashout method") - currency: Literal["USD"] = Field( - default="USD", + currency: Literal[Currency.USD] = Field( + default=Currency.USD, description="The currency of the cashout. Only USD is supported.", ) original_currency: Currency | None = Field( diff --git a/generalresearch/models/thl/wallet/definitions.py b/generalresearch/models/thl/wallet/definitions.py index 2d1eb8d..4d9729b 100644 --- a/generalresearch/models/thl/wallet/definitions.py +++ b/generalresearch/models/thl/wallet/definitions.py @@ -57,6 +57,8 @@ class Currency(StrEnum): # Mexican Peso MXN = "MXN" +SUPPORTED_CURRENCIES = {currency.value for currency in Currency} + CURRENCY_FORMATTER = { "USD": lambda x: f"${x / 100:,.2f}", -- cgit v1.2.3 From 5a5474d3f3dda3030ae888c9716efdc5dc4b8931 Mon Sep 17 00:00:00 2001 From: stuppie Date: Thu, 17 Sep 2026 13:17:47 -0600 Subject: rip out maxmind ipinfo geoip managers. update with grip client-based calls. plugging in user_request_redeem and all the associated checks. wip... --- generalresearch/managers/thl/ipinfo.py | 748 +-------------------- generalresearch/managers/thl/payout.py | 223 ------ generalresearch/managers/thl/userhealth.py | 158 ++--- generalresearch/managers/thl/wallet/__init__.py | 4 +- .../managers/thl/wallet/cashout_tasks.py | 3 + generalresearch/managers/thl/wallet/tango.py | 41 +- generalresearch/managers/thl/wallet/user_payout.py | 392 +++++++++++ generalresearch/models/thl/ipinfo.py | 133 +--- generalresearch/models/thl/maxmind/__init__.py | 0 generalresearch/models/thl/maxmind/definitions.py | 22 - generalresearch/models/thl/user_iphistory.py | 85 +-- .../models/thl/wallet/cashout_method.py | 18 + 12 files changed, 560 insertions(+), 1267 deletions(-) create mode 100644 generalresearch/managers/thl/wallet/cashout_tasks.py create mode 100644 generalresearch/managers/thl/wallet/user_payout.py delete mode 100644 generalresearch/models/thl/maxmind/__init__.py delete mode 100644 generalresearch/models/thl/maxmind/definitions.py diff --git a/generalresearch/managers/thl/ipinfo.py b/generalresearch/managers/thl/ipinfo.py index 93914c3..575738b 100644 --- a/generalresearch/managers/thl/ipinfo.py +++ b/generalresearch/managers/thl/ipinfo.py @@ -1,736 +1,38 @@ -from __future__ import annotations - -import ipaddress from collections.abc import Collection -from decimal import Decimal -from typing import TYPE_CHECKING - -import faker -from grip_client.enums import AccessType -from more_itertools import chunked -from psycopg import Cursor -from pydantic import PositiveInt - -from generalresearch.managers.base import ( - PostgresManager, - PostgresManagerWithRedis, -) -from generalresearch.models.thl.ipinfo import ( - GeoIPInformation, - IPGeoname, - IPInformation, - normalize_ip, -) - -if TYPE_CHECKING: - from generalresearch.models.custom_types import ( - CountryISOLike, - IPvAnyAddressStr, - ) - from generalresearch.pg_helper import PostgresConfig - -fake = faker.Faker() - - -class IPGeonameManager(PostgresManager): - - def create_basic( - self, - geoname_id: PositiveInt, - is_in_european_union: bool, - country_iso: CountryISOLike, - country_name: str, - continent_code: str, - continent_name: str, - ) -> IPGeoname: - instance = IPGeoname.model_validate( - { - "geoname_id": geoname_id, - "country_iso": country_iso, - "is_in_european_union": is_in_european_union, - "country_name": country_name, - "continent_code": continent_code, - "continent_name": continent_name, - } - ) - self.pg_config.execute_write( - query=""" - INSERT INTO thl_geoname ( - geoname_id, country_iso, is_in_european_union, country_name, - continent_code, continent_name, updated - ) - VALUES ( - %(geoname_id)s, %(country_iso)s, %(is_in_european_union)s, - %(country_name)s, %(continent_code)s, %(continent_name)s, - %(updated)s - ) - ON CONFLICT (geoname_id) DO NOTHING; - """, - params=instance.model_dump(mode="json"), - ) - return instance - - def create_or_update(self, ipgeo: IPGeoname): - keys = list(ipgeo.model_fields.keys()) - data = ipgeo.model_dump_mysql() - - keys_str = ", ".join(keys) - values_str = ", ".join([f"%({k})s" for k in keys]) - update_cols = set(keys) - {"geoname_id"} - update_str = ", ".join([f"{k} = EXCLUDED.{k}" for k in update_cols]) - - query = f""" - INSERT INTO thl_geoname ({keys_str}) - VALUES ({values_str}) - ON CONFLICT (geoname_id) - DO UPDATE SET {update_str} - """ - self.pg_config.execute_write(query=query, params=data) - - def create( - self, - geoname_id: PositiveInt, - continent_code: str, - continent_name: str, - country_iso: str | None, - country_name: str | None = None, - subdivision_1_iso: str | None = None, - subdivision_1_name: str | None = None, - subdivision_2_iso: str | None = None, - subdivision_2_name: str | None = None, - city_name: str | None = None, - metro_code: int | None = None, - time_zone: str | None = None, - is_in_european_union: bool | None = None, - ) -> IPGeoname: - - instance = IPGeoname.model_validate( - { - "geoname_id": geoname_id, - "continent_code": continent_code, - "continent_name": continent_name, - "country_iso": country_iso, - "country_name": country_name, - "subdivision_1_iso": subdivision_1_iso, - "subdivision_1_name": subdivision_1_name, - "subdivision_2_iso": subdivision_2_iso, - "subdivision_2_name": subdivision_2_name, - "city_name": city_name, - "metro_code": metro_code, - "time_zone": time_zone, - "is_in_european_union": is_in_european_union, - } - ) - - self.pg_config.execute_write( - query=""" - INSERT INTO thl_geoname - ( geoname_id, continent_code, continent_name, - country_iso, country_name, - subdivision_1_iso, subdivision_1_name, - subdivision_2_iso, subdivision_2_name, - city_name, metro_code, time_zone, is_in_european_union, - updated - ) - VALUES ( - %(geoname_id)s, %(continent_code)s, %(continent_name)s, - %(country_iso)s, %(country_name)s, - %(subdivision_1_iso)s, %(subdivision_1_name)s, - %(subdivision_2_iso)s, %(subdivision_2_name)s, - %(city_name)s, %(metro_code)s, %(time_zone)s, - %(is_in_european_union)s, %(updated)s - ) - ON CONFLICT (geoname_id) DO NOTHING; - """, - params=instance.model_dump(mode="json"), - ) - - return instance - - def get_by_id(self, geoname_id: PositiveInt) -> IPGeoname: - return self.fetch_geoname_ids(filter_ids=[geoname_id])[0] - - def fetch_geoname_ids( - self, - filter_ids: list[PositiveInt], - ) -> list[IPGeoname]: - - if len(filter_ids) == 0: - return [] - - with self.pg_config.make_connection() as conn, conn.cursor() as c: - res = [] - for chunk in chunked(filter_ids, 500): - res.extend( - self.fetch_geoname_ids_( - c=c, - filter_ids=chunk, - ) - ) - return res - - def fetch_geoname_ids_( - self, - c: Cursor, - filter_ids: list[PositiveInt], - ) -> list[IPGeoname]: - - assert len(filter_ids) <= 500, "chunk me" - - c.execute( - query=""" - SELECT g.geoname_id, - g.continent_code, g.continent_name, - g.country_iso, g.country_name, - g.subdivision_1_iso, g.subdivision_1_name, - g.subdivision_2_iso, g.subdivision_2_name, - g.city_name, g.metro_code, - g.time_zone, g.is_in_european_union, - g.updated - FROM thl_geoname AS g - WHERE g.geoname_id = ANY(%s); - """, - params=[filter_ids], - ) - return [IPGeoname.from_mysql(i) for i in c.fetchall()] - - -class IPInformationManager(PostgresManager): - - def create_basic( - self, - ip: IPvAnyAddressStr, - geoname_id: PositiveInt, - country_iso: str, - registered_country_iso: str, - ) -> IPInformation: - instance = IPInformation.model_validate( - { - "ip": ip, - "geoname_id": geoname_id, - "country_iso": country_iso, - "registered_country_iso": registered_country_iso, - } - ) - instance.normalize_ip() - self.pg_config.execute_write( - query=""" - INSERT INTO thl_ipinformation - (ip, country_iso, registered_country_iso, geoname_id, updated) - VALUES (%(ip)s, %(country_iso)s, %(registered_country_iso)s, - %(geoname_id)s, %(updated)s) - ON CONFLICT (ip) DO NOTHING; - """, - params=instance.model_dump(mode="json"), - ) - return instance - - def create( - self, - ip: IPvAnyAddressStr, - geoname_id: PositiveInt | None = None, - country_iso: str | None = None, - registered_country_iso: str | None = None, - is_anonymous: bool | None = None, - is_anonymous_vpn: bool | None = None, - is_hosting_provider: bool | None = None, - is_public_proxy: bool | None = None, - is_tor_exit_node: bool | None = None, - is_residential_proxy: bool | None = None, - autonomous_system_number: PositiveInt | None = None, - autonomous_system_organization: str | None = None, - domain: str | None = None, - isp: str | None = None, - mobile_country_code: str | None = None, - mobile_network_code: str | None = None, - network: str | None = None, - organization: str | None = None, - static_ip_score: float | None = None, - user_type: AccessType | None = None, - postal_code: str | None = None, - latitude: Decimal | None = None, - longitude: Decimal | None = None, - accuracy_radius: int | None = None, - ) -> IPInformation: - - instance = IPInformation.model_validate( - { - "ip": ip, - "geoname_id": geoname_id, - "country_iso": country_iso, - "registered_country_iso": registered_country_iso, - "is_anonymous": is_anonymous, - "is_anonymous_vpn": is_anonymous_vpn, - "is_hosting_provider": is_hosting_provider, - "is_public_proxy": is_public_proxy, - "is_tor_exit_node": is_tor_exit_node, - "is_residential_proxy": is_residential_proxy, - "autonomous_system_number": autonomous_system_number, - "autonomous_system_organization": autonomous_system_organization, - "domain": domain, - "isp": isp, - "mobile_country_code": mobile_country_code, - "mobile_network_code": mobile_network_code, - "network": network, - "organization": organization, - "static_ip_score": static_ip_score, - "user_type": user_type, - "postal_code": postal_code, - "latitude": latitude, - "longitude": longitude, - "accuracy_radius": accuracy_radius, - } - ) - instance.normalize_ip() - - self.pg_config.execute_write( - query=""" - INSERT INTO thl_ipinformation - ( ip, geoname_id, - country_iso, registered_country_iso, - is_anonymous, is_anonymous_vpn, - is_hosting_provider, is_public_proxy, - is_tor_exit_node, is_residential_proxy, - autonomous_system_number, autonomous_system_organization, - domain, isp, - mobile_country_code, mobile_network_code, - network, organization, static_ip_score, - user_type, postal_code, latitude, longitude, - accuracy_radius, - updated - ) - VALUES ( - %(ip)s, %(geoname_id)s, - %(country_iso)s, %(registered_country_iso)s, - %(is_anonymous)s, %(is_anonymous_vpn)s, - %(is_hosting_provider)s, %(is_public_proxy)s, - %(is_tor_exit_node)s, %(is_residential_proxy)s, - %(autonomous_system_number)s, %(autonomous_system_organization)s, - %(domain)s, %(isp)s, - %(mobile_country_code)s, %(mobile_network_code)s, - %(network)s, %(organization)s, %(static_ip_score)s, - %(user_type)s, %(postal_code)s, %(latitude)s, %(longitude)s, - %(accuracy_radius)s, - %(updated)s - ) - ON CONFLICT (ip) DO NOTHING; - """, - params=instance.model_dump(mode="json"), - ) - - return instance - - def create_or_update(self, ipinfo: IPInformation): - ipinfo.normalize_ip() - keys = [key for key, field in ipinfo.model_fields.items() if not field.exclude] - data = ipinfo.model_dump_mysql() - - keys_str = ", ".join(keys) - values_str = ", ".join([f"%({k})s" for k in keys]) - update_cols = set(keys) - {"ip"} - update_str = ", ".join([f"{k} = EXCLUDED.{k}" for k in update_cols]) - - query = f""" - INSERT INTO thl_ipinformation ({keys_str}) - VALUES ({values_str}) - ON CONFLICT (ip) DO UPDATE - SET {update_str} - """ - self.pg_config.execute_write(query, params=data) - - def get_ip_info(self, ip: IPvAnyAddressStr) -> IPInformation | None: - res = self.fetch_ip_information(filter_ips=[ip]) - if len(res) != 1: - return None - - return res[0] - - def fetch_ip_information( - self, - filter_ips: list[IPvAnyAddressStr], - ) -> list[IPInformation]: - - if len(filter_ips) == 0: - return [] - - with self.pg_config.make_connection() as conn, conn.cursor() as c: - res = [] - for chunk in chunked(filter_ips, 500): - res.extend( - self.fetch_ip_information_( - c=c, - filter_ips=chunk, - ) - ) - return res - def fetch_ip_information_( - self, - c: Cursor, - filter_ips: list[IPvAnyAddressStr], - ) -> list[IPInformation]: - """ - IPs are converted to normalized form (/64 network exploded) for DB lookup, - and are then matched back to the original queried form for return. - e.g. '2600:1700:ece0:9410:055d:faf3:c15d:06e4' is passed in, - it gets converted to '2600:1700:ece0:9410:0000:0000:0000:0000' for db lookup, - the result gets - """ +from grip_client import GRIPMMDBClient - assert len(filter_ips) <= 500, "chunk me" - normalized_ip_lookup = {ip: normalize_ip(ip)[0] for ip in filter_ips} - normalized_ips = set(normalized_ip_lookup.values()) +from generalresearch.managers.base import Manager +from generalresearch.models.custom_types import IPvAnyAddressStr +from generalresearch.models.thl.ipinfo import GeoIPInformation - c.execute( - query=""" - SELECT i.ip, i.geoname_id, - i.country_iso, i.registered_country_iso, - i.is_anonymous, i.is_anonymous_vpn, i.is_hosting_provider, - i.is_public_proxy, i.is_tor_exit_node, i.is_residential_proxy, - i.autonomous_system_number, i.autonomous_system_organization, - i.domain, i.isp, - i.mobile_country_code, i.mobile_network_code, - i.network, i.organization, - i.static_ip_score, i.user_type, i.postal_code, - i.latitude, i.longitude, i.accuracy_radius, - i.updated - FROM thl_ipinformation AS i - WHERE i.ip = ANY(%s) - """, - params=[list(normalized_ips)], - ) +class GeoIpInfoManager(Manager): + def __init__(self, grip_token: str, **kwargs) -> None: + self.grip_mmdb = GRIPMMDBClient(token=grip_token, **kwargs) - return [IPInformation.from_mysql(i) for i in c.fetchall()] - - @staticmethod - def monitor_ipinformation(pg_config: PostgresConfig) -> None: - """Continually check our IPInformation table to ensure location information - is being saved properly""" - - # Check the percentage of IPs that don't have a country_iso in the past 12 hours - # Split query is 1000x faster - query = """ - SELECT COUNT(*) AS numerator - FROM thl_ipinformation - WHERE updated >= NOW() - INTERVAL '12 hours' - AND country_iso IS NULL; - """ - # numerator = list(pg_config.execute_sql_query(query=query))[0]["numerator"] - - query = """ - SELECT COUNT(1) AS denominator - FROM thl_ipinformation - WHERE updated >= NOW() - INTERVAL '12 hours' - """ - denominator = next(iter(pg_config.execute_sql_query(query=query)))[ - "denominator" - ] - if denominator == 0: - pass - - # percent_empty = numerator / (denominator or 1) - # TODO: Post to telegraf / grafana - - -class GeoIpInfoManager(PostgresManagerWithRedis): - - def get(self, ip_address: IPvAnyAddressStr) -> GeoIPInformation | None: - res = self.get_cache(ip_address) - if res: - return res - res = self.get_mysql_if_exists(ip_address) - if res: - self.set_cache(res) - return res + def get(self, ip_address: IPvAnyAddressStr) -> GeoIPInformation: + d = self.get_multi({ip_address}) + return d[ip_address] def get_multi( self, ip_addresses: Collection[IPvAnyAddressStr] - ) -> dict[IPvAnyAddressStr, GeoIPInformation | None]: + ) -> dict[IPvAnyAddressStr, GeoIPInformation]: if not ip_addresses: return {} - # To deploy this, we still have (for the next 28 days) users who's - # ipv6 history was looked up and saved using the full /128. We need - # to pull those if the /64 doesn't exist. - # See notes in get_cache_multi & get_mysql_multi - res = self.get_cache_multi(ip_addresses=ip_addresses) - missing_ips = {k for k, v in res.items() if v is None and k in ip_addresses} - res_mysql = self.get_mysql_multi(ips=missing_ips) - self.set_cache_multi({k: v for k, v in res_mysql.items() if v}) - res.update(res_mysql) - return res - - def set_cache_multi( - self, ipinfo_map: dict[IPvAnyAddressStr, GeoIPInformation] - ) -> None: - """Set multiple GeoIPInformation objects in Redis in one call.""" - if not ipinfo_map: - return - - pipe = self.redis_client.pipeline(transaction=False) - expire_seconds = 3 * 24 * 3600 - for ip, ipinfo in ipinfo_map.items(): - pipe.set( - self.get_cache_key(ip), - ipinfo.model_dump_json(), - ex=expire_seconds, - ) - pipe.execute() - - @staticmethod - def compress_ip(ip: str) -> str: - """ - To support looking up an ip in the db before we switched - to using the exploded form. (remove me 28 days after 2025-11-15) - """ - addr = ipaddress.ip_address(ip) - if addr.version == 4: - return str(addr) - return addr.compressed - - def get_cache_multi( - self, ip_addresses: Collection[IPvAnyAddressStr] - ) -> dict[IPvAnyAddressStr, GeoIPInformation | None]: - """Get multiple GeoIPInformation objects from Redis in one call. - - Returns a dict mapping IP address -> GeoIPInformation (or None if not in cache). - """ - if not ip_addresses: - return {} - # We must do it like this b/c we could have multiple /128 ips that normalize - # to the same normalized /64 ip, and we don't want to "loose" them. - ip_norm_lookup = {ip: normalize_ip(ip) for ip in ip_addresses} - normalized_ips = {v[0] for v in ip_norm_lookup.values()} - # also lookup exact matches (can remove this 28 days from 2025-11-15) - normalized_ips.update(ip_addresses) - # also lookup compressed form ... (remove me also) - normalized_ips.update({self.compress_ip(ip) for ip in ip_addresses}) - - keys = [self.get_cache_key(ip) for ip in normalized_ips] - res = self.redis_client.mget(keys) - res = [GeoIPInformation.model_validate_json(raw) for raw in res if raw] - gs = {x.ip: x for x in res} - - res2 = {} - for ip, (normalized_ip, lookup_prefix) in ip_norm_lookup.items(): - if normalized_ip not in gs: - # try the non-normalized (remove me also 28 days from 2025-11-15) - if ip in gs: - res2[ip] = gs[ip].model_copy() - continue - res2[ip] = None - continue - g = gs[normalized_ip] - g.ip = ip - g.lookup_prefix = lookup_prefix - res2[g.ip] = g.model_copy() - return res2 - - def get_cache_key(self, ip_address: IPvAnyAddressStr) -> str: - return self.cache_prefix + f"thl:GeoIpInfoManager:{ip_address}" - - def clear_cache(self, ip_address: IPvAnyAddressStr) -> None: - # typically for testing - self.redis_client.delete(self.get_cache_key(ip_address=ip_address)) - - def set_cache(self, ipinfo: GeoIPInformation): - ipinfo = ipinfo.model_copy() - ipinfo.normalize_ip() - data = ipinfo.model_dump_json() - return self.redis_client.set( - self.get_cache_key(ip_address=ipinfo.ip), data, ex=3 * 24 * 3600 - ) - - def get_cache(self, ip_address: IPvAnyAddressStr) -> GeoIPInformation | None: - normalized_ip, lookup_prefix = normalize_ip(ip_address) - res: str = self.get_cache_raw(normalized_ip) - if not res: - return None - g = GeoIPInformation.model_validate_json(res) - g.ip = ip_address - g.lookup_prefix = lookup_prefix - return g - - def get_cache_raw(self, ip_address: IPvAnyAddressStr) -> str: - return self.redis_client.get(self.get_cache_key(ip_address=ip_address)) - - def get_mysql_if_exists(self, ip_address: IPvAnyAddressStr): - try: - return self.get_mysql(ip_address=ip_address) - except AssertionError: - return None - - def get_mysql_raw(self, ip_address: IPvAnyAddressStr): - query = """ - SELECT - geo.geoname_id, - geo.continent_name, - LOWER(geo.continent_code) AS continent_code, - geo.country_name, - LOWER(geo.country_iso) AS geo_country_iso, - geo.subdivision_1_iso, - geo.subdivision_1_name, - geo.subdivision_2_iso, - geo.subdivision_2_name, - geo.city_name, - geo.metro_code, - geo.time_zone, - geo.is_in_european_union, - LOWER(ipinfo.country_iso) AS country_iso, - ipinfo.registered_country_iso, - ipinfo.is_anonymous, - ipinfo.is_anonymous_vpn, - ipinfo.is_hosting_provider, - ipinfo.is_public_proxy, - ipinfo.is_tor_exit_node, - ipinfo.is_residential_proxy, - ipinfo.autonomous_system_number, - ipinfo.autonomous_system_organization, - ipinfo.domain, - ipinfo.isp, - ipinfo.mobile_country_code, - ipinfo.mobile_network_code, - ipinfo.network, - ipinfo.organization, - ipinfo.static_ip_score, - ipinfo.user_type, - ipinfo.postal_code, - CAST(ipinfo.latitude AS float) AS latitude, - CAST(ipinfo.longitude AS float) AS longitude, - ipinfo.accuracy_radius, - ipinfo.ip, - ipinfo.updated - FROM thl_ipinformation AS ipinfo - LEFT JOIN thl_geoname AS geo - ON ipinfo.geoname_id = geo.geoname_id - WHERE ipinfo.ip = %s - """ - res = self.pg_config.execute_sql_query(query=query, params=[ip_address]) - assert len(res) == 1 - d = res[0] - if d.get("geo_country_iso") and (d["geo_country_iso"] != d["country_iso"]): - raise ValueError( - f'mismatch between ipinfo country {d["country_iso"]} and geoname country {d["geo_country_iso"]}' + ips = set(ip_addresses) + d = {ip: self.grip_mmdb.lookup(ip) for ip in ips} + d = { + ip: GeoIPInformation( + ip=ip, + country_iso=v.country.country_iso.lower() + if v.country.country_iso + else None, + is_anonymous=v.anonymous.is_anonymous, + autonomous_system_number=v.asn.asn, + autonomous_system_organization=v.asn.network_operator, + access_type=v.asn.access_type, ) + for ip, v in d.items() + } return d - - def get_mysql(self, ip_address: IPvAnyAddressStr): - normalized_ip, lookup_prefix = normalize_ip(ip_address) - d = self.get_mysql_raw(normalized_ip) - g = GeoIPInformation.from_mysql(d) - g.ip = ip_address - g.lookup_prefix = lookup_prefix - return g - - def recreate_cache(self, ip_address: IPvAnyAddressStr) -> GeoIPInformation: - res = self.get_mysql(ip_address) - self.set_cache(res) - return res - - def get_mysql_multi( - self, - ips: Collection[IPvAnyAddressStr], - ) -> dict[IPvAnyAddressStr, GeoIPInformation | None]: - - if len(ips) == 0: - return {} - - with self.pg_config.make_connection() as conn, conn.cursor() as c: - res = {} - for chunk in chunked(ips, 500): - inner = self.get_mysql_multi_chunk( - c=c, - ips=chunk, - ) - res.update(inner) - return res - - def get_mysql_multi_chunk( - self, - c: Cursor, - ips: list[IPvAnyAddressStr], - ) -> dict[IPvAnyAddressStr, GeoIPInformation | None]: - - assert len(ips) <= 500, "chunk me" - - # We must do it like this b/c we could have multiple /128 ips that normalize - # to the same normalized /64 ip, and we don't want to "loose" them. - ip_norm_lookup = {ip: normalize_ip(ip) for ip in ips} - normalized_ips = {v[0] for v in ip_norm_lookup.values()} - # also lookup exact matches (can remove this 28 days from 2025-11-15) - normalized_ips.update(ips) - # also lookup compressed form ... (remove me also) - normalized_ips.update({self.compress_ip(ip) for ip in ips}) - - c.execute( - query=""" - SELECT - geo.geoname_id, - geo.continent_name, - LOWER(geo.continent_code) AS continent_code, - geo.country_name, - LOWER(geo.country_iso) AS geo_country_iso, - geo.subdivision_1_iso, - geo.subdivision_1_name, - geo.subdivision_2_iso, - geo.subdivision_2_name, - geo.city_name, - geo.metro_code, - geo.time_zone, - geo.is_in_european_union, - LOWER(ipinfo.country_iso) AS country_iso, - ipinfo.registered_country_iso, - ipinfo.is_anonymous, - ipinfo.is_anonymous_vpn, - ipinfo.is_hosting_provider, - ipinfo.is_public_proxy, - ipinfo.is_tor_exit_node, - ipinfo.is_residential_proxy, - ipinfo.autonomous_system_number, - ipinfo.autonomous_system_organization, - ipinfo.domain, - ipinfo.isp, - ipinfo.mobile_country_code, - ipinfo.mobile_network_code, - ipinfo.network, - ipinfo.organization, - ipinfo.static_ip_score, - ipinfo.user_type, - ipinfo.postal_code, - CAST(ipinfo.latitude AS float) AS latitude, - CAST(ipinfo.longitude AS float) AS longitude, - ipinfo.accuracy_radius, - ipinfo.ip, - ipinfo.updated - FROM thl_ipinformation AS ipinfo - LEFT JOIN thl_geoname AS geo - ON ipinfo.geoname_id = geo.geoname_id - WHERE ipinfo.ip = ANY(%s) - """, - params=[list(normalized_ips)], - ) - - res = c.fetchall() - for d in res: - if d.get("geo_country_iso") and (d["geo_country_iso"] != d["country_iso"]): - raise ValueError( - f'mismatch between ipinfo country {d["country_iso"]} and geoname country {d["geo_country_iso"]}' - ) - - gs = [GeoIPInformation.from_mysql(i) for i in res] - gs = {g.ip: g for g in gs} - res2 = {} - - for ip, (normalized_ip, lookup_prefix) in ip_norm_lookup.items(): - if normalized_ip not in gs: - # also can remove 28 days after 2025-11-15 - if ip in gs: - res2[ip] = gs[ip].model_copy() - continue - res2[ip] = None - continue - g = gs[normalized_ip] - g.ip = ip - g.lookup_prefix = lookup_prefix - res2[g.ip] = g.model_copy() - return res2 diff --git a/generalresearch/managers/thl/payout.py b/generalresearch/managers/thl/payout.py index 59dc2e9..4014cf8 100644 --- a/generalresearch/managers/thl/payout.py +++ b/generalresearch/managers/thl/payout.py @@ -34,10 +34,6 @@ from generalresearch.models.thl.payout import ( PayoutEvent, UserPayoutEvent, ) -from generalresearch.models.thl.wallet.cashout_method import ( - CashMailOrderData, - CashoutRequestInfo, -) from generalresearch.models.thl.wallet.definitions import PayoutType if TYPE_CHECKING: @@ -105,225 +101,6 @@ class PayoutEventManager(PostgresManagerWithRedis): conn.commit() -class UserPayoutEventManager(PayoutEventManager): - def get_by_uuid(self, pe_uuid: UUIDStr) -> UserPayoutEvent: - - res = self.pg_config.execute_sql_query( - query=""" - SELECT ep.uuid, - ep.debit_account_uuid, - ep.cashout_method_uuid, - ep.created, ep.amount, ep.status, ep.ext_ref_id, ep.payout_type, - ep.request_data::jsonb, - ep.order_data::jsonb, - -- User Payout specific - ac.name as description, - la.reference_type as account_reference_type, - la.reference_uuid as account_reference_uuid - FROM event_payout AS ep - LEFT JOIN accounting_cashoutmethod AS ac - ON ep.cashout_method_uuid = ac.id - LEFT JOIN ledger_account AS la - ON la.uuid = ep.debit_account_uuid - WHERE ep.uuid = %s - """, - params=[pe_uuid], - ) - - assert len(res) == 1, f"{pe_uuid} expected 1 result, got {len(res)}" - - d = res[0] - pe = UserPayoutEvent.model_validate(d) - if pe.order_data and pe.payout_type == PayoutType.CASH_IN_MAIL: - pe.order_data = CashMailOrderData.model_validate(pe.order_data) - - return pe - - def get_payout_detail(self, pe_uuid: UUIDStr) -> CashoutRequestInfo: - # This gets the payout event, and then extracts information for - # the purposes of returning to the user. - pe = self.get_by_uuid(pe_uuid=pe_uuid) - - transaction_info = {} - order: dict[str, Any] = pe.order_data - if pe.payout_type == PayoutType.TANGO and pe.status == PayoutStatus.COMPLETE: - reward = order["reward"] - if "credentialList" in reward: - reward["credential_list"] = reward.pop("credentialList") - if "redemptionInstructions" in reward: - reward["redemption_instructions"] = reward.pop("redemptionInstructions") - transaction_info = order["reward"] - elif pe.payout_type == PayoutType.PAYPAL and pe.status == PayoutStatus.COMPLETE: - info = {"transaction_id": order["transaction_id"]} - transaction_info = info - elif ( - pe.payout_type == PayoutType.CASH_IN_MAIL - and pe.status == PayoutStatus.COMPLETE - ): - transaction_info = pe.order_data.model_dump(mode="json") - - return CashoutRequestInfo( - id=pe_uuid, - status=pe.status, - description=pe.description, - transaction_info=transaction_info, - message="", - ) - - def filter_by( - self, - reference_uuid: str | None = None, - debit_account_uuids: Collection[UUIDStr] | None = None, - amount: int | None = None, - created: datetime | None = None, - created_after: datetime | None = None, - product_ids: str | None = None, - bp_user_ids: Collection[str] | None = None, - cashout_method_uuids: Collection[UUIDStr] | None = None, - cashout_types: Collection[PayoutType] | None = None, - statuses: Collection[PayoutStatus] | None = None, - ) -> list[UserPayoutEvent]: - """Try to retrieve payout events by the product_id/user_uuid, amount, - and optionally timestamp. - - WARNING: This is only on the "payout events" table and nothing to - do with the Ledger itself. Therefore, the product_ids query - doesn't return Brokerage Product Payouts (the ACH or Wire events - to Suppliers) as part of the query. - - *** IT IS ONLY FOR USER PAYOUTS *** - - Note: what used to be in thl-grpcs "ListCashoutRequests" calling - "list_cashout_requests" was merged into this. - """ - args = [] - filters = [] - - if reference_uuid: - # This could be a product_id or a user_uuid - filters.append("la.reference_uuid = %s") - args.append(reference_uuid) - - if debit_account_uuids: - # Or we could use the bp_wallet or user_wallet's account uuid - # instead of looking up by the product/user - filters.append("ep.debit_account_uuid = ANY(%s)") - args.append(debit_account_uuids) - if amount: - filters.append("ep.amount = %s") - args.append(amount) - if created: - filters.append("ep.created = %s") - args.append(created.replace(tzinfo=None)) - if created_after: - filters.append("ep.created >= %s") - args.append(created_after.replace(tzinfo=None)) - if product_ids: - filters.append("product_id = ANY(%s)") - args.append(product_ids) - if bp_user_ids: - filters.append("product_user_id = ANY(%s)") - args.append(bp_user_ids) - if cashout_method_uuids: - filters.append("cashout_method_uuid = ANY(%s)") - args.append(cashout_method_uuids) - if cashout_types: - filters.append("payout_type = ANY(%s)") - args.append([x.value for x in cashout_types]) - if statuses: - filters.append("status = ANY(%s)") - args.append([x.value for x in statuses]) - - assert len(filters) > 0, "must pass at least 1 filter" - filter_str = "WHERE " + " AND ".join(filters) - - res = self.pg_config.execute_sql_query( - query=f""" - SELECT - ep.uuid, ep.debit_account_uuid, - ep.created, ep.amount, ep.status, - ep.ext_ref_id, ep.payout_type, ep.cashout_method_uuid, - ep.order_data::jsonb, - ep.request_data::jsonb, - ac.name as description, - la.reference_type as account_reference_type, - la.reference_uuid as account_reference_uuid - FROM event_payout AS ep - LEFT JOIN accounting_cashoutmethod AS ac - ON ep.cashout_method_uuid = ac.id - LEFT JOIN ledger_account AS la - ON la.uuid = ep.debit_account_uuid - LEFT JOIN thl_user u - ON la.reference_uuid = u.uuid - {filter_str} - """, - params=args, - ) - - pes = [] - for d in res: - pes.append(UserPayoutEvent.model_validate(d)) - return pes - - def create( - self, - debit_account_uuid: UUIDStr, - cashout_method_uuid: UUIDStr, - payout_type: PayoutType, - amount: PositiveInt, - # --- Optional: Default / Default Factory --- - uuid: UUIDStr | None = None, - status: PayoutStatus | None = None, - created: AwareDatetimeISO | None = None, - request_data: dict[str, Any] | None = None, - # --- Optional: None --- - account_reference_type: str | None = None, - account_reference_uuid: UUIDStr | None = None, - description: str | None = None, - ext_ref_id: str | None = None, - order_data: dict[str, Any] | CashMailOrderData | None = None, - ) -> UserPayoutEvent: - - payout_event = UserPayoutEvent( - uuid=uuid or uuid4().hex, - debit_account_uuid=debit_account_uuid, - account_reference_type=account_reference_type, - account_reference_uuid=account_reference_uuid, - cashout_method_uuid=cashout_method_uuid, - description=description, - created=created or datetime.now(tz=UTC), - amount=amount, - status=status or PayoutStatus.PENDING, - ext_ref_id=ext_ref_id, - payout_type=payout_type, - request_data=request_data or {}, - order_data=order_data, - ) - d = payout_event.model_dump_postgres() - - with self.pg_config.make_connection() as conn: - with conn.cursor() as c: - c.execute( - query=""" - INSERT INTO event_payout ( - uuid, debit_account_uuid, created, - cashout_method_uuid, amount, status, - ext_ref_id, payout_type, order_data, - request_data - ) VALUES ( - %(uuid)s, %(debit_account_uuid)s, %(created)s, - %(cashout_method_uuid)s, %(amount)s, %(status)s, - %(ext_ref_id)s, %(payout_type)s, %(order_data)s, - %(request_data)s - ); - """, - params=d, - ) - assert c.rowcount == 1, f"expected 1 row inserted, got {c.rowcount}" - conn.commit() - - return payout_event - class BrokerageProductPayoutEventManager(PayoutEventManager): # This is what makes a PayoutEvent a Brokerage Product Payout diff --git a/generalresearch/managers/thl/userhealth.py b/generalresearch/managers/thl/userhealth.py index b986256..0df67aa 100644 --- a/generalresearch/managers/thl/userhealth.py +++ b/generalresearch/managers/thl/userhealth.py @@ -6,7 +6,6 @@ from datetime import UTC, datetime, timedelta from itertools import zip_longest from typing import TYPE_CHECKING, Any -import faker from pydantic import NonNegativeInt, PositiveInt from generalresearch.decorators import LOG @@ -14,16 +13,17 @@ from generalresearch.managers.base import ( PostgresManager, PostgresManagerWithRedis, ) -from generalresearch.managers.thl.ipinfo import GeoIpInfoManager from generalresearch.models.thl.user_iphistory import ( IPRecord, UserIPHistory, UserIPRecord, ) +from generalresearch.models.thl.user_ref import UserRef from generalresearch.models.thl.userhealth import AuditLog if TYPE_CHECKING: from generalresearch.managers.base import Permission + from generalresearch.managers.thl.ipinfo import GeoIpInfoManager from generalresearch.models.custom_types import IPvAnyAddressStr from generalresearch.models.thl.product import Product from generalresearch.models.thl.user import User @@ -31,46 +31,18 @@ if TYPE_CHECKING: from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig -fake = faker.Faker() - class UserIpHistoryManager(PostgresManagerWithRedis): - def __init__( - self, - pg_config: PostgresConfig, - redis_config: RedisConfig, - permissions: Collection[Permission] | None = None, - cache_prefix: str | None = None, - ): - super().__init__( - pg_config=pg_config, - redis_config=redis_config, - permissions=permissions, - cache_prefix=cache_prefix, - ) - self.geoipinfo_manager = GeoIpInfoManager( - pg_config=pg_config, - redis_config=redis_config, - cache_prefix=cache_prefix, - ) - def get_redis_key(self, user_id: int) -> str: - return f"generalreserach:user-ip-history:{user_id}" + return f"generalreserach:user-ip-history-v2:{user_id}" def get_user_ip_records_sql(self, user_id: int) -> list[UserIPRecord]: # The IP metadata is ONLY for the 'ip', NOT for any forwarded ips. # This might get called immediately after a write, so use the non-rr res = self.pg_config.execute_sql_query( query=""" - SELECT iph.ip, iph.created, iph.user_id, - geo.subdivision_1_iso, - ipinfo.country_iso, - ipinfo.is_anonymous + SELECT iph.ip, iph.created FROM userhealth_iphistory iph - LEFT JOIN thl_ipinformation AS ipinfo - ON iph.ip = ipinfo.ip - LEFT JOIN thl_geoname AS geo - ON ipinfo.geoname_id = geo.geoname_id WHERE iph.user_id = %s AND created > NOW() - INTERVAL '28 days' ORDER BY iph.created DESC @@ -95,76 +67,96 @@ class UserIpHistoryManager(PostgresManagerWithRedis): value = iph.model_dump_json() self.redis_client.set(self.get_redis_key(user_id), value, ex=3 * 24 * 3600) - def recreate_user_ip_history_cache(self, user_id: int) -> None: - self.delete_user_ip_history_cache(user_id=user_id) - records = self.get_user_ip_records_sql(user_id=user_id) + def recreate_user_ip_history_cache(self, user: UserRef) -> None: + self.delete_user_ip_history_cache(user_id=user.user_id) + records = self.get_user_ip_records_sql(user_id=user.user_id) # todo: we may get dns records from somewhere else here ... - iph = UserIPHistory(user_id=user_id, ips=records) - self.set_user_ip_history_cache(user_id=user_id, iph=iph) + iph = UserIPHistory(user=user, ips=records) + self.set_user_ip_history_cache(user_id=user.user_id, iph=iph) + + def get_user_ip_history( + self, user: UserRef | User, geoip_info_manager: GeoIpInfoManager | None = None + ) -> UserIPHistory: + user = user if isinstance(user, UserRef) else user.to_user_ref() - def get_user_ip_history(self, user_id: int) -> UserIPHistory: - assert isinstance(user_id, int) - iph = self.get_user_ip_history_cache(user_id=user_id) + iph = self.get_user_ip_history_cache(user_id=user.user_id) if iph: LOG.debug(f"get_user_ip_history got in cache: {iph.model_dump_json()}") - else: LOG.debug("get_user_ip_history cache not found, using mysql") - records = self.get_user_ip_records_sql(user_id=user_id) + records = self.get_user_ip_records_sql(user_id=user.user_id) # todo: we may get dns records from somewhere else here ... - iph = UserIPHistory(user_id=user_id, ips=records) - self.set_user_ip_history_cache(user_id=user_id, iph=iph) - - iph.enrich_ips(pg_config=self.pg_config, redis_config=self.redis_config) + iph = UserIPHistory(user=user, ips=records) + self.set_user_ip_history_cache(user_id=user.user_id, iph=iph) + if geoip_info_manager: + iph.enrich_ips(geoip_info_manager=geoip_info_manager) return iph - def get_user_latest_ip(self, user: User, exclude_anon: bool = False) -> str | None: - record = self.get_user_latest_ip_record(user=user, exclude_anon=exclude_anon) - if record: - return record.ip - return None - def get_user_latest_ip_record( - self, user: User, exclude_anon: bool = False + self, + user: UserRef | User, + exclude_anon: bool = False, + geoip_info_manager: GeoIpInfoManager | None = None, ) -> UserIPRecord | None: - iphistory = self.get_user_ip_history(user_id=user.user_id) - - if iphistory.ips: - if exclude_anon: - return next( - filter( - lambda x: not x.information.is_anonymous, - iphistory.ips[::-1], - ), - None, - ) - else: - return iphistory.ips[-1] + iphistory = self.get_user_ip_history( + user=user, geoip_info_manager=geoip_info_manager + ) + if not iphistory or not iphistory.ips: + return None + + # Note: UserIPHistory.ips is sorted by 'created DESC' !!! + # This logic was changed at some point? Or the py-utils + # get_user_latest_ip_record was changed. Please be careful here... + if exclude_anon: + assert geoip_info_manager is not None, "Must pass geoip_info_manager" + for ipr in iphistory.ips: + if not ipr.information.is_anonymous: + return ipr + else: + ipr = iphistory.ips[0] + return ipr + def get_user_latest_ip( + self, + user: UserRef | User, + exclude_anon: bool = False, + geoip_info_manager: GeoIpInfoManager | None = None, + ) -> str | None: + record = self.get_user_latest_ip_record( + user=user, exclude_anon=exclude_anon, geoip_info_manager=geoip_info_manager + ) + if record: + return record.ip return None def get_user_latest_country( - self, user: User, exclude_anon: bool = False + self, + user: UserRef | User, + geoip_info_manager: GeoIpInfoManager, + exclude_anon: bool = False, ) -> str | None: """Get the country the user is in, based off their latest ip.""" - ipr = self.get_user_latest_ip_record(user, exclude_anon=exclude_anon) + ipr = self.get_user_latest_ip_record( + user, geoip_info_manager=geoip_info_manager, exclude_anon=exclude_anon + ) # The ipr.information should exist, but it is possible the user has # no IP history at all, so the record is None return ipr.country_iso if ipr is not None else None - def is_user_anonymous(self, user: User) -> bool | None: + def is_user_anonymous( + self, user: UserRef | User, geoip_info_manager: GeoIpInfoManager + ) -> bool | None: # Get the user's latest ip. is it marked as anonymous? - # Note: it is possible we only did a "basic" lookup of this IP so - # we don't know if they are anonymous. Default to False # Return None if the user has no IP history at all - ipr = self.get_user_latest_ip_record(user) + ipr = self.get_user_latest_ip_record( + user, geoip_info_manager=geoip_info_manager + ) if ipr: return ipr.is_anonymous if ipr.is_anonymous is not None else False return None class IPRecordManager(PostgresManagerWithRedis): - def __init__( self, pg_config: PostgresConfig, @@ -187,7 +179,7 @@ class IPRecordManager(PostgresManagerWithRedis): def create_unpack( self, - user_id: PositiveInt, + user: UserRef, ip: IPvAnyAddressStr, forwarded_ips: list[str], ) -> IPRecord: @@ -196,11 +188,11 @@ class IPRecordManager(PostgresManagerWithRedis): padded = list(forwarded_ips) + [None] * (6 - len(forwarded_ips)) - return self.create(user_id, ip, *padded) + return self.create(user, ip, *padded) def create( self, - user_id: PositiveInt, + user: UserRef, ip: IPvAnyAddressStr, forwarded_ip1: IPvAnyAddressStr, forwarded_ip2: IPvAnyAddressStr, @@ -211,7 +203,7 @@ class IPRecordManager(PostgresManagerWithRedis): ) -> IPRecord: data = { - "user_id": user_id, + "user_id": user.user_id, "ip": ipaddress.ip_address(ip).exploded, "created": datetime.now(tz=UTC), } @@ -253,11 +245,12 @@ class IPRecordManager(PostgresManagerWithRedis): """, params=data, ) - self.recreate_user_ip_history_cache(user_id=user_id) + self.recreate_user_ip_history_cache(user=user) return IPRecord.from_mysql(data) - def get_user_latest_ip_record(self, user: User) -> IPRecord | None: + def get_user_latest_ip_record(self, user: UserRef | User) -> IPRecord | None: + user = user if isinstance(user, UserRef) else user.to_user_ref() res = self.filter_ip_records(user_ids=[user.user_id], limit=1) if res: return res[0] @@ -317,14 +310,11 @@ class IPRecordManager(PostgresManagerWithRedis): return [IPRecord.from_mysql(i) for i in res] - def recreate_user_ip_history_cache(self, user_id: int): - return self.user_ip_history_manager.recreate_user_ip_history_cache( - user_id=user_id - ) + def recreate_user_ip_history_cache(self, user: UserRef): + return self.user_ip_history_manager.recreate_user_ip_history_cache(user=user) class AuditLogManager(PostgresManager): - def create( self, user_id: PositiveInt, diff --git a/generalresearch/managers/thl/wallet/__init__.py b/generalresearch/managers/thl/wallet/__init__.py index 1f0c30b..006908b 100644 --- a/generalresearch/managers/thl/wallet/__init__.py +++ b/generalresearch/managers/thl/wallet/__init__.py @@ -1,6 +1,7 @@ from decimal import Decimal from typing import TYPE_CHECKING, Any +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager from generalresearch.managers.thl.wallet.approve import ( approve_amt_cashout, approve_paypal_order, @@ -34,6 +35,7 @@ def manage_pending_cashout( user_ip_history_manager: UserIpHistoryManager, user_manager: UserManager, ledger_manager: ThlLedgerManager, + geoip_info_manager:GeoIpInfoManager, order_data: dict[str, Any] | CashMailOrderData | None = None, tango_client: TangoClient | None = None, ) -> UserPayoutEvent: @@ -63,7 +65,7 @@ def manage_pending_cashout( "manage_pending_cashout called on user without managed wallet" ) assert not user.blocked, "manage_pending_cashout: Blocked user" - assert not user_ip_history_manager.is_user_anonymous(user), ( + assert not user_ip_history_manager.is_user_anonymous(user,geoip_info_manager=geoip_info_manager), ( "manage_pending_cashout: Anonymous user" ) diff --git a/generalresearch/managers/thl/wallet/cashout_tasks.py b/generalresearch/managers/thl/wallet/cashout_tasks.py new file mode 100644 index 0000000..b28b04f --- /dev/null +++ b/generalresearch/managers/thl/wallet/cashout_tasks.py @@ -0,0 +1,3 @@ + + + diff --git a/generalresearch/managers/thl/wallet/tango.py b/generalresearch/managers/thl/wallet/tango.py index aa780c3..fc6763b 100644 --- a/generalresearch/managers/thl/wallet/tango.py +++ b/generalresearch/managers/thl/wallet/tango.py @@ -1,7 +1,6 @@ from __future__ import annotations from datetime import timedelta -from decimal import Decimal from threading import Lock from typing import Any @@ -10,6 +9,11 @@ from cachetools import TTLCache, cachedmethod from generalresearch.currency import USDCent from generalresearch.managers.thl.cashout_method import CashoutMethodManager from generalresearch.managers.thl.tango_api import TangoClient +from generalresearch.models.thl.wallet.cashout_method import ( + TangoCashoutMethodData, + CashoutMethod, + TangoCashoutMethodRequestData, +) from generalresearch.models.thl.wallet.definitions import ( CURRENCY_FORMATTER, Currency, @@ -57,27 +61,26 @@ class TangoManager: return self.tango_client.get_order(tango_order_id) def make_request( - self, amount_usd: Decimal, cashout_method: Any, external_ref_id: str - ) -> dict[str, Any]: + self, amount: USDCent, cashout_method: CashoutMethod, payout_event_id: str + ) -> TangoCashoutMethodRequestData: """Build the data needed to place a Tango order.""" - assert type(amount_usd) is Decimal + assert type(amount) is USDCent utid = cashout_method.data.utid - amount: Decimal | float = amount_usd currency = cashout_method.original_currency - currency_code = getattr(currency, "value", currency) - if currency_code and currency_code != "USD": - amount = round(float(amount) / self.get_exchange_rates()[currency_code], 2) - return { - "accountIdentifier": self.tango_account_id, - "customerIdentifier": self.tango_customer_id, - "utid": utid, - "amount": str(amount), - "amount_usd": str(amount_usd), - "campaign": "300large", - "sendEmail": False, - "externalRefID": external_ref_id, - "description": self.get_name(utid), - } + if currency and currency != Currency.USD: + amount = round(float(amount) / self.get_exchange_rates()[currency], 2) + return TangoCashoutMethodRequestData.model_validate( + { + "accountIdentifier": self.tango_account_id, + "customerIdentifier": self.tango_customer_id, + "utid": utid, + "amount": str(amount), + "campaign": "300large", + "sendEmail": False, + "externalRefID": payout_event_id, + "description": self.get_name(utid), + } + ) @cachedmethod( cache=lambda self: self._name_cache, diff --git a/generalresearch/managers/thl/wallet/user_payout.py b/generalresearch/managers/thl/wallet/user_payout.py new file mode 100644 index 0000000..88505c0 --- /dev/null +++ b/generalresearch/managers/thl/wallet/user_payout.py @@ -0,0 +1,392 @@ +import uuid +from collections.abc import Collection +from datetime import UTC, datetime +from typing import Any +from uuid import uuid4 + +import slack +from pydantic import PositiveInt +from redis import Redis + +from generalresearch.currency import USDCent +from generalresearch.managers.thl.cashout_method import CashoutMethodManager +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager +from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager +from generalresearch.managers.thl.payout import PayoutEventManager +from generalresearch.managers.thl.userhealth import UserIpHistoryManager +from generalresearch.managers.thl.wallet.tango import TangoManager +from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr +from generalresearch.models.thl.definitions import PayoutStatus +from generalresearch.models.thl.payout import UserPayoutEvent +from generalresearch.models.thl.user import User +from generalresearch.models.thl.wallet.cashout_method import ( + CashMailOrderData, + CashoutMethod, + CashoutRequestInfo, + PaypalCashoutMethodRequestData, + TangoCashoutMethodRequestData, + CashMailCashoutMethodRequestData, +) +from generalresearch.models.thl.wallet.definitions import PayoutType + + +class UserPayoutEventManager(PayoutEventManager): + def get_by_uuid(self, pe_uuid: UUIDStr) -> UserPayoutEvent: + + res = self.pg_config.execute_sql_query( + query=""" + SELECT ep.uuid, + ep.debit_account_uuid, + ep.cashout_method_uuid, + ep.created, ep.amount, ep.status, ep.ext_ref_id, ep.payout_type, + ep.request_data::jsonb, + ep.order_data::jsonb, + -- User Payout specific + ac.name as description, + la.reference_type as account_reference_type, + la.reference_uuid as account_reference_uuid + FROM event_payout AS ep + LEFT JOIN accounting_cashoutmethod AS ac + ON ep.cashout_method_uuid = ac.id + LEFT JOIN ledger_account AS la + ON la.uuid = ep.debit_account_uuid + WHERE ep.uuid = %s + """, + params=[pe_uuid], + ) + + assert len(res) == 1, f"{pe_uuid} expected 1 result, got {len(res)}" + + d = res[0] + pe = UserPayoutEvent.model_validate(d) + if pe.order_data and pe.payout_type == PayoutType.CASH_IN_MAIL: + pe.order_data = CashMailOrderData.model_validate(pe.order_data) + + return pe + + def get_payout_detail(self, pe_uuid: UUIDStr) -> CashoutRequestInfo: + # This gets the payout event, and then extracts information for + # the purposes of returning to the user. + pe = self.get_by_uuid(pe_uuid=pe_uuid) + + transaction_info = {} + order: dict[str, Any] = pe.order_data + if pe.payout_type == PayoutType.TANGO and pe.status == PayoutStatus.COMPLETE: + reward = order["reward"] + if "credentialList" in reward: + reward["credential_list"] = reward.pop("credentialList") + if "redemptionInstructions" in reward: + reward["redemption_instructions"] = reward.pop("redemptionInstructions") + transaction_info = order["reward"] + elif pe.payout_type == PayoutType.PAYPAL and pe.status == PayoutStatus.COMPLETE: + info = {"transaction_id": order["transaction_id"]} + transaction_info = info + elif ( + pe.payout_type == PayoutType.CASH_IN_MAIL + and pe.status == PayoutStatus.COMPLETE + ): + transaction_info = pe.order_data.model_dump(mode="json") + + return CashoutRequestInfo( + id=pe_uuid, + status=pe.status, + description=pe.description, + transaction_info=transaction_info, + message="", + ) + + def filter_by( + self, + reference_uuid: str | None = None, + debit_account_uuids: Collection[UUIDStr] | None = None, + amount: int | None = None, + created: datetime | None = None, + created_after: datetime | None = None, + product_ids: str | None = None, + bp_user_ids: Collection[str] | None = None, + cashout_method_uuids: Collection[UUIDStr] | None = None, + cashout_types: Collection[PayoutType] | None = None, + statuses: Collection[PayoutStatus] | None = None, + ) -> list[UserPayoutEvent]: + """Try to retrieve payout events by the product_id/user_uuid, amount, + and optionally timestamp. + + WARNING: This is only on the "payout events" table and nothing to + do with the Ledger itself. Therefore, the product_ids query + doesn't return Brokerage Product Payouts (the ACH or Wire events + to Suppliers) as part of the query. + + *** IT IS ONLY FOR USER PAYOUTS *** + + Note: what used to be in thl-grpcs "ListCashoutRequests" calling + "list_cashout_requests" was merged into this. + """ + args = [] + filters = [] + + if reference_uuid: + # This could be a product_id or a user_uuid + filters.append("la.reference_uuid = %s") + args.append(reference_uuid) + + if debit_account_uuids: + # Or we could use the bp_wallet or user_wallet's account uuid + # instead of looking up by the product/user + filters.append("ep.debit_account_uuid = ANY(%s)") + args.append(debit_account_uuids) + if amount: + filters.append("ep.amount = %s") + args.append(amount) + if created: + filters.append("ep.created = %s") + args.append(created.replace(tzinfo=None)) + if created_after: + filters.append("ep.created >= %s") + args.append(created_after.replace(tzinfo=None)) + if product_ids: + filters.append("product_id = ANY(%s)") + args.append(product_ids) + if bp_user_ids: + filters.append("product_user_id = ANY(%s)") + args.append(bp_user_ids) + if cashout_method_uuids: + filters.append("cashout_method_uuid = ANY(%s)") + args.append(cashout_method_uuids) + if cashout_types: + filters.append("payout_type = ANY(%s)") + args.append([x.value for x in cashout_types]) + if statuses: + filters.append("status = ANY(%s)") + args.append([x.value for x in statuses]) + + assert len(filters) > 0, "must pass at least 1 filter" + filter_str = "WHERE " + " AND ".join(filters) + + res = self.pg_config.execute_sql_query( + query=f""" + SELECT + ep.uuid, ep.debit_account_uuid, + ep.created, ep.amount, ep.status, + ep.ext_ref_id, ep.payout_type, ep.cashout_method_uuid, + ep.order_data::jsonb, + ep.request_data::jsonb, + ac.name as description, + la.reference_type as account_reference_type, + la.reference_uuid as account_reference_uuid + FROM event_payout AS ep + LEFT JOIN accounting_cashoutmethod AS ac + ON ep.cashout_method_uuid = ac.id + LEFT JOIN ledger_account AS la + ON la.uuid = ep.debit_account_uuid + LEFT JOIN thl_user u + ON la.reference_uuid = u.uuid + {filter_str} + """, + params=args, + ) + + pes = [] + for d in res: + pes.append(UserPayoutEvent.model_validate(d)) + return pes + + def create( + self, + debit_account_uuid: UUIDStr, + cashout_method_uuid: UUIDStr, + payout_type: PayoutType, + amount: PositiveInt, + # --- Optional: Default / Default Factory --- + uuid: UUIDStr | None = None, + status: PayoutStatus | None = None, + created: AwareDatetimeISO | None = None, + request_data: dict[str, Any] | None = None, + # --- Optional: None --- + account_reference_type: str | None = None, + account_reference_uuid: UUIDStr | None = None, + description: str | None = None, + ext_ref_id: str | None = None, + order_data: dict[str, Any] | CashMailOrderData | None = None, + ) -> UserPayoutEvent: + + payout_event = UserPayoutEvent( + uuid=uuid or uuid4().hex, + debit_account_uuid=debit_account_uuid, + account_reference_type=account_reference_type, + account_reference_uuid=account_reference_uuid, + cashout_method_uuid=cashout_method_uuid, + description=description, + created=created or datetime.now(tz=UTC), + amount=amount, + status=status or PayoutStatus.PENDING, + ext_ref_id=ext_ref_id, + payout_type=payout_type, + request_data=request_data or {}, + order_data=order_data, + ) + d = payout_event.model_dump_postgres() + + with self.pg_config.make_connection() as conn: + with conn.cursor() as c: + c.execute( + query=""" + INSERT INTO event_payout ( + uuid, debit_account_uuid, created, + cashout_method_uuid, amount, status, + ext_ref_id, payout_type, order_data, + request_data + ) VALUES ( + %(uuid)s, %(debit_account_uuid)s, %(created)s, + %(cashout_method_uuid)s, %(amount)s, %(status)s, + %(ext_ref_id)s, %(payout_type)s, %(order_data)s, + %(request_data)s + ); + """, + params=d, + ) + assert c.rowcount == 1, f"expected 1 row inserted, got {c.rowcount}" + conn.commit() + + return payout_event + + def user_request_redeem( + self, + user: User, + country_iso: str, + cashout_method_id: str, + amount: USDCent, + tango_manager: TangoManager, + cashout_method_manager: CashoutMethodManager, + ledger_manager: ThlLedgerManager, + user_ip_history_manager: UserIpHistoryManager, + geoip_info_manager: GeoIpInfoManager, + redis_client: Redis, + slack_client: slack.WebClient | None = None, + ) -> UserPayoutEvent: + """ + A user has requested to be paid from their wallet balance. + :param user: User + :param cashout_method_id: the cashout method to use + :param amount_usd: In decimal USD. + """ + now = datetime.now(tz=UTC) + user.prefetch_product(pg_config=self.pg_config) + usd_exchange_rates = tango_manager.get_exchange_rates() + cashout_methods = cashout_method_manager.get_user_cashout_methods( + user, country_iso=country_iso, usd_exchange_rate=usd_exchange_rates + ) + assert cashout_method_id in cashout_methods, ( + f"cashout method '{cashout_method_id}' not found" + ) + cashout_method = cashout_methods[cashout_method_id] + + # Make sure the amount requested is allowed for this cashout method + try: + cashout_method.validate_requested_amount(amount=amount) + except ValueError as e: + # Outer function is expecting an assertion error. + raise AssertionError(str(e)) + + """ + These checks are run when a user requests any kind of payout: + - user wallet enabled on BP + - is user blocked + - is user anonymous + - does the user have a redeemable_balance <= amount requested (except for + an AMT assignment request) + """ + assert type(amount) is USDCent + user.prefetch_product(pg_config=self.pg_config) + assert user.product.user_wallet_enabled, ( + "SubmitUserCashout called on user without managed wallet" + ) + assert not user.blocked, "Blocked user requesting redemption" + assert 0 < amount <= 250_00, ( + f"Amount must be between 0 and $250.00. Got {amount.to_usd_str()}" + ) + + product = user.product + banned_countries = user.product.user_health_config.banned_countries + + assert not user_ip_history_manager.is_user_anonymous( + user, geoip_info_manager=geoip_info_manager + ), "Anonymous user requesting redemption" + ipr = user_ip_history_manager.get_user_latest_ip_record( + user, geoip_info_manager=geoip_info_manager + ) + if ipr is not None and ipr.country_iso in banned_countries: + raise AssertionError("Banned country requesting redemption") + + wallet_balance = ledger_manager.get_user_wallet_balance(user) + if product.user_wallet_config.balance_type == "wallet_balance": + redeemable_amount = wallet_balance + elif product.user_wallet_config.balance_type == "redeemable_balance": + redeemable_amount = ledger_manager.get_user_redeemable_wallet_balance(user, wallet_balance) + else: + raise ValueError(f"unexpected balance_type={product.user_wallet_config.balance_type}") + + assert amount <= redeemable_amount, ( + f"User requesting more than their redeemable balance ({amount} > {redeemable_amount}}" + ) + + # Simple dedupe mechanism. Don't allow more than 1 per user_id per minute per cashout_method. + flag_just_set = bool( + redis_client.set( + f"user_request_redeem:{user.user_id}:{cashout_method}", + 1, + nx=True, + ex=60, + ) + ) + assert flag_just_set, "User requesting more than 1 per min per cashout method" + + payout_type = cashout_method.type + + if slack_client: + slack_client.chat_postMessage( + channel="cashouts", + text=f"{user.product.name} {cashout_method.type} " + f"{user.user_id} {amount.to_usd_str()}", + ) + + pe_uuid = uuid.uuid4().hex + + if payout_type == PayoutType.TANGO: + request_data: TangoCashoutMethodRequestData = tango_manager.make_request( + amount, cashout_method, pe_uuid + ) + elif payout_type == PayoutType.PAYPAL: + request_data: PaypalCashoutMethodRequestData = make_request_paypal(cashout_method) + elif payout_type == PayoutType.CASH_IN_MAIL: + request_data: CashMailCashoutMethodRequestData = CashMailCashoutMethodRequestData.model_validate(cashout_method.data.model_dump()) + else: + raise ValueError(f"unknown {payout_type=}") + + account = ledger_manager.get_account_or_create_user_wallet(user) + + pe = self.create( + uuid=pe_uuid, + debit_account_uuid=account.uuid, + cashout_method_uuid=cashout_method_id, + amount=amount, + created=now, + payout_type=payout_type, + request_data=request_data.model_dump(mode="json"), + ) + + ledger_manager.create_tx_user_payout_request(user, payout_event=pe, created=now) + return pe + + + + + +def make_request_paypal( + cashout_method: CashoutMethod, +) -> PaypalCashoutMethodRequestData: + return PaypalCashoutMethodRequestData.model_validate( + { + "email": cashout_method.data.email, + "interface": "api", + } + ) diff --git a/generalresearch/models/thl/ipinfo.py b/generalresearch/models/thl/ipinfo.py index c4ad50c..be9ac14 100644 --- a/generalresearch/models/thl/ipinfo.py +++ b/generalresearch/models/thl/ipinfo.py @@ -2,16 +2,17 @@ from __future__ import annotations import ipaddress from datetime import UTC, datetime -from typing import TYPE_CHECKING, Any, Literal, Self +from typing import Any, Literal, Self from faker import Faker +from grip_client import AccessType from pydantic import ( BaseModel, ConfigDict, Field, PositiveInt, - PrivateAttr, field_validator, + IPvAnyAddress, ) from generalresearch.models.custom_types import ( @@ -19,17 +20,13 @@ from generalresearch.models.custom_types import ( CountryISOLike, IPvAnyAddressStr, ) -from generalresearch.models.thl.maxmind.definitions import UserType - -if TYPE_CHECKING: - from generalresearch.managers.thl.ipinfo import IPGeonameManager fake = Faker() PrefixLength = Literal["/128", "/64", "/32"] -def normalize_ip(ip: IPvAnyAddressStr) -> tuple[str, PrefixLength]: +def normalize_ip(ip: str | IPvAnyAddress) -> tuple[str, PrefixLength]: """ Normalize an IP address for MySQL storage. @@ -41,7 +38,7 @@ def normalize_ip(ip: IPvAnyAddressStr) -> tuple[str, PrefixLength]: """ addr = ipaddress.ip_address(ip) if addr.version == 4: - return ip, "/32" + return addr.exploded, "/32" net64 = ipaddress.IPv6Network((addr, 64), strict=False) return net64.network_address.exploded, "/64" @@ -126,138 +123,34 @@ class IPGeoname(BaseModel): class IPInformation(BaseModel): - ip: IPvAnyAddressStr = Field() - # This doesn't get stored in mysql/redis, b/c we only look up by the normalized ip - lookup_prefix: PrefixLength | None = Field(default=None, exclude=True) + """ + Fields we'll always pull from GRIP's mmdb files at minimum + """ - geoname_id: PositiveInt | None = Field(default=None) + ip: IPvAnyAddressStr = Field() - country_iso: CountryISOLike = Field( + country_iso: CountryISOLike | None = Field( description="The ISO code of the country associated with the IP address.", examples=[fake.country_code().lower()], ) - registered_country_iso: CountryISOLike | None = Field( - default=None, - description="The ISO code of the country where the IP address is registered.", - examples=[fake.country_code().lower()], - ) is_anonymous: bool | None = Field( default=None, description="Indicates whether the IP address is associated with an " "anonymous source (e.g., VPN, proxy).", examples=[False], ) - is_anonymous_vpn: bool | None = Field(default=None) - is_hosting_provider: bool | None = Field(default=None) - is_public_proxy: bool | None = Field(default=None) - is_tor_exit_node: bool | None = Field(default=None) - is_residential_proxy: bool | None = Field(default=None) autonomous_system_number: PositiveInt | None = Field(default=None) autonomous_system_organization: str | None = Field(default=None, max_length=255) - domain: str | None = Field(default=None, max_length=255) - isp: str | None = Field( - default=None, - description="The Internet Service Provider associated with the IP address.", - examples=["Comcast"], - ) - - mobile_country_code: str | None = Field(default=None, max_length=3) - mobile_network_code: str | None = Field(default=None, max_length=3) - - network: str | None = Field(default=None, max_length=56) - organization: str | None = Field(default=None, max_length=255) - - static_ip_score: float | None = Field( - default=None, - description="A score indicating the likelihood that the IP address is static.", - ) - user_type: UserType | None = Field( + access_type: AccessType | None = Field( default=None, description="The type of user associated with the IP address " "(e.g., 'residential', 'business').", - examples=[UserType.RESIDENTIAL], - ) - postal_code: str | None = Field( - default=None, - description="The postal code associated with the IP address.", - examples=[fake.postcode()], - ) - - latitude: float | None = Field( - description="The latitude coordinate of the IP address location.", - default=None, - examples=[float(fake.latitude())], - ) - longitude: float | None = Field( - description="The longitude coordinate of the IP address location.", - default=None, - examples=[float(fake.longitude())], - ) - - accuracy_radius: int | None = Field( - default=None, - description="The approximate radius of accuracy for the latitude " - "and longitude, in kilometers.", - examples=[fake.random_int(min=25, max=250)], - ) - - updated: AwareDatetimeISO = Field( - default_factory=lambda: datetime.now(tz=UTC), + examples=[AccessType.RESIDENTIAL], ) - _geoname: IPGeoname | None = PrivateAttr(default=None) - - @field_validator("country_iso", "registered_country_iso", mode="before") - def make_lower(cls, value: str | None) -> str | None: - if value is not None: - return value.lower() - - return value - - @property - def basic(self) -> bool: - # This could be almost any field, but we're checking here if GRIP - # was run on this record. If not, then most of the optional - # fields will be None - return self.is_anonymous is None - - @property - def geoname(self) -> IPGeoname | None: - return self._geoname or None - - def normalize_ip(self): - normalized_ip, lookup_prefix = normalize_ip(self.ip) - self.ip = normalized_ip - self.lookup_prefix = lookup_prefix - - # --- prefetch_* --- - def prefetch_geoname( - self, - ip_gm: IPGeonameManager, - ) -> None: - if self.geoname_id is None: - raise ValueError("Must provide geoname_id") - - # from generalresearch.managers.thl.ipinfo import IPGeonameManager - # ip_gm = IPGeonameManager(pg_config=pg_config) - - self._geoname = ip_gm.get_by_id(geoname_id=self.geoname_id) - - # --- ORM --- - def model_dump_mysql(self): - d = self.model_dump(mode="json") - d["updated"] = self.updated - return d - - @classmethod - def from_mysql(cls, d: dict[str, Any]) -> Self: - d["updated"] = d["updated"].replace(tzinfo=UTC) - - return cls.model_validate(d) - -class GeoIPInformation(IPInformation, IPGeoname): +class GeoIPInformation(IPInformation): model_config = ConfigDict(extra="ignore") diff --git a/generalresearch/models/thl/maxmind/__init__.py b/generalresearch/models/thl/maxmind/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/generalresearch/models/thl/maxmind/definitions.py b/generalresearch/models/thl/maxmind/definitions.py deleted file mode 100644 index 01431c7..0000000 --- a/generalresearch/models/thl/maxmind/definitions.py +++ /dev/null @@ -1,22 +0,0 @@ -from enum import Enum - -from generalresearch.utils.enum import ReprEnumMeta - - -class UserType(Enum, metaclass=ReprEnumMeta): - # https://support.maxmind.com/hc/en-us/articles/4408430082971-IP-Trait-Risk-Data#h_01FN6V8JMQMWZGWNPPAW77ZPY4 - BUSINESS = "business" - CAFE = "cafe" - CELLULAR = "cellular" - COLLEGE = "college" - CDN = "content_delivery_network" - CPN = "consumer_privacy_network" - GOVERNMENT = "government" - HOSTING = "hosting" - LIBRARY = "library" - MILITARY = "military" - RESIDENTIAL = "residential" - ROUTER = "router" - SCHOOL = "school" - SEARCH_ENGINE = "search_engine_spider" - TRAVELER = "traveler" diff --git a/generalresearch/models/thl/user_iphistory.py b/generalresearch/models/thl/user_iphistory.py index a7eadf4..0e09de0 100644 --- a/generalresearch/models/thl/user_iphistory.py +++ b/generalresearch/models/thl/user_iphistory.py @@ -4,7 +4,6 @@ import ipaddress from datetime import UTC, datetime, timedelta from typing import TYPE_CHECKING, Self -from faker import Faker from pydantic import ( BaseModel, ConfigDict, @@ -19,15 +18,12 @@ from generalresearch.models.custom_types import ( IPvAnyAddressStr, ) from generalresearch.models.thl.ipinfo import GeoIPInformation, normalize_ip -from generalresearch.models.thl.user import User +from generalresearch.models.thl.user_ref import UserRef if TYPE_CHECKING: from grip_client.enums import AccessType - from generalresearch.pg_helper import PostgresConfig - from generalresearch.redis_helper import RedisConfig - -fake = Faker() + from generalresearch.managers.thl.ipinfo import GeoIpInfoManager class UserIPRecord(BaseModel): @@ -43,30 +39,12 @@ class UserIPRecord(BaseModel): @property def is_anonymous(self) -> bool: - # default to False even if insights is not looked up - return ( - self.information.is_anonymous - if self.information - and self.information.basic is False - and self.information.is_anonymous is not None - else False - ) - - @property - def user_type(self) -> AccessType | None: - return self.information.user_type if self.information else None + # Default to false if information is not looked up + return (self.information.is_anonymous if self.information else False) or False @property def access_type(self) -> AccessType | None: - return self.information.user_type if self.information else None - - @property - def subdivision_1_iso(self) -> str | None: - return self.information.subdivision_1_iso if self.information else None - - @property - def subdivision_2_iso(self) -> str | None: - return self.information.subdivision_2_iso if self.information else None + return self.information.access_type if self.information else None class IPRecord(BaseModel): @@ -93,27 +71,6 @@ class IPRecord(BaseModel): ) -> bool: return not (ip == self.ip and forwarded_ips == self.forwarded_ips) - # --- prefetch_* --- - def prefetch_ipinfo( - self, - pg_config: PostgresConfig, - redis_config: RedisConfig, - include_forwarded: bool = True, - ) -> None: - from generalresearch.managers.thl.ipinfo import GeoIpInfoManager - - m = GeoIpInfoManager(pg_config=pg_config, redis_config=redis_config) - - if include_forwarded: - ips = {self.ip} - ips.update(set(self.forwarded_ips)) - res = m.get_multi(ips) - self.information = res.get(self.ip) - for x in self.forwarded_ip_records: - x.information = res.get(x.ip) - else: - self.information = m.get(ip_address=self.ip) - # --- ORM --- @classmethod def from_mysql(cls, d: dict) -> Self: @@ -146,7 +103,7 @@ class IPRecord(BaseModel): class UserIPHistory(BaseModel): model_config = ConfigDict(validate_assignment=True) - user_id: PositiveInt = Field() + user: UserRef = Field() # In thl-gprc, we run "audit_ip_history()", and so a user should # get blocked after 100 IP switches or 30 unique IPs @@ -165,9 +122,6 @@ class UserIPHistory(BaseModel): default=None, description="These are any IP addresses that came in " ) - # -- prefetch_ fields - user: User | None = Field(default=None) - @field_validator("ips", mode="after") @classmethod def ips_timestamp(cls, ips): @@ -180,30 +134,11 @@ class UserIPHistory(BaseModel): reverse=True, ) - def prefetch_user( - self, - pg_config: PostgresConfig, - redis_config: RedisConfig, - pg_config_rr: PostgresConfig, - ) -> None: - from generalresearch.managers.thl.user_manager.user_manager import ( - UserManager, - ) - - um = UserManager( - pg_config=pg_config, - pg_config_rr=pg_config_rr, - redis=redis_config.dsn, - ) - self.user = um.get_user(user_id=self.user_id) - - def enrich_ips(self, pg_config: PostgresConfig, redis_config: RedisConfig) -> None: - from generalresearch.managers.thl.ipinfo import GeoIpInfoManager - - m = GeoIpInfoManager(pg_config=pg_config, redis_config=redis_config) - + def enrich_ips(self, geoip_info_manager: GeoIpInfoManager) -> None: + if not self.ips: + return ip_addresses = {x.ip for x in self.ips if x.information is None} - res = m.get_multi(ip_addresses=ip_addresses) + res = geoip_info_manager.get_multi(ip_addresses=ip_addresses) for x in self.ips: if res.get(x.ip): x.information = res[x.ip] diff --git a/generalresearch/models/thl/wallet/cashout_method.py b/generalresearch/models/thl/wallet/cashout_method.py index 1e96818..d9bc34c 100644 --- a/generalresearch/models/thl/wallet/cashout_method.py +++ b/generalresearch/models/thl/wallet/cashout_method.py @@ -214,6 +214,8 @@ class CashMailCashoutMethodData(BaseModel): description="Delivery address where payment should be sent" ) +class CashMailCashoutMethodRequestData(CashMailCashoutMethodData): + pass class PaypalCashoutMethodData(BaseModel): type: Literal[PayoutType.PAYPAL] = Field(default=PayoutType.PAYPAL) @@ -223,6 +225,22 @@ class PaypalCashoutMethodData(BaseModel): examples=["test@example.com"], ) +class PaypalCashoutMethodRequestData(BaseModel): + email: EmailStr = Field( + description="Email address of the paypal user", + examples=["test@example.com"], + ) + interface: Literal["api"] = Field(default="api") + +class TangoCashoutMethodRequestData(BaseModel): + accountIdentifier: str = Field() + customerIdentifier: str = Field() + utid: str = Field(description="tango utid") + amount: USDCent = Field(description="Amount to be paid out") + campaign: Literal["300large"] + sendEmail: bool = Field(default=False) + externalRefID: str = Field(description="External Ref ID") + description: str = Field() class TangoCashoutMethodData(BaseModel): type: Literal[PayoutType.TANGO] = Field(default=PayoutType.TANGO) -- cgit v1.2.3 From 8006c422a34d3bc4645915ac18b43bef1b2432a4 Mon Sep 17 00:00:00 2001 From: stuppie Date: Thu, 17 Sep 2026 17:40:39 -0600 Subject: paypal and tango cashout request handling --- generalresearch/managers/thl/cashout_method.py | 71 ++++++++--- generalresearch/managers/thl/paypal.py | 139 ++++++++------------- generalresearch/managers/thl/wallet/__init__.py | 42 +++---- generalresearch/managers/thl/wallet/approve.py | 17 +-- generalresearch/managers/thl/wallet/tango.py | 42 +------ generalresearch/managers/thl/wallet/tango_tasks.py | 12 +- generalresearch/managers/thl/wallet/user_payout.py | 73 ++++++++--- generalresearch/models/thl/ipinfo.py | 12 ++ .../models/thl/wallet/cashout_method.py | 40 ++---- generalresearch/models/thl/wallet/definitions.py | 22 ++-- 10 files changed, 241 insertions(+), 229 deletions(-) diff --git a/generalresearch/managers/thl/cashout_method.py b/generalresearch/managers/thl/cashout_method.py index dad1ef1..46bde38 100644 --- a/generalresearch/managers/thl/cashout_method.py +++ b/generalresearch/managers/thl/cashout_method.py @@ -12,12 +12,14 @@ from generalresearch.currency import USDCent from generalresearch.managers.base import PostgresManager from generalresearch.models.thl.user_ref import UserRef from generalresearch.models.thl.wallet.definitions import ( + CURRENCY_FORMATTER, SUPPORTED_CURRENCIES, Currency, PayoutType, ) if TYPE_CHECKING: + from generalresearch.managers.thl.wallet.tango import TangoManager from generalresearch.models.thl.user import User from generalresearch.models.thl.wallet.cashout_method import ( CashMailCashoutMethodData, @@ -75,8 +77,8 @@ class CashoutMethodManager(PostgresManager): ) def create_cash_in_mail_cashout_method( - self, data: CashMailCashoutMethodData, user: User - ) -> str: + self, data: CashMailCashoutMethodData, user: UserRef | User + ) -> CashoutMethod: """ Each user can create 1 or more "cash in mail" cashout method. This stores their address and possible shipping requests ? Each address @@ -87,6 +89,8 @@ class CashoutMethodManager(PostgresManager): # todo: validate shipping address? from generalresearch.models.thl.wallet.cashout_method import CashoutMethod + user = user if isinstance(user, UserRef) else user.to_user_ref() + cm = CashoutMethod( name="Cash in Mail", description="USPS delivery of cash", @@ -98,7 +102,7 @@ class CashoutMethodManager(PostgresManager): max_value=25000, # $250.00 data=data, type=PayoutType.CASH_IN_MAIL, - user=user.to_user_ref(), + user=user, ext_id=data.delivery_address.md5sum(), ) @@ -113,25 +117,25 @@ class CashoutMethodManager(PostgresManager): if res: # Already exists with the same address assert len(res) == 1 - return res[0].id + return res[0] self.create(cm) - return cm.id + return cm def create_paypal_cashout_method( - self, data: PaypalCashoutMethodData, user: User - ) -> str: + self, data: PaypalCashoutMethodData, user: UserRef | User + ) -> CashoutMethod: """ If it already exists, and the emails are the same, do nothing. If the - email is different, raises an error + email is different, it raises an error - :param data: - :param user: :return: the uuid of the created cashout method """ from generalresearch.models.thl.wallet.cashout_method import CashoutMethod + user = user if isinstance(user, UserRef) else user.to_user_ref() + cm = CashoutMethod( name="PayPal", description="Cashout via PayPal", @@ -142,7 +146,7 @@ class CashoutMethodManager(PostgresManager): max_value=25_000, # $250.00 data=data, type=PayoutType.PAYPAL, - user=user.to_user_ref(), + user=user, ext_id=data.email, ) # Make sure this user doesn't already have one @@ -151,7 +155,7 @@ class CashoutMethodManager(PostgresManager): assert len(res) == 1 if res[0].data.email == data.email: # Already exists with the same email, just return it - return res[0].id + return res[0] else: raise ValueError( "User already has a cashout method of this type. " @@ -159,7 +163,7 @@ class CashoutMethodManager(PostgresManager): ) else: self.create(cm) - return cm.id + return cm @staticmethod def make_filter_str( @@ -350,7 +354,9 @@ class CashoutMethodManager(PostgresManager): return {x.id: x for x in cms} @staticmethod - def format_from_db(x: dict[str, Any], user: User | None = None) -> CashoutMethod: + def format_from_db( + x: dict[str, Any], user: User | UserRef | None = None + ) -> CashoutMethod: x["id"] = UUID(x["id"]).hex # The data column here is inconsistent. Pulling keys from the mysql 'data' col @@ -368,7 +374,42 @@ class CashoutMethodManager(PostgresManager): x["data"].update(x.pop("_data_")) x["data"]["type"] = x["type"] if user and x["type"] in {PayoutType.PAYPAL, PayoutType.CASH_IN_MAIL}: - x["user"] = user.to_user_ref() + user = user if isinstance(user, UserRef) else user.to_user_ref() + x["user"] = user x["original_currency"] = x.get("currency") or Currency.USD x["currency"] = Currency.USD return CashoutMethod.model_validate(x) + + def get_expected_foreign_redemption_value( + self, cashout_method_id: str, amount: USDCent, tango_manager: TangoManager + ) -> tuple[int, Currency]: + """ + Convert USD cents to a Tango card's smallest currency unit. + for e.g.: A user wants a variable value CAD visa card. He redeems $10 USD (amount=1000), + this function returns the amount that will be redeemed in CAD. + :param cashout_method_id: ID of tango card. expected to be non USD. If USD, just returns the amount. + :param amount: amount the user is redeeming from their wallet in USD integer cents. + :return: amount that will be redeemed through tango on their foreign card, in the card's + (specified by the UTID) foreign currency (in integer units of the lowest denomination) + # example CAD visa: U121653 + """ + assert type(amount) is USDCent + # We don't **need** the method to be live? They can see the rate but won't + # be able to request it. + res = self.filter(uuid=cashout_method_id, is_live=None) + assert len(res) == 1, f"No cashout method found with id {cashout_method_id}" + cm = res[0] + if cm.type == PayoutType.TANGO: + assert cm.original_currency is not None + if cm.original_currency == Currency.USD: + return int(amount), Currency.USD + foreign_amount = round( + int(amount) / tango_manager.get_exchange_rates()[cm.original_currency] + ) + return foreign_amount, cm.original_currency + else: + # todo: bitcoin... etc + return int(amount), Currency.USD + + def format_currency(self, amount: int, currency: Currency): + return CURRENCY_FORMATTER[currency](amount) diff --git a/generalresearch/managers/thl/paypal.py b/generalresearch/managers/thl/paypal.py index 5ef0966..7e33da0 100644 --- a/generalresearch/managers/thl/paypal.py +++ b/generalresearch/managers/thl/paypal.py @@ -1,8 +1,9 @@ from __future__ import annotations -from datetime import datetime, timedelta, timezone +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta from decimal import Decimal -from typing import Any, Mapping +from typing import Any import requests @@ -10,16 +11,11 @@ from generalresearch.currency import USDCent from generalresearch.managers.thl.ledger_manager.thl_ledger import ( ThlLedgerManager, ) -from generalresearch.managers.thl.payout import UserPayoutEventManager from generalresearch.managers.thl.user_manager.user_manager import UserManager +from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager from generalresearch.models.thl.definitions import PayoutStatus from generalresearch.models.thl.payout import UserPayoutEvent -from generalresearch.models.thl.user import User -from generalresearch.models.thl.wallet import PayoutType -from generalresearch.models.thl.wallet.cashout_method import ( - CashoutMethod, - PaypalCashoutMethodData, -) +from generalresearch.models.thl.wallet.definitions import PayoutType PAYPAL_SANDBOX_URL = "https://api-m.sandbox.paypal.com" PAYPAL_PROD_URL = "https://api-m.paypal.com" @@ -59,7 +55,7 @@ class PayPalPayoutManager: self._access_token_expires_at: datetime | None = None def _get_access_token(self) -> str: - now = datetime.now(tz=timezone.utc) + now = datetime.now(tz=UTC) if ( self._access_token and self._access_token_expires_at @@ -204,83 +200,55 @@ class PayPalPayoutManager: ) return result.get("verification_status") == "SUCCESS" + def attempt_paypal_payout( + self, + *, + payout_event: UserPayoutEvent, + user_payout_event_manager: UserPayoutEventManager, + note: str | None = None, + email_subject: str | None = None, + ) -> UserPayoutEvent: + """Submit a one-item PayPal payout. + + The payout-event UUID is used as both sender IDs, making a retry at + PayPal idempotent for 30 days. The PayPal-generated batch ID is persisted + in ``ext_ref_id`` and the initial API response is kept in ``order_data``. + + If submission raises, the payout event and its ledger reservation remain + in place. This is intentional: a timeout or 5xx response is ambiguous and + rolling back could allow the same money to be paid twice. The exception + includes the payout-event UUID for reconciliation or a same-ID retry. + """ + assert payout_event.payout_type == PayoutType.PAYPAL, ( + "payout_event must be PayPal" + ) + assert payout_event.status == PayoutStatus.PENDING, "status must be PENDING" + amount_cents = USDCent(payout_event.amount) + recipient_email = payout_event.request_data["email"] -def create_paypal_payout( - *, - user: User, - cashout_method: CashoutMethod, - amount_cents: USDCent, - user_payout_event_manager: UserPayoutEventManager, - ledger_manager: ThlLedgerManager, - paypal: PayPalPayoutManager, - note: str | None = None, - email_subject: str | None = None, -) -> UserPayoutEvent: - """Create, reserve funds for, and submit a one-item PayPal payout. - - The payout-event UUID is used as both sender IDs, making a retry at - PayPal idempotent for 30 days. The PayPal-generated batch ID is persisted - in ``ext_ref_id`` and the initial API response is kept in ``order_data``. - - If submission raises, the payout event and its ledger reservation remain - in place. This is intentional: a timeout or 5xx response is ambiguous and - rolling back could allow the same money to be paid twice. The exception - includes the payout-event UUID for reconciliation or a same-ID retry. - """ - if cashout_method.type != PayoutType.PAYPAL: - raise ValueError("cashout_method must be a PayPal cashout method") - if not isinstance(cashout_method.data, PaypalCashoutMethodData): - raise ValueError("cashout_method does not contain PayPal data") - if ( - user.user_id is None - or cashout_method.user is None - or cashout_method.user.user_id != user.user_id - ): - raise ValueError("cashout_method does not belong to user") - - cashout_method.validate_requested_amount(amount_cents) - user_account = ledger_manager.get_account_or_create_user_wallet(user=user) - payout_event = user_payout_event_manager.create( - debit_account_uuid=user_account.uuid, - cashout_method_uuid=cashout_method.id, - payout_type=PayoutType.PAYPAL, - amount=amount_cents, - status=PayoutStatus.PENDING, - account_reference_type="user", - account_reference_uuid=user.uuid, - description=cashout_method.name, - request_data={ - "interface": "api", - "recipient_email": str(cashout_method.data.email), - }, - ) - ledger_manager.create_tx_user_payout_request( - user=user, - payout_event=payout_event, - ) + try: + paypal_response = self.send_payment( + recipient_email=str(recipient_email), + amount_cents=amount_cents, + sender_batch_id=payout_event.uuid, + sender_item_id=payout_event.uuid, + note=note, + email_subject=email_subject, + ) + payout_batch_id = paypal_response["batch_header"]["payout_batch_id"] + except Exception as exc: + user_payout_event_manager.update(payout_event, status=PayoutStatus.FAILED) + raise PayPalError( + f"PayPal submission failed for payout event {payout_event.uuid}" + ) from exc - try: - paypal_response = paypal.send_payment( - recipient_email=str(cashout_method.data.email), - amount_cents=amount_cents, - sender_batch_id=payout_event.uuid, - sender_item_id=payout_event.uuid, - note=note, - email_subject=email_subject, + user_payout_event_manager.update( + payout_event=payout_event, + status=PayoutStatus.APPROVED, + ext_ref_id=payout_batch_id, + order_data=paypal_response, ) - payout_batch_id = paypal_response["batch_header"]["payout_batch_id"] - except Exception as exc: - raise PayPalError( - f"PayPal submission failed for payout event {payout_event.uuid}" - ) from exc - - user_payout_event_manager.update( - payout_event=payout_event, - status=PayoutStatus.PENDING, - ext_ref_id=payout_batch_id, - order_data=paypal_response, - ) - return payout_event + return payout_event def handle_paypal_payout_webhook( @@ -360,8 +328,7 @@ def handle_paypal_payout_webhook( user = user_manager.get_user(user_uuid=payout_event.account_reference_uuid) user.prefetch_product(pg_config=ledger_manager.pg_config) complete_tag = ( - f"{ledger_manager.currency.value}:user_payout:" - f"{payout_event.uuid}:complete" + f"{ledger_manager.currency.value}:user_payout:{payout_event.uuid}:complete" ) complete_transactions = ledger_manager.get_tx_ids_by_tag(complete_tag) if len(complete_transactions) > 1: diff --git a/generalresearch/managers/thl/wallet/__init__.py b/generalresearch/managers/thl/wallet/__init__.py index 006908b..08ac110 100644 --- a/generalresearch/managers/thl/wallet/__init__.py +++ b/generalresearch/managers/thl/wallet/__init__.py @@ -6,6 +6,7 @@ from generalresearch.managers.thl.wallet.approve import ( approve_amt_cashout, approve_paypal_order, ) +from generalresearch.managers.thl.wallet.tango_tasks import complete_tango_order from generalresearch.models.thl.definitions import PayoutStatus from generalresearch.models.thl.wallet.definitions import PayoutType @@ -13,10 +14,8 @@ if TYPE_CHECKING: from generalresearch.managers.thl.ledger_manager.thl_ledger import ( ThlLedgerManager, ) - from generalresearch.managers.thl.payout import ( - PayoutEventManager, - UserPayoutEventManager, - ) + from generalresearch.managers.thl.payout import UserPayoutEventManager + from generalresearch.managers.thl.paypal import PayPalPayoutManager from generalresearch.managers.thl.tango_api import TangoClient from generalresearch.managers.thl.user_manager.user_manager import ( UserManager, @@ -35,9 +34,10 @@ def manage_pending_cashout( user_ip_history_manager: UserIpHistoryManager, user_manager: UserManager, ledger_manager: ThlLedgerManager, - geoip_info_manager:GeoIpInfoManager, + geoip_info_manager: GeoIpInfoManager, order_data: dict[str, Any] | CashMailOrderData | None = None, tango_client: TangoClient | None = None, + paypal_client: PayPalPayoutManager | None = None, ) -> UserPayoutEvent: """ Called by a UI actions performed by Todd. This rejects/approves/cancels @@ -65,45 +65,45 @@ def manage_pending_cashout( "manage_pending_cashout called on user without managed wallet" ) assert not user.blocked, "manage_pending_cashout: Blocked user" - assert not user_ip_history_manager.is_user_anonymous(user,geoip_info_manager=geoip_info_manager), ( - "manage_pending_cashout: Anonymous user" - ) + assert not user_ip_history_manager.is_user_anonymous( + user, geoip_info_manager=geoip_info_manager + ), "manage_pending_cashout: Anonymous user" # Just assign it with direct casting/type annotation - payout_event_manager: PayoutEventManager = user_payout_event_manager + # payout_event_manager: PayoutEventManager = user_payout_event_manager if new_status == PayoutStatus.APPROVED: if pe.payout_type == PayoutType.TANGO: - from generalresearch.managers.thl.wallet.tango import ( - complete_tango_order, - ) - assert tango_client is not None complete_tango_order( user=user, payout_event=pe, - payout_event_manager=payout_event_manager, + user_payout_event_manager=user_payout_event_manager, ledger_manager=ledger_manager, tango_client=tango_client, ) elif pe.payout_type == PayoutType.PAYPAL: + assert paypal_client is not None + approve_paypal_order( - payout_event=pe, payout_event_manager=payout_event_manager + payout_event=pe, + user_payout_event_manager=user_payout_event_manager, + paypal_client=paypal_client, ) elif pe.payout_type in {PayoutType.AMT_BONUS, PayoutType.AMT_HIT}: approve_amt_cashout( user=user, payout_event=pe, - payout_event_manager=payout_event_manager, + payout_event_manager=user_payout_event_manager, ledger_manager=ledger_manager, ) elif pe.payout_type == PayoutType.CASH_IN_MAIL: assert order_data, "must pass order_data" - payout_event_manager.update( + user_payout_event_manager.update( pe, status=PayoutStatus.APPROVED, order_data=order_data ) ledger_manager.create_tx_user_payout_complete( @@ -132,24 +132,24 @@ def manage_pending_cashout( # uses manual_complete_paypal_order() raise ValueError("user custom paypal script for this") - payout_event_manager.update(pe, status=new_status) + user_payout_event_manager.update(pe, status=new_status) return pe elif new_status == PayoutStatus.REJECTED: # They lose the money in their wallet at this point, no ledger txs occur. - payout_event_manager.update(pe, status=new_status) + user_payout_event_manager.update(pe, status=new_status) return pe elif new_status == PayoutStatus.CANCELLED: # create another ledger item putting the money back into their wallet. - payout_event_manager.update(pe, status=new_status) + user_payout_event_manager.update(pe, status=new_status) ledger_manager.create_tx_user_payout_cancelled(user, payout_event=pe) return pe elif new_status == PayoutStatus.FAILED: # We just update the status (like in PayoutStatus.REJECTED). No ledger xs - payout_event_manager.update(pe, status=new_status) + user_payout_event_manager.update(pe, status=new_status) return pe else: diff --git a/generalresearch/managers/thl/wallet/approve.py b/generalresearch/managers/thl/wallet/approve.py index a1c01d2..44382fa 100644 --- a/generalresearch/managers/thl/wallet/approve.py +++ b/generalresearch/managers/thl/wallet/approve.py @@ -1,5 +1,3 @@ -from __future__ import annotations - from typing import TYPE_CHECKING from generalresearch.models.thl.definitions import PayoutStatus @@ -9,6 +7,8 @@ if TYPE_CHECKING: ThlLedgerManager, ) from generalresearch.managers.thl.payout import PayoutEventManager + from generalresearch.managers.thl.paypal import PayPalPayoutManager + from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager from generalresearch.models.thl.payout import UserPayoutEvent from generalresearch.models.thl.user import User @@ -34,7 +34,9 @@ def approve_amt_cashout( def approve_paypal_order( - payout_event: UserPayoutEvent, payout_event_manager: PayoutEventManager + payout_event: UserPayoutEvent, + user_payout_event_manager: UserPayoutEventManager, + paypal_client: PayPalPayoutManager | None = None, ): """ The order has been approved, but it hasn't actually been sent. @@ -46,17 +48,18 @@ def approve_paypal_order( "attempting to manage payout that is not pending (or you can retry a failed order)" ) - payout_event_manager.update(payout_event, status=PayoutStatus.APPROVED) - interface = payout_event.request_data.get("interface") if interface == "api": # todo: Use the Payouts API to sent this payout, and then update the DB - pass + paypal_client.attempt_paypal_payout( + payout_event=payout_event, + user_payout_event_manager=user_payout_event_manager, + ) else: # Flow monitoring for payouts where the type is paypal, the status is # approved, and the interface is web, and then it'll send the payout and # update the status to complete (and create a ledger item for the fee) - pass + user_payout_event_manager.update(payout_event, status=PayoutStatus.APPROVED) return payout_event diff --git a/generalresearch/managers/thl/wallet/tango.py b/generalresearch/managers/thl/wallet/tango.py index fc6763b..ca993ea 100644 --- a/generalresearch/managers/thl/wallet/tango.py +++ b/generalresearch/managers/thl/wallet/tango.py @@ -2,24 +2,25 @@ from __future__ import annotations from datetime import timedelta from threading import Lock -from typing import Any +from typing import TYPE_CHECKING, Any from cachetools import TTLCache, cachedmethod from generalresearch.currency import USDCent -from generalresearch.managers.thl.cashout_method import CashoutMethodManager -from generalresearch.managers.thl.tango_api import TangoClient from generalresearch.models.thl.wallet.cashout_method import ( - TangoCashoutMethodData, CashoutMethod, TangoCashoutMethodRequestData, ) from generalresearch.models.thl.wallet.definitions import ( - CURRENCY_FORMATTER, Currency, PayoutType, ) +if TYPE_CHECKING: + from generalresearch.managers.thl.cashout_method import CashoutMethodManager + from generalresearch.managers.thl.tango_api import TangoClient + + class TangoManager: def __init__( @@ -93,37 +94,6 @@ class TangoManager: cashout_method = next(iter(methods), None) return cashout_method.name if cashout_method else "Tango Gift Card" - def get_expected_redemption_value( - self, cashout_method_id: str, amount: USDCent - ) -> tuple[int, Currency]: - """ - Convert USD cents to a Tango card's smallest currency unit. - for e.g.: A user wants a variable value CAD visa card. He redeems $10 USD (amount=1000), - this function returns the amount that will be redeemed in CAD. - :param cashout_method_id: ID of tango card. expected to be non USD. If USD, just returns the amount. - :param amount: amount the user is redeeming from their wallet in USD integer cents. - :return: amount that will be redeemed through tango on their foreign card, in the card's - (specified by the UTID) foreign currency (in integer units of the lowest denomination) - # example CAD visa: U121653 - """ - assert type(amount) is USDCent - res = self.cashout_method_manager.filter(uuid=cashout_method_id, is_live=True) - if not res: - raise ValueError(f"no cashout method found for {cashout_method_id!r}") - cashout_method = res[0] - assert cashout_method.type == PayoutType.TANGO - assert cashout_method.original_currency is not None - - currency = cashout_method.original_currency - if currency == Currency.USD: - return int(amount), currency - - foreign_amount = round(int(amount) / self.get_exchange_rates()[currency]) - return foreign_amount, currency - - def format_currency(self, amount: int, currency: Currency): - return CURRENCY_FORMATTER[currency](amount) - def clear_caches(self) -> None: with self._exchange_rate_lock: self._exchange_rate_cache.clear() diff --git a/generalresearch/managers/thl/wallet/tango_tasks.py b/generalresearch/managers/thl/wallet/tango_tasks.py index 038f67f..b5e810c 100644 --- a/generalresearch/managers/thl/wallet/tango_tasks.py +++ b/generalresearch/managers/thl/wallet/tango_tasks.py @@ -12,8 +12,8 @@ if TYPE_CHECKING: from generalresearch.managers.thl.ledger_manager.thl_ledger import ( ThlLedgerManager, ) - from generalresearch.managers.thl.payout import PayoutEventManager from generalresearch.managers.thl.tango_api import TangoClient + from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager from generalresearch.models.thl.payout import UserPayoutEvent from generalresearch.models.thl.user import User @@ -21,7 +21,7 @@ if TYPE_CHECKING: def complete_tango_order( user: User, payout_event: UserPayoutEvent, - payout_event_manager: PayoutEventManager, + user_payout_event_manager: UserPayoutEventManager, ledger_manager: ThlLedgerManager, tango_client: TangoClient, ): @@ -35,7 +35,9 @@ def complete_tango_order( assert payout_event.status in { PayoutStatus.PENDING, PayoutStatus.FAILED, - }, "attempting to manage payout that is not pending (or you can retry a failed order)" + }, ( + "attempting to manage payout that is not pending (or you can retry a failed order)" + ) request = payout_event.request_data ref_id = request["externalRefID"] # amount_usd = Decimal(payout_event.request_data["amount_usd"]) @@ -53,11 +55,11 @@ def complete_tango_order( # todo: its possible the order went through, but something else was wrong # we should try to retrieve the order by its ref_id and confirm it really # failed... - payout_event_manager.update(payout_event, status=PayoutStatus.FAILED) + user_payout_event_manager.update(payout_event, status=PayoutStatus.FAILED) return payout_event # update TangoPayoutEvent with the order data - payout_event_manager.update( + user_payout_event_manager.update( payout_event, status=order["status"], ext_ref_id=order["referenceOrderID"], diff --git a/generalresearch/managers/thl/wallet/user_payout.py b/generalresearch/managers/thl/wallet/user_payout.py index 88505c0..ca47611 100644 --- a/generalresearch/managers/thl/wallet/user_payout.py +++ b/generalresearch/managers/thl/wallet/user_payout.py @@ -11,6 +11,7 @@ from redis import Redis from generalresearch.currency import USDCent from generalresearch.managers.thl.cashout_method import CashoutMethodManager from generalresearch.managers.thl.ipinfo import GeoIpInfoManager +from generalresearch.managers.thl.ledger_manager.exceptions import LedgerTransactionCreateError from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.managers.thl.payout import PayoutEventManager from generalresearch.managers.thl.userhealth import UserIpHistoryManager @@ -20,12 +21,12 @@ from generalresearch.models.thl.definitions import PayoutStatus from generalresearch.models.thl.payout import UserPayoutEvent from generalresearch.models.thl.user import User from generalresearch.models.thl.wallet.cashout_method import ( + CashMailCashoutMethodRequestData, CashMailOrderData, CashoutMethod, CashoutRequestInfo, PaypalCashoutMethodRequestData, TangoCashoutMethodRequestData, - CashMailCashoutMethodRequestData, ) from generalresearch.models.thl.wallet.definitions import PayoutType @@ -70,7 +71,7 @@ class UserPayoutEventManager(PayoutEventManager): pe = self.get_by_uuid(pe_uuid=pe_uuid) transaction_info = {} - order: dict[str, Any] = pe.order_data + order: dict[str, Any] | CashMailOrderData = pe.order_data if pe.payout_type == PayoutType.TANGO and pe.status == PayoutStatus.COMPLETE: reward = order["reward"] if "credentialList" in reward: @@ -85,7 +86,7 @@ class UserPayoutEventManager(PayoutEventManager): pe.payout_type == PayoutType.CASH_IN_MAIL and pe.status == PayoutStatus.COMPLETE ): - transaction_info = pe.order_data.model_dump(mode="json") + transaction_info = order.model_dump(mode="json") return CashoutRequestInfo( id=pe_uuid, @@ -249,10 +250,38 @@ class UserPayoutEventManager(PayoutEventManager): return payout_event + def try_user_request_redeem( + self, + user: User, + cashout_method_id: str, + amount: USDCent, + tango_manager: TangoManager, + cashout_method_manager: CashoutMethodManager, + ledger_manager: ThlLedgerManager, + user_ip_history_manager: UserIpHistoryManager, + geoip_info_manager: GeoIpInfoManager, + redis_client: Redis, + slack_client: slack.WebClient | None = None, + ) -> tuple[UserPayoutEvent | None, str | None]: + try: + return self.user_request_redeem( + user=user, + cashout_method_id=cashout_method_id, + amount=amount, + tango_manager=tango_manager, + cashout_method_manager=cashout_method_manager, + ledger_manager=ledger_manager, + user_ip_history_manager=user_ip_history_manager, + geoip_info_manager=geoip_info_manager, + redis_client=redis_client, + slack_client=slack_client, + ), None + except (AssertionError, LedgerTransactionCreateError) as e: + return None, str(e) + def user_request_redeem( self, user: User, - country_iso: str, cashout_method_id: str, amount: USDCent, tango_manager: TangoManager, @@ -260,7 +289,7 @@ class UserPayoutEventManager(PayoutEventManager): ledger_manager: ThlLedgerManager, user_ip_history_manager: UserIpHistoryManager, geoip_info_manager: GeoIpInfoManager, - redis_client: Redis, + redis_client: Redis, slack_client: slack.WebClient | None = None, ) -> UserPayoutEvent: """ @@ -271,6 +300,12 @@ class UserPayoutEventManager(PayoutEventManager): """ now = datetime.now(tz=UTC) user.prefetch_product(pg_config=self.pg_config) + + country_iso = user_ip_history_manager.get_user_latest_country( + user, geoip_info_manager + ) + assert country_iso, "user has no country" + usd_exchange_rates = tango_manager.get_exchange_rates() cashout_methods = cashout_method_manager.get_user_cashout_methods( user, country_iso=country_iso, usd_exchange_rate=usd_exchange_rates @@ -311,22 +346,23 @@ class UserPayoutEventManager(PayoutEventManager): assert not user_ip_history_manager.is_user_anonymous( user, geoip_info_manager=geoip_info_manager ), "Anonymous user requesting redemption" - ipr = user_ip_history_manager.get_user_latest_ip_record( - user, geoip_info_manager=geoip_info_manager - ) - if ipr is not None and ipr.country_iso in banned_countries: + if country_iso in banned_countries: raise AssertionError("Banned country requesting redemption") wallet_balance = ledger_manager.get_user_wallet_balance(user) if product.user_wallet_config.balance_type == "wallet_balance": redeemable_amount = wallet_balance elif product.user_wallet_config.balance_type == "redeemable_balance": - redeemable_amount = ledger_manager.get_user_redeemable_wallet_balance(user, wallet_balance) + redeemable_amount = ledger_manager.get_user_redeemable_wallet_balance( + user, wallet_balance + ) else: - raise ValueError(f"unexpected balance_type={product.user_wallet_config.balance_type}") + raise ValueError( + f"unexpected balance_type={product.user_wallet_config.balance_type}" + ) assert amount <= redeemable_amount, ( - f"User requesting more than their redeemable balance ({amount} > {redeemable_amount}}" + f"User requesting more than their redeemable balance ({amount} > {redeemable_amount})" ) # Simple dedupe mechanism. Don't allow more than 1 per user_id per minute per cashout_method. @@ -356,9 +392,15 @@ class UserPayoutEventManager(PayoutEventManager): amount, cashout_method, pe_uuid ) elif payout_type == PayoutType.PAYPAL: - request_data: PaypalCashoutMethodRequestData = make_request_paypal(cashout_method) + request_data: PaypalCashoutMethodRequestData = make_request_paypal( + cashout_method + ) elif payout_type == PayoutType.CASH_IN_MAIL: - request_data: CashMailCashoutMethodRequestData = CashMailCashoutMethodRequestData.model_validate(cashout_method.data.model_dump()) + request_data: CashMailCashoutMethodRequestData = ( + CashMailCashoutMethodRequestData.model_validate( + cashout_method.data.model_dump() + ) + ) else: raise ValueError(f"unknown {payout_type=}") @@ -378,9 +420,6 @@ class UserPayoutEventManager(PayoutEventManager): return pe - - - def make_request_paypal( cashout_method: CashoutMethod, ) -> PaypalCashoutMethodRequestData: diff --git a/generalresearch/models/thl/ipinfo.py b/generalresearch/models/thl/ipinfo.py index be9ac14..9fe86e4 100644 --- a/generalresearch/models/thl/ipinfo.py +++ b/generalresearch/models/thl/ipinfo.py @@ -133,6 +133,18 @@ class IPInformation(BaseModel): description="The ISO code of the country associated with the IP address.", examples=[fake.country_code().lower()], ) + country_name: str | None = Field(default=None, max_length=64) + + subdivision_1_iso: str | None = Field( + default=None, + description="The ISO code of the primary subdivision (e.g., state or province).", + max_length=3, + ) + subdivision_1_name: str | None = Field( + default=None, + description="The name of the primary subdivision (e.g., state or province).", + max_length=255, + ) is_anonymous: bool | None = Field( default=None, diff --git a/generalresearch/models/thl/wallet/cashout_method.py b/generalresearch/models/thl/wallet/cashout_method.py index d9bc34c..a82fa31 100644 --- a/generalresearch/models/thl/wallet/cashout_method.py +++ b/generalresearch/models/thl/wallet/cashout_method.py @@ -29,7 +29,6 @@ from generalresearch.models.thl.locales import CountryISO from generalresearch.models.thl.user_identifiers import BPUIDStr from generalresearch.models.thl.user_ref import UserRef from generalresearch.models.thl.wallet.definitions import Currency, PayoutType -from generalresearch.utils.enum import ReprEnumMeta logger = logging.getLogger() @@ -119,11 +118,12 @@ class CashoutMethodBase(BaseModel): # return None # return self.min_value * self.usd_exchange_rate - def validate_requested_amount(self, amount: PositiveInt): + def validate_requested_amount(self, amount: USDCent): """ Check if 'amount' is a valid amount that can be requested. :param amount: The amount to be requested in USD Cents """ + amount = int(amount) if amount <= 0: raise ValueError("Amount must be positive") if not self.min_value <= amount <= self.max_value: @@ -214,9 +214,11 @@ class CashMailCashoutMethodData(BaseModel): description="Delivery address where payment should be sent" ) + class CashMailCashoutMethodRequestData(CashMailCashoutMethodData): pass + class PaypalCashoutMethodData(BaseModel): type: Literal[PayoutType.PAYPAL] = Field(default=PayoutType.PAYPAL) @@ -225,6 +227,7 @@ class PaypalCashoutMethodData(BaseModel): examples=["test@example.com"], ) + class PaypalCashoutMethodRequestData(BaseModel): email: EmailStr = Field( description="Email address of the paypal user", @@ -232,6 +235,7 @@ class PaypalCashoutMethodRequestData(BaseModel): ) interface: Literal["api"] = Field(default="api") + class TangoCashoutMethodRequestData(BaseModel): accountIdentifier: str = Field() customerIdentifier: str = Field() @@ -242,6 +246,7 @@ class TangoCashoutMethodRequestData(BaseModel): externalRefID: str = Field(description="External Ref ID") description: str = Field() + class TangoCashoutMethodData(BaseModel): type: Literal[PayoutType.TANGO] = Field(default=PayoutType.TANGO) utid: str = Field(description="tango utid") @@ -408,33 +413,6 @@ example_foreign_value = { } -class RedemptionCurrency(StrEnum, metaclass=ReprEnumMeta): - """ - Supported Currencies for Foreign Redemptions - """ - - # US Dollars. Smallest Unit: Cents. - USD = "USD" - # Canadian Dollars. Smallest Unit: Cents. - CAD = "CAD" - # British Pounds. Smallest Unit: Pence. - GBP = "GBP" - # Euros. Smallest Unit: Cents. - EUR = "EUR" - # Indian Rupees. Smallest Unit: Paise. - INR = "INR" - # Australian Dollars. Smallest Unit: Cents. - AUD = "AUD" - # Polish Zloty. Smallest Unit: Grosz. - PLN = "PLN" - # Swedish Krona. Smallest Unit: Öre. - SEK = "SEK" - # Singapore Dollars. Smallest Unit: Cents. - SGD = "SGD" - # Mexican Pesos. Smallest Unit: Centavos. - MXN = "MXN" - - class CashoutMethodForeignValue(BaseModel): """ Shows the expected value of a redemption in a foreign currency. @@ -445,8 +423,8 @@ class CashoutMethodForeignValue(BaseModel): value: NonNegativeInt = Field( description="Value of the redemption in the currency's smallest unit." ) - currency: RedemptionCurrency = Field( - description=RedemptionCurrency.as_openapi_with_value_descriptions() + currency: Currency = Field( + description=Currency.as_openapi_with_value_descriptions() ) value_string: str = Field( description="A string representation of the value in the currency." diff --git a/generalresearch/models/thl/wallet/definitions.py b/generalresearch/models/thl/wallet/definitions.py index 4d9729b..4474745 100644 --- a/generalresearch/models/thl/wallet/definitions.py +++ b/generalresearch/models/thl/wallet/definitions.py @@ -35,26 +35,26 @@ class PayoutType(StrEnum, metaclass=ReprEnumMeta): AMT_ASSIGNMENT = "AMT_ASSIGNMENT" -class Currency(StrEnum): - # United States Dollar +class Currency(StrEnum, metaclass=ReprEnumMeta): + # United States Dollar. Smallest Unit: Cents. USD = "USD" - # Canadian Dollar + # Canadian Dollar. Smallest Unit: Cents. CAD = "CAD" - # British Pound Sterling + # British Pound. Smallest Unit: Pence. GBP = "GBP" - # Euro + # Euro. Smallest Unit: Cents. EUR = "EUR" - # Indian Rupee + # Indian Rupee. Smallest Unit: Paise. INR = "INR" - # Australian Dollar + # Australian Dollar. Smallest Unit: Cents. AUD = "AUD" - # Polish Zloty + # Polish Zloty. Smallest Unit: Grosz. PLN = "PLN" - # Swedish Krona + # Swedish Krona. Smallest Unit: Öre. SEK = "SEK" - # Singapore Dollar + # Singapore Dollar. Smallest Unit: Cents. SGD = "SGD" - # Mexican Peso + # Mexican Peso. Smallest Unit: Centavos. MXN = "MXN" SUPPORTED_CURRENCIES = {currency.value for currency in Currency} -- cgit v1.2.3 From 82be325a1f73f1ec7734b64607a23b564cc5d2ac Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 11:15:19 -0600 Subject: handle_paypal_payout_webhook fixes --- generalresearch/managers/thl/paypal.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/generalresearch/managers/thl/paypal.py b/generalresearch/managers/thl/paypal.py index 7e33da0..e6154c8 100644 --- a/generalresearch/managers/thl/paypal.py +++ b/generalresearch/managers/thl/paypal.py @@ -270,20 +270,15 @@ def handle_paypal_payout_webhook( raise PayPalError("PayPal webhook signature verification failed") event_type = webhook_event.get("event_type", "") - if not event_type.startswith(("PAYMENT.PAYOUTSBATCH.", "PAYMENT.PAYOUTS-ITEM.")): + if not event_type.startswith("PAYMENT.PAYOUTS-ITEM."): return None resource = webhook_event.get("resource") if not isinstance(resource, dict): raise PayPalError("PayPal payout webhook has no resource") - sender_batch_header = resource.get("sender_batch_header") or {} payout_item = resource.get("payout_item") or {} - payout_event_uuid = ( - resource.get("sender_batch_id") - or sender_batch_header.get("sender_batch_id") - or payout_item.get("sender_item_id") - ) + payout_event_uuid = payout_item.get("sender_item_id") if not payout_event_uuid: raise PayPalError("PayPal payout webhook has no sender payout ID") -- cgit v1.2.3 From 203066cb5228ce3bcadb940b9aca1b6a36dceb4e Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 11:16:22 -0600 Subject: user ip history and geo ip information fixes, fixtures, and tests --- generalresearch/managers/thl/userhealth.py | 65 ++++------ generalresearch/models/thl/ipinfo.py | 10 +- test_utils/managers/conftest.py | 107 +++++++++------ test_utils/models/thl/conftest.py | 174 +++++-------------------- tests/managers/thl/test_ipinfo.py | 201 +++++++---------------------- tests/managers/thl/test_userhealth.py | 189 ++++++++++----------------- 6 files changed, 246 insertions(+), 500 deletions(-) diff --git a/generalresearch/managers/thl/userhealth.py b/generalresearch/managers/thl/userhealth.py index 0df67aa..1aeab44 100644 --- a/generalresearch/managers/thl/userhealth.py +++ b/generalresearch/managers/thl/userhealth.py @@ -33,6 +33,10 @@ if TYPE_CHECKING: class UserIpHistoryManager(PostgresManagerWithRedis): + def __init__(self, geoip_info_manager: GeoIpInfoManager | None, **kwargs) -> None: + super().__init__(**kwargs) + self.geoip_info_manager = geoip_info_manager + def get_redis_key(self, user_id: int) -> str: return f"generalreserach:user-ip-history-v2:{user_id}" @@ -74,9 +78,8 @@ class UserIpHistoryManager(PostgresManagerWithRedis): iph = UserIPHistory(user=user, ips=records) self.set_user_ip_history_cache(user_id=user.user_id, iph=iph) - def get_user_ip_history( - self, user: UserRef | User, geoip_info_manager: GeoIpInfoManager | None = None - ) -> UserIPHistory: + def get_user_ip_history(self, user: UserRef | User) -> UserIPHistory: + assert self.geoip_info_manager is not None, "GeoIpInfoManager is required" user = user if isinstance(user, UserRef) else user.to_user_ref() iph = self.get_user_ip_history_cache(user_id=user.user_id) @@ -88,19 +91,15 @@ class UserIpHistoryManager(PostgresManagerWithRedis): # todo: we may get dns records from somewhere else here ... iph = UserIPHistory(user=user, ips=records) self.set_user_ip_history_cache(user_id=user.user_id, iph=iph) - if geoip_info_manager: - iph.enrich_ips(geoip_info_manager=geoip_info_manager) + iph.enrich_ips(geoip_info_manager=self.geoip_info_manager) return iph def get_user_latest_ip_record( self, user: UserRef | User, exclude_anon: bool = False, - geoip_info_manager: GeoIpInfoManager | None = None, ) -> UserIPRecord | None: - iphistory = self.get_user_ip_history( - user=user, geoip_info_manager=geoip_info_manager - ) + iphistory = self.get_user_ip_history(user=user) if not iphistory or not iphistory.ips: return None @@ -108,7 +107,6 @@ class UserIpHistoryManager(PostgresManagerWithRedis): # This logic was changed at some point? Or the py-utils # get_user_latest_ip_record was changed. Please be careful here... if exclude_anon: - assert geoip_info_manager is not None, "Must pass geoip_info_manager" for ipr in iphistory.ips: if not ipr.information.is_anonymous: return ipr @@ -120,11 +118,8 @@ class UserIpHistoryManager(PostgresManagerWithRedis): self, user: UserRef | User, exclude_anon: bool = False, - geoip_info_manager: GeoIpInfoManager | None = None, ) -> str | None: - record = self.get_user_latest_ip_record( - user=user, exclude_anon=exclude_anon, geoip_info_manager=geoip_info_manager - ) + record = self.get_user_latest_ip_record(user=user, exclude_anon=exclude_anon) if record: return record.ip return None @@ -132,25 +127,18 @@ class UserIpHistoryManager(PostgresManagerWithRedis): def get_user_latest_country( self, user: UserRef | User, - geoip_info_manager: GeoIpInfoManager, exclude_anon: bool = False, ) -> str | None: """Get the country the user is in, based off their latest ip.""" - ipr = self.get_user_latest_ip_record( - user, geoip_info_manager=geoip_info_manager, exclude_anon=exclude_anon - ) + ipr = self.get_user_latest_ip_record(user, exclude_anon=exclude_anon) # The ipr.information should exist, but it is possible the user has # no IP history at all, so the record is None return ipr.country_iso if ipr is not None else None - def is_user_anonymous( - self, user: UserRef | User, geoip_info_manager: GeoIpInfoManager - ) -> bool | None: + def is_user_anonymous(self, user: UserRef | User) -> bool | None: # Get the user's latest ip. is it marked as anonymous? # Return None if the user has no IP history at all - ipr = self.get_user_latest_ip_record( - user, geoip_info_manager=geoip_info_manager - ) + ipr = self.get_user_latest_ip_record(user) if ipr: return ipr.is_anonymous if ipr.is_anonymous is not None else False return None @@ -175,11 +163,12 @@ class IPRecordManager(PostgresManagerWithRedis): redis_config=self.redis_config, cache_prefix=self.cache_prefix, permissions=self.permissions, + geoip_info_manager=None, ) def create_unpack( self, - user: UserRef, + user_id: int, ip: IPvAnyAddressStr, forwarded_ips: list[str], ) -> IPRecord: @@ -188,22 +177,22 @@ class IPRecordManager(PostgresManagerWithRedis): padded = list(forwarded_ips) + [None] * (6 - len(forwarded_ips)) - return self.create(user, ip, *padded) + return self.create(user_id, ip, *padded) def create( self, - user: UserRef, + user_id: int, ip: IPvAnyAddressStr, - forwarded_ip1: IPvAnyAddressStr, - forwarded_ip2: IPvAnyAddressStr, - forwarded_ip3: IPvAnyAddressStr, - forwarded_ip4: IPvAnyAddressStr, - forwarded_ip5: IPvAnyAddressStr, - forwarded_ip6: IPvAnyAddressStr, + forwarded_ip1: IPvAnyAddressStr | None, + forwarded_ip2: IPvAnyAddressStr | None, + forwarded_ip3: IPvAnyAddressStr | None, + forwarded_ip4: IPvAnyAddressStr | None, + forwarded_ip5: IPvAnyAddressStr | None, + forwarded_ip6: IPvAnyAddressStr | None, ) -> IPRecord: data = { - "user_id": user.user_id, + "user_id": user_id, "ip": ipaddress.ip_address(ip).exploded, "created": datetime.now(tz=UTC), } @@ -245,7 +234,7 @@ class IPRecordManager(PostgresManagerWithRedis): """, params=data, ) - self.recreate_user_ip_history_cache(user=user) + self.delete_user_ip_history_cache(user_id=user_id) return IPRecord.from_mysql(data) @@ -310,8 +299,10 @@ class IPRecordManager(PostgresManagerWithRedis): return [IPRecord.from_mysql(i) for i in res] - def recreate_user_ip_history_cache(self, user: UserRef): - return self.user_ip_history_manager.recreate_user_ip_history_cache(user=user) + def delete_user_ip_history_cache(self, user_id: int): + return self.user_ip_history_manager.delete_user_ip_history_cache( + user_id=user_id + ) class AuditLogManager(PostgresManager): diff --git a/generalresearch/models/thl/ipinfo.py b/generalresearch/models/thl/ipinfo.py index 9fe86e4..0dff6b4 100644 --- a/generalresearch/models/thl/ipinfo.py +++ b/generalresearch/models/thl/ipinfo.py @@ -10,9 +10,9 @@ from pydantic import ( BaseModel, ConfigDict, Field, + IPvAnyAddress, PositiveInt, field_validator, - IPvAnyAddress, ) from generalresearch.models.custom_types import ( @@ -122,11 +122,13 @@ class IPGeoname(BaseModel): return cls.model_validate(d) -class IPInformation(BaseModel): +class GeoIPInformation(BaseModel): """ Fields we'll always pull from GRIP's mmdb files at minimum """ + model_config = ConfigDict(extra="ignore") + ip: IPvAnyAddressStr = Field() country_iso: CountryISOLike | None = Field( @@ -162,7 +164,3 @@ class IPInformation(BaseModel): "(e.g., 'residential', 'business').", examples=[AccessType.RESIDENTIAL], ) - - -class GeoIPInformation(IPInformation): - model_config = ConfigDict(extra="ignore") diff --git a/test_utils/managers/conftest.py b/test_utils/managers/conftest.py index 391e6bf..72be79e 100644 --- a/test_utils/managers/conftest.py +++ b/test_utils/managers/conftest.py @@ -1,19 +1,33 @@ from __future__ import annotations import random +from collections import defaultdict from collections.abc import Callable from datetime import datetime from typing import TYPE_CHECKING +from unittest.mock import Mock from uuid import uuid4 import pytest +from grip_client.mmdb.models import ( + GRIPAnonymousRecord, + GRIPAsnRecord, + GRIPCountryRecord, + GRIPMMDBLookupResult, +) from generalresearch.managers.thl.cashout_method import ( CashoutMethodManager, ) +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager from generalresearch.managers.thl.user_streak import ( UserStreakManager, ) +from generalresearch.managers.thl.userhealth import ( + AuditLogManager, + IPRecordManager, + UserIpHistoryManager, +) from generalresearch.models.definitions import Source from generalresearch.models.thl.wallet.cashout_method import ( CashoutMethod, @@ -24,15 +38,6 @@ from generalresearch.models.thl.wallet.definitions import Currency, PayoutType if TYPE_CHECKING: from generalresearch.managers.spectrum.survey import SpectrumSurveyManager from generalresearch.managers.thl.buyer import BuyerManager - from generalresearch.managers.thl.ipinfo import ( - GeoIpInfoManager, - IPGeonameManager, - ) - from generalresearch.managers.thl.userhealth import ( - AuditLogManager, - IPRecordManager, - UserIpHistoryManager, - ) from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig @@ -51,41 +56,39 @@ def audit_log_manager(thl_web_rw: PostgresConfig) -> AuditLogManager: return AuditLogManager(pg_config=thl_web_rw) -@pytest.fixture(scope="session") -def ip_geoname_manager(thl_web_rw: PostgresConfig) -> IPGeonameManager: - assert thl_web_rw.dsn.path - assert "/unittest-" in thl_web_rw.dsn.path - - from generalresearch.managers.thl.ipinfo import IPGeonameManager - - return IPGeonameManager(pg_config=thl_web_rw) - - -@pytest.fixture(scope="session") +@pytest.fixture def ip_record_manager( - thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig + thl_web_rw: PostgresConfig, + thl_redis_config: RedisConfig, + geoip_info_manager: GeoIpInfoManager, ) -> IPRecordManager: assert thl_web_rw.dsn.path assert "/unittest-" in thl_web_rw.dsn.path from generalresearch.managers.thl.userhealth import IPRecordManager - return IPRecordManager(pg_config=thl_web_rw, redis_config=thl_redis_config) + return IPRecordManager( + pg_config=thl_web_rw, + redis_config=thl_redis_config, + geoip_info_manager=geoip_info_manager, + ) -@pytest.fixture(scope="session") +@pytest.fixture def user_iphistory_manager( - thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig + thl_web_rw: PostgresConfig, + thl_redis_config: RedisConfig, + geoip_info_manager: GeoIpInfoManager, ) -> UserIpHistoryManager: assert thl_web_rw.dsn.path assert "/unittest-" in thl_web_rw.dsn.path - from generalresearch.managers.thl.userhealth import ( - UserIpHistoryManager, + return UserIpHistoryManager( + pg_config=thl_web_rw, + redis_config=thl_redis_config, + geoip_info_manager=geoip_info_manager, ) - return UserIpHistoryManager(pg_config=thl_web_rw, redis_config=thl_redis_config) - @pytest.fixture(scope="function") def user_iphistory_manager_clear_cache(user_iphistory_manager, user: User): @@ -96,16 +99,42 @@ def user_iphistory_manager_clear_cache(user_iphistory_manager, user: User): user_iphistory_manager.delete_user_ip_history_cache(user_id=user.user_id) -@pytest.fixture(scope="session") -def geoipinfo_manager( - thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig -) -> GeoIpInfoManager: - assert thl_web_rw.dsn.path - assert "/unittest-" in thl_web_rw.dsn.path +@pytest.fixture +def grip_lookup_results() -> dict[str, GRIPMMDBLookupResult]: + return defaultdict( + lambda: GRIPMMDBLookupResult( + country=GRIPCountryRecord(), + anonymous=GRIPAnonymousRecord(), + asn=GRIPAsnRecord(), + ), + { + "8.8.8.8": GRIPMMDBLookupResult( + country=GRIPCountryRecord(country_iso="US"), + anonymous=GRIPAnonymousRecord(is_anonymous=False), + asn=GRIPAsnRecord( + asn=15169, + network_operator="Google", + ), + ), + "1.1.1.1": GRIPMMDBLookupResult( + country=GRIPCountryRecord(country_iso="AU"), + anonymous=GRIPAnonymousRecord(is_anonymous=True), + asn=GRIPAsnRecord( + asn=13335, + network_operator="Cloudflare", + ), + ), + }, + ) - from generalresearch.managers.thl.ipinfo import GeoIpInfoManager - return GeoIpInfoManager(pg_config=thl_web_rw, redis_config=thl_redis_config) +@pytest.fixture(scope="function") +def geoip_info_manager( + grip_lookup_results: dict[str, GRIPMMDBLookupResult], +) -> GeoIpInfoManager: + manager = GeoIpInfoManager(grip_token="test-token") + manager.grip_mmdb.lookup = Mock(side_effect=grip_lookup_results.__getitem__) + return manager @pytest.fixture(scope="session") @@ -193,10 +222,10 @@ def example_tango_cashout_methods( last_updated=datetime.fromisoformat("2021-06-23T20:45:38.239182Z"), is_live=True, type=PayoutType.TANGO, - ext_id='U025035', + ext_id="U025035", name="Safeway eGift Card $25", data=TangoCashoutMethodData( - value_type="fixed", countries=["US"], utid='U025035' + value_type="fixed", countries=["US"], utid="U025035" ), user=None, image_url="https://d30s7yzk2az89n.cloudfront.net/images/brands/b694446-1200w-326ppi.png", @@ -209,7 +238,7 @@ def example_tango_cashout_methods( last_updated=datetime.fromisoformat("2021-06-23T20:45:38.239182Z"), is_live=True, type=PayoutType.TANGO, - ext_id='U006961', + ext_id="U006961", name="Amazon.it Gift Certificate", data=TangoCashoutMethodData( value_type="variable", countries=["IT"], utid="U006961" diff --git a/test_utils/models/thl/conftest.py b/test_utils/models/thl/conftest.py index d20fdc4..433003c 100644 --- a/test_utils/models/thl/conftest.py +++ b/test_utils/models/thl/conftest.py @@ -14,10 +14,9 @@ from grip_client.enums import AccessType from pydantic import PositiveInt from generalresearch.currency import USDCent -from generalresearch.managers.thl.payout import ( - BusinessPayoutEventManager, - UserPayoutEventManager, -) +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager +from generalresearch.managers.thl.payout import BusinessPayoutEventManager +from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager from generalresearch.models.custom_types import ( AwareDatetimeISO, IPvAnyAddressStr, @@ -29,6 +28,7 @@ from generalresearch.models.thl.definitions import ( PayoutStatus, Status, ) +from generalresearch.models.thl.ipinfo import GeoIPInformation from generalresearch.models.thl.payout import UserPayoutEvent from generalresearch.models.thl.user import User from generalresearch.models.thl.userhealth import AuditLogLevel @@ -36,10 +36,6 @@ from generalresearch.models.thl.wallet.definitions import PayoutType from generalresearch.pg_helper import PostgresConfig if TYPE_CHECKING: - from generalresearch.managers.thl.ipinfo import ( - IPGeonameManager, - IPInformationManager, - ) from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.managers.thl.product import ProductManager from generalresearch.managers.thl.session import SessionManager @@ -50,7 +46,6 @@ if TYPE_CHECKING: from generalresearch.models.gr.business import Business from generalresearch.models.gr.team import Team from generalresearch.models.legacy.bucket import Bucket - from generalresearch.models.thl.ipinfo import IPGeoname, IPInformation from generalresearch.models.thl.payout import BrokerageProductPayoutEvent from generalresearch.models.thl.product import ( PayoutConfig, @@ -438,143 +433,44 @@ def unsaved_product(product_factory: Callable[..., Product]) -> Product: return product_factory(save=False) -# --- IP Geoname --- - - -@pytest.fixture -def ip_geoname_factory( - ip_geoname_manager: IPGeonameManager, -) -> Callable[..., IPGeoname]: - - def _inner( - save: bool = True, - geoname_id: PositiveInt | None = None, - continent_code: str | None = None, - continent_name: str | None = None, - country_iso: str | None = None, - country_name: str | None = None, - subdivision_1_iso: str | None = None, - subdivision_1_name: str | None = None, - subdivision_2_iso: str | None = None, - subdivision_2_name: str | None = None, - city_name: str | None = None, - metro_code: int | None = None, - time_zone: str | None = None, - is_in_european_union: bool | None = None, - ) -> IPGeoname: - if save: - return ip_geoname_manager.create( - geoname_id=geoname_id or randint(1, 999_999_999), - continent_code=continent_code or "na", - continent_name=continent_name or "North America", - country_iso=country_iso or "us", - country_name=country_name or "United States", - subdivision_1_iso=subdivision_1_iso or "fl", - subdivision_1_name=subdivision_1_name or "Florida", - subdivision_2_iso=subdivision_2_iso, - subdivision_2_name=subdivision_2_name, - city_name=city_name, - metro_code=metro_code, - time_zone=time_zone, - is_in_european_union=is_in_european_union, - ) - else: - raise ValueError("Unsaved IPGeoname not yet supported") - - return _inner - - -@pytest.fixture() -def ip_geoname(ip_geoname_factory: Callable[..., IPGeoname]) -> IPGeoname: - return ip_geoname_factory(save=True) - - -@pytest.fixture() -def unsaved_ip_geoname(ip_geoname_factory: Callable[..., IPGeoname]) -> IPGeoname: - return ip_geoname_factory(save=True) - - -# --- IP Information --- +# --- GeoIP Information --- @pytest.fixture -def ip_information_factory( - ip_information_manager: IPInformationManager, -) -> Callable[..., IPInformation]: +def geoip_information_factory( + geoip_info_manager: GeoIpInfoManager, +) -> Callable[..., GeoIPInformation]: def _inner( - save: bool = True, ip: IPvAnyAddressStr | None = None, - geoname_id: PositiveInt | None = None, country_iso: str | None = None, - registered_country_iso: str | None = None, + country_name: str | None = None, is_anonymous: bool | None = None, - is_anonymous_vpn: bool | None = None, - is_hosting_provider: bool | None = None, - is_public_proxy: bool | None = None, - is_tor_exit_node: bool | None = None, - is_residential_proxy: bool | None = None, autonomous_system_number: PositiveInt | None = None, autonomous_system_organization: str | None = None, - domain: str | None = None, - isp: str | None = None, - mobile_country_code: str | None = None, - mobile_network_code: str | None = None, - network: str | None = None, - organization: str | None = None, - static_ip_score: float | None = None, - user_type: AccessType | None = None, - postal_code: str | None = None, - latitude: Decimal | None = None, - longitude: Decimal | None = None, - accuracy_radius: int | None = None, - ) -> IPInformation: - - if save: - return ip_information_manager.create( - ip=ip or fake.ipv4_public(), - geoname_id=geoname_id, - country_iso=country_iso or fake.country_code(), - registered_country_iso=registered_country_iso, - is_anonymous=is_anonymous, - is_anonymous_vpn=is_anonymous_vpn, - is_hosting_provider=is_hosting_provider, - is_public_proxy=is_public_proxy, - is_tor_exit_node=is_tor_exit_node, - is_residential_proxy=is_residential_proxy, - autonomous_system_number=autonomous_system_number, - autonomous_system_organization=autonomous_system_organization, - domain=domain, - isp=isp, - mobile_country_code=mobile_country_code, - mobile_network_code=mobile_network_code, - network=network, - organization=organization, - static_ip_score=static_ip_score, - user_type=user_type, - postal_code=postal_code, - latitude=latitude, - longitude=longitude, - accuracy_radius=accuracy_radius, - ) - else: - raise ValueError("Unsaved IP Information not supported yet") + access_type: AccessType | None = None, + ) -> GeoIPInformation: + + return GeoIPInformation( + country_iso=country_iso or fake.country_code("alpha-2"), + access_type=access_type, + is_anonymous=is_anonymous, + autonomous_system_number=autonomous_system_number, + autonomous_system_organization=autonomous_system_organization, + country_name=country_name, + ip=ip or fake.ipv4_public(), + subdivision_1_iso=None, + subdivision_1_name=None, + ) return _inner @pytest.fixture -def ip_information( - ip_information_factory: Callable[..., IPInformation], -) -> IPInformation: - return ip_information_factory(save=True) - - -@pytest.fixture() -def unsaved_ip_information( - ip_information_factory: Callable[..., IPInformation], -) -> IPInformation: - return ip_information_factory(save=False) +def geoip_information( + geoip_information_factory: Callable[..., GeoIPInformation], +) -> GeoIPInformation: + return geoip_information_factory() # --- IP Record --- @@ -584,7 +480,7 @@ def unsaved_ip_information( def ip_record_factory(ip_record_manager: IPRecordManager) -> Callable[..., IPRecord]: def _inner( - user_id: PositiveInt, + user: User, save: bool = True, ip: IPvAnyAddressStr | None = None, forwarded_ip1: IPvAnyAddressStr | None = None, @@ -597,15 +493,13 @@ def ip_record_factory(ip_record_manager: IPRecordManager) -> Callable[..., IPRec if save: return ip_record_manager.create( - user_id=user_id, + user_id=user.to_user_ref().user_id, ip=ip or fake.ipv4_public(), - forwarded_ip1=(forwarded_ip1 or fake.ipv4_public()), - forwarded_ip2=( - forwarded_ip2 or fake.ipv6() if random() < 0.5 else None - ), - forwarded_ip3=( - forwarded_ip3 or fake.ipv4_public() if random() < 0.25 else None - ), + forwarded_ip1=forwarded_ip1 or fake.ipv4_public(), + forwarded_ip2=forwarded_ip2 or fake.ipv6() if random() < 0.5 else None, + forwarded_ip3=forwarded_ip3 or fake.ipv4_public() + if random() < 0.25 + else None, forwarded_ip4=forwarded_ip4, forwarded_ip5=forwarded_ip5, forwarded_ip6=forwarded_ip6, diff --git a/tests/managers/thl/test_ipinfo.py b/tests/managers/thl/test_ipinfo.py index c021eb9..f39d50e 100644 --- a/tests/managers/thl/test_ipinfo.py +++ b/tests/managers/thl/test_ipinfo.py @@ -1,160 +1,47 @@ -from collections.abc import Callable -from typing import TYPE_CHECKING - -import faker - -from generalresearch.managers.thl.ipinfo import ( - GeoIpInfoManager, - IPGeonameManager, - IPInformationManager, -) -from generalresearch.models.thl.ipinfo import ( - GeoIPInformation, - IPGeoname, - IPInformation, -) - -if TYPE_CHECKING: - from generalresearch.pg_helper import PostgresConfig - from generalresearch.redis_helper import RedisConfig - -fake = faker.Faker() - - -class TestIPGeonameManager: - - def test_init( - self, thl_web_rr: PostgresConfig, ip_geoname_manager: IPGeonameManager - ): - - instance = IPGeonameManager(pg_config=thl_web_rr) - assert isinstance(instance, IPGeonameManager) - assert isinstance(ip_geoname_manager, IPGeonameManager) - - def test_create( - self, - ip_geoname_factory: Callable[..., IPGeoname], - ip_geoname_manager: IPGeonameManager, - ): - - instance = ip_geoname_factory() - assert isinstance(instance, IPGeoname) - - res = ip_geoname_manager.fetch_geoname_ids(filter_ids=[instance.geoname_id]) - assert res[0].model_dump_json() == instance.model_dump_json() - - -class TestIPInformationManager: - - def test_init( - self, thl_web_rr: PostgresConfig, ip_information_manager: IPInformationManager - ): - instance = IPInformationManager(pg_config=thl_web_rr) - assert isinstance(instance, IPInformationManager) - assert isinstance(ip_information_manager, IPInformationManager) - - def test_create( - self, - ip_information_factory: Callable[..., IPInformation], - ip_information_manager: IPInformationManager, - ): - instance = ip_information_factory() - assert isinstance(instance, IPInformation) - - res = ip_information_manager.fetch_ip_information(filter_ips=[instance.ip]) - assert res[0].model_dump_json() == instance.model_dump_json() - - def test_prefetch_geoname( - self, - ip_information: IPInformation, - ip_geoname: IPGeoname, - thl_web_rr: PostgresConfig, - ): - assert isinstance(ip_information, IPInformation) - - assert ip_information.geoname_id == ip_geoname.geoname_id - assert ip_information.geoname is None - - ip_information.prefetch_geoname(pg_config=thl_web_rr) - assert isinstance(ip_information.geoname, IPGeoname) +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager +from generalresearch.models.thl.ipinfo import GeoIPInformation class TestGeoIpInfoManager: - def test_init( - self, - thl_web_rr: PostgresConfig, - thl_redis_config: RedisConfig, - geoipinfo_manager: GeoIpInfoManager, - ): - instance = GeoIpInfoManager(pg_config=thl_web_rr, redis_config=thl_redis_config) - assert isinstance(instance, GeoIpInfoManager) - assert isinstance(geoipinfo_manager, GeoIpInfoManager) - - def test_multi( - self, - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, - geoipinfo_manager: GeoIpInfoManager, - ): - ip = fake.ipv4_public() - ip_information_factory(ip=ip, geoname=ip_geoname) - ips = [ip] - - # This only looks up in redis. They don't exist yet - res = geoipinfo_manager.get_cache_multi(ip_addresses=ips) - assert res == {ip: None} - - # Looks up in redis, if not exists, looks in mysql, then sets - # the caches that didn't exist. - res = geoipinfo_manager.get_multi(ip_addresses=ips) - assert res[ip] is not None - - ip2 = fake.ipv4_public() - ip_information_factory(ip=ip2, geoname=ip_geoname) - ips = [ip, ip2] - res = geoipinfo_manager.get_cache_multi(ip_addresses=ips) - assert res[ip] is not None - assert res[ip2] is None - res = geoipinfo_manager.get_multi(ip_addresses=ips) - assert res[ip] is not None - assert res[ip2] is not None - res = geoipinfo_manager.get_cache_multi(ip_addresses=ips) - assert res[ip] is not None - assert res[ip2] is not None - - def test_multi_ipv6( - self, - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, - geoipinfo_manager: GeoIpInfoManager, - ): - ip = fake.ipv6() - # Make another IP that will be in the same /64 block. - ip2 = ip[:-1] + "a" if ip[-1] != "a" else ip[:-1] + "b" - ip_information_factory(ip=ip, geoname=ip_geoname) - ips = [ip, ip2] - print(f"{ips=}") - - # This only looks up in redis. They don't exist yet - res = geoipinfo_manager.get_cache_multi(ip_addresses=ips) - assert res == {ip: None, ip2: None} - - # Looks up in redis, if not exists, looks in mysql, then sets - # the caches that didn't exist. - res = geoipinfo_manager.get_multi(ip_addresses=ips) - - res1 = res[ip] - assert isinstance(res1, GeoIPInformation) - assert res1.ip == ip - assert res1.lookup_prefix == "/64" - - res2 = res[ip2] - assert isinstance(res2, GeoIPInformation) - assert res2.ip == ip2 - assert res2.lookup_prefix == "/64" - # they should be the same basically, except for the ip - - def test_doesnt_exist(self, geoipinfo_manager: GeoIpInfoManager): - ip = fake.ipv4_public() - res = geoipinfo_manager.get_multi(ip_addresses=[ip]) - assert res == {ip: None} + def test_get(self, geoipinfo_manager: GeoIpInfoManager): + result = geoip_info_manager.get("8.8.8.8") + + assert result == GeoIPInformation( + ip="8.8.8.8", + country_iso="us", + is_anonymous=False, + autonomous_system_number=15169, + autonomous_system_organization="Google", + access_type=None, + ) + geoip_info_manager.grip_mmdb.lookup.assert_called_once_with("8.8.8.8") + + def test_get_multi(self, geoipinfo_manager: GeoIpInfoManager): + result = geoip_info_manager.get_multi(["8.8.8.8", "1.1.1.1", "8.8.8.8"]) + + assert result == { + "8.8.8.8": GeoIPInformation( + ip="8.8.8.8", + country_iso="us", + is_anonymous=False, + autonomous_system_number=15169, + autonomous_system_organization="Google", + access_type=None, + ), + "1.1.1.1": GeoIPInformation( + ip="1.1.1.1", + country_iso="au", + is_anonymous=True, + autonomous_system_number=13335, + autonomous_system_organization="Cloudflare", + access_type=None, + ), + } + assert geoip_info_manager.grip_mmdb.lookup.call_count == 2 + assert { + call.args[0] for call in geoip_info_manager.grip_mmdb.lookup.call_args_list + } == {"8.8.8.8", "1.1.1.1"} + + def test_get_multi_empty(self, geoipinfo_manager: GeoIpInfoManager): + assert geoip_info_manager.get_multi([]) == {} + geoip_info_manager.grip_mmdb.lookup.assert_not_called() diff --git a/tests/managers/thl/test_userhealth.py b/tests/managers/thl/test_userhealth.py index 268b110..8256185 100644 --- a/tests/managers/thl/test_userhealth.py +++ b/tests/managers/thl/test_userhealth.py @@ -3,11 +3,13 @@ from __future__ import annotations from collections.abc import Callable from datetime import UTC, datetime from typing import TYPE_CHECKING +from unittest.mock import Mock from uuid import uuid4 import faker import pytest +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager from generalresearch.managers.thl.userhealth import ( AuditLogManager, IPRecordManager, @@ -23,10 +25,6 @@ from generalresearch.models.thl.user_iphistory import ( from generalresearch.models.thl.userhealth import AuditLog, AuditLogLevel if TYPE_CHECKING: - from generalresearch.models.thl.ipinfo import ( - IPGeoname, - IPInformation, - ) from generalresearch.models.thl.product import Product from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig @@ -36,7 +34,6 @@ fake = faker.Faker() class TestAuditLog: - def test_init(self, thl_web_rr: PostgresConfig, audit_log_manager: AuditLogManager): alm = AuditLogManager(pg_config=thl_web_rr) @@ -56,7 +53,7 @@ class TestAuditLog: user_id=user.user_id, level=level, event_type=uuid4().hex ) assert isinstance(instance, AuditLog) - assert instance.id != 1 + assert instance.id != 0 def test_get_by_id(self, audit_log: AuditLog, audit_log_manager: AuditLogManager): @@ -225,14 +222,18 @@ class TestAuditLog: class TestIPRecordManager: - def test_init( self, thl_web_rr: PostgresConfig, thl_redis_config: RedisConfig, + geoip_info_manager: GeoIpInfoManager, ip_record_manager: IPRecordManager, ): - instance = IPRecordManager(pg_config=thl_web_rr, redis_config=thl_redis_config) + instance = IPRecordManager( + pg_config=thl_web_rr, + redis_config=thl_redis_config, + geoip_info_manager=geoip_info_manager, + ) assert isinstance(instance, IPRecordManager) assert isinstance(ip_record_manager, IPRecordManager) @@ -240,10 +241,11 @@ class TestIPRecordManager: self, ip_record_manager: IPRecordManager, user: User, - ip_information: IPInformation, ip_record_factory: Callable[..., IPRecord], ): - instance = ip_record_factory(user_id=user.user_id, ip=ip_information.ip) + ip = fake.ipv4_public() + + instance = ip_record_factory(user=user, ip=ip) assert isinstance(instance, IPRecord) assert isinstance(instance.forwarded_ips, list) @@ -260,40 +262,12 @@ class TestIPRecordManager: def test_prefetch_info( self, ip_record_factory: Callable[..., IPRecord], - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, user: User, thl_web_rr: PostgresConfig, thl_redis_config: RedisConfig, ): - - ip = fake.ipv4_public() - ip_information_factory(ip=ip, geoname=ip_geoname) - ipr: IPRecord = ip_record_factory(user_id=user.user_id, ip=ip) - assert isinstance(ipr, IPRecord) - - assert ipr.information is None - assert len(ipr.forwarded_ip_records) >= 1 - assert isinstance(ipr.forwarded_ip_records, list) - fipr = ipr.forwarded_ip_records[0] - assert fipr.information is None - - ipr.prefetch_ipinfo( - pg_config=thl_web_rr, - redis_config=thl_redis_config, - include_forwarded=True, - ) - assert isinstance(ipr.information, GeoIPInformation) - assert ipr.information.ip == ipr.ip == ip - assert fipr.information is None, "the ipinfo doesn't exist in the db yet" - - ip_information_factory(ip=fipr.ip, geoname=ip_geoname) - ipr.prefetch_ipinfo( - pg_config=thl_web_rr, - redis_config=thl_redis_config, - include_forwarded=True, - ) - assert fipr.information is not None + # No more prefetch info here. Moved into UserIPHistory.enrich_ips + pass @pytest.mark.usefixtures("user_iphistory_manager_clear_cache") @@ -302,57 +276,86 @@ class TestUserIpHistoryManager: self, thl_web_rr: PostgresConfig, thl_redis_config: RedisConfig, + geoip_info_manager: GeoIpInfoManager, user_iphistory_manager: UserIpHistoryManager, ): instance = UserIpHistoryManager( - pg_config=thl_web_rr, redis_config=thl_redis_config + pg_config=thl_web_rr, + redis_config=thl_redis_config, + geoip_info_manager=geoip_info_manager, ) assert isinstance(instance, UserIpHistoryManager) assert isinstance(user_iphistory_manager, UserIpHistoryManager) - def test_latest_record( + def test_latest_record_and_enrich( self, user_iphistory_manager: UserIpHistoryManager, user: User, ip_record_factory: Callable[..., IPRecord], - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, + geoip_information_factory: Callable[..., GeoIPInformation], + geoip_info_manager: GeoIpInfoManager, ): ip = fake.ipv4_public() - ip_information_factory(ip=ip, geoname_id=ip_geoname.geoname_id, is_anonymous=True) - ipr1: IPRecord = ip_record_factory(user_id=user.user_id, ip=ip) + information = geoip_information_factory( + ip=ip, is_anonymous=True, country_iso="de" + ) + lookup_results = {ip: information} + geoip_info_manager.get_multi = Mock( + side_effect=lambda ip_addresses: { + address: lookup_results[address] for address in ip_addresses + } + ) - ipr = user_iphistory_manager.get_user_latest_ip_record(user=user) - assert isinstance(ipr, IPRecord) + ipr1 = ip_record_factory(user=user, ip=ip) + ipr = user_iphistory_manager.get_user_latest_ip_record( + user=user, + ) assert ipr.ip == ipr1.ip assert ipr.is_anonymous assert isinstance(ipr.information, GeoIPInformation) - assert ipr.information.lookup_prefix == "/32" - ip = fake.ipv6() - ip_information_factory(ip=ip, geoname_id=ip_geoname.geoname_id) - ipr2: IPRecord = ip_record_factory(user_id=user.user_id, ip=ip) + assert ( + user_iphistory_manager.get_user_latest_country( + user=user, + ) + == "de" + ) + + ip2 = fake.ipv6() + ipr2: IPRecord = ip_record_factory(user=user, ip=ip2) + lookup_results[ipr2.ip] = geoip_information_factory( + ip=ipr2.ip, + country_iso="us", + is_anonymous=False, + ) - ipr = user_iphistory_manager.get_user_latest_ip_record(user=user) - assert isinstance(ipr, IPRecord) + ipr = user_iphistory_manager.get_user_latest_ip_record( + user=user, + ) assert ipr.ip == ipr2.ip assert isinstance(ipr.information, GeoIPInformation) - assert ipr.information.lookup_prefix == "/64" assert ipr.information is not None assert not ipr.is_anonymous - country_iso = user_iphistory_manager.get_user_latest_country(user=user) - assert country_iso == ip_geoname.country_iso + assert ( + user_iphistory_manager.get_user_latest_country( + user=user, + ) + == "us" + ) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) + iph = user_iphistory_manager.get_user_ip_history(user=user) assert isinstance(iph, UserIPHistory) assert isinstance(iph.ips, list) assert iph.ips[0].information is not None assert iph.ips[1].information is not None - assert iph.ips[0].country_iso == country_iso - assert iph.ips[0].is_anonymous - assert iph.ips[0].ip == ipr1.ip - assert iph.ips[1].ip == ipr2.ip + assert iph.ips[0].country_iso == "us" + assert iph.ips[1].country_iso == "de" + assert not iph.ips[0].is_anonymous + assert iph.ips[1].is_anonymous + # ordered by created DESCENDING!!!!!!!!!!!!!!1 + assert iph.ips[0].ip == ipr2.ip + assert iph.ips[1].ip == ipr1.ip def test_virgin( self, @@ -360,65 +363,9 @@ class TestUserIpHistoryManager: user_iphistory_manager: UserIpHistoryManager, ip_record_factory: Callable[..., IPRecord], ): - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) + iph = user_iphistory_manager.get_user_ip_history(user=user) assert len(iph.ips) == 0 - ip_record_factory(user_id=user.user_id, ip=fake.ipv4_public()) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) + ip_record_factory(user=user, ip=fake.ipv4_public()) + iph = user_iphistory_manager.get_user_ip_history(user=user) assert len(iph.ips) == 1 - - def test_out_of_order( - self, - ip_record_factory: Callable[..., IPRecord], - user: User, - user_iphistory_manager: UserIpHistoryManager, - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, - ): - # Create the user-ip association BEFORE the ip even exists in the ipinfo table - ip = fake.ipv4_public() - ip_record_factory(user_id=user.user_id, ip=ip) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) - assert isinstance(iph, UserIPHistory) - assert isinstance(iph.ips, list) - assert len(iph.ips) == 1 - ipr = iph.ips[0] - assert ipr.information is None - assert not ipr.is_anonymous - - ip_information_factory(ip=ip, geoname_id=ip_geoname.geoname_id, is_anonymous=True) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) - assert isinstance(iph, UserIPHistory) - assert isinstance(iph.ips, list) - assert len(iph.ips) == 1 - ipr = iph.ips[0] - assert ipr.information is not None - assert ipr.is_anonymous - - def test_out_of_order_ipv6( - self, - ip_record_factory: Callable[..., IPRecord], - user: User, - user_iphistory_manager: UserIpHistoryManager, - ip_information_factory: Callable[..., IPInformation], - ip_geoname: IPGeoname, - ): - # Create the user-ip association BEFORE the ip even exists in the ipinfo table - ip = fake.ipv6() - ip_record_factory(user_id=user.user_id, ip=ip) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) - assert isinstance(iph, UserIPHistory) - assert isinstance(iph.ips, list) - assert len(iph.ips) == 1 - ipr = iph.ips[0] - assert ipr.information is None - assert not ipr.is_anonymous - - ip_information_factory(ip=ip, geoname_id=ip_geoname.geoname_id, is_anonymous=True) - iph = user_iphistory_manager.get_user_ip_history(user_id=user.user_id) - assert isinstance(iph, UserIPHistory) - assert isinstance(iph.ips, list) - assert len(iph.ips) == 1 - ipr = iph.ips[0] - assert ipr.information is not None - assert ipr.is_anonymous -- cgit v1.2.3 From 1a98418317c115b92739639cf0b837bd974fc5d6 Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 11:34:27 -0600 Subject: fix more tests --- generalresearch/managers/thl/payout.py | 2 +- .../managers/thl/wallet/cashout_tasks.py | 3 --- generalresearch/managers/thl/wallet/user_payout.py | 4 ++- test_utils/managers/thl/conftest.py | 31 ++-------------------- test_utils/models/ledger/conftest.py | 16 +++++------ tests/managers/thl/test_cashout_method.py | 4 --- tests/managers/thl/test_ipinfo.py | 6 ++--- tests/managers/thl/test_ledger/test_thl_lm_tx.py | 27 +++++++++++++++++++ tests/managers/thl/test_ledger/test_thl_pem.py | 5 ++-- tests/managers/thl/test_payout.py | 19 +++++++------ 10 files changed, 56 insertions(+), 61 deletions(-) delete mode 100644 generalresearch/managers/thl/wallet/cashout_tasks.py diff --git a/generalresearch/managers/thl/payout.py b/generalresearch/managers/thl/payout.py index 4014cf8..3968ea7 100644 --- a/generalresearch/managers/thl/payout.py +++ b/generalresearch/managers/thl/payout.py @@ -879,7 +879,7 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis): amount: USDCent, ext_ref_id: str, created: datetime | None = None, - ): + ) -> BusinessPayoutEvent: """ This should NOT be called directly normally. It is just a shortcut for tests. However, instead of just making a naked BP payout, diff --git a/generalresearch/managers/thl/wallet/cashout_tasks.py b/generalresearch/managers/thl/wallet/cashout_tasks.py deleted file mode 100644 index b28b04f..0000000 --- a/generalresearch/managers/thl/wallet/cashout_tasks.py +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/generalresearch/managers/thl/wallet/user_payout.py b/generalresearch/managers/thl/wallet/user_payout.py index ca47611..a1c628b 100644 --- a/generalresearch/managers/thl/wallet/user_payout.py +++ b/generalresearch/managers/thl/wallet/user_payout.py @@ -11,7 +11,9 @@ from redis import Redis from generalresearch.currency import USDCent from generalresearch.managers.thl.cashout_method import CashoutMethodManager from generalresearch.managers.thl.ipinfo import GeoIpInfoManager -from generalresearch.managers.thl.ledger_manager.exceptions import LedgerTransactionCreateError +from generalresearch.managers.thl.ledger_manager.exceptions import ( + LedgerTransactionCreateError, +) from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.managers.thl.payout import PayoutEventManager from generalresearch.managers.thl.userhealth import UserIpHistoryManager diff --git a/test_utils/managers/thl/conftest.py b/test_utils/managers/thl/conftest.py index 3392559..5b96680 100644 --- a/test_utils/managers/thl/conftest.py +++ b/test_utils/managers/thl/conftest.py @@ -23,15 +23,10 @@ if TYPE_CHECKING: from generalresearch.config import GRLBaseSettings from generalresearch.managers.thl.buyer import BuyerManager from generalresearch.managers.thl.category import CategoryManager - from generalresearch.managers.thl.ipinfo import ( - IPGeonameManager, - IPInformationManager, - ) from generalresearch.managers.thl.payout import ( BrokerageProductPayoutEventManager, BusinessPayoutEventManager, PayoutEventManager, - UserPayoutEventManager, ) from generalresearch.managers.thl.product import ProductManager from generalresearch.managers.thl.session import SessionManager @@ -52,6 +47,7 @@ if TYPE_CHECKING: WallCacheManager, WallManager, ) + from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager # === Msc === @@ -139,7 +135,7 @@ def user_payout_event_manager( assert thl_web_rw.dsn.path assert "/unittest-" in thl_web_rw.dsn.path - from generalresearch.managers.thl.payout import UserPayoutEventManager + from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager return UserPayoutEventManager( pg_config=thl_web_rw, @@ -342,29 +338,6 @@ def surveypenalty_manager(thl_redis_config: RedisConfig): return SurveyPenaltyManager(redis_config=thl_redis_config) -# --- IP Geolocation --- - - -@pytest.fixture -def ip_geoname_manager(thl_web_rw: PostgresConfig) -> IPGeonameManager: - from generalresearch.managers.thl.ipinfo import IPGeonameManager - - return IPGeonameManager(pg_config=thl_web_rw) - - -# --- IP Information --- - - -@pytest.fixture(scope="session") -def ip_information_manager(thl_web_rw: PostgresConfig) -> IPInformationManager: - assert thl_web_rw.dsn.path - assert "/unittest-" in thl_web_rw.dsn.path - - from generalresearch.managers.thl.ipinfo import IPInformationManager - - return IPInformationManager(pg_config=thl_web_rw) - - # --- IP Record --- diff --git a/test_utils/models/ledger/conftest.py b/test_utils/models/ledger/conftest.py index 9ee0df2..e9ddbc9 100644 --- a/test_utils/models/ledger/conftest.py +++ b/test_utils/models/ledger/conftest.py @@ -39,7 +39,6 @@ if TYPE_CHECKING: ThlLedgerManager, ) from generalresearch.managers.thl.payout import ( - BrokerageProductPayoutEventManager, BusinessPayoutEventManager, ) from generalresearch.managers.thl.session import SessionManager @@ -202,7 +201,7 @@ def bp_payout_event( @pytest.fixture def bp_payout_event_factory( - brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager, + business_payout_event_manager: BusinessPayoutEventManager, thl_ledger_manager: ThlLedgerManager, ) -> Callable[..., BrokerageProductPayoutEvent]: @@ -210,14 +209,13 @@ def bp_payout_event_factory( product: Product, usd_cent: USDCent, ext_ref_id: str | None = None ) -> BrokerageProductPayoutEvent: - return brokerage_product_payout_event_manager.create_bp_payout_event( + bus_pe = business_payout_event_manager.create_bp_payout_event( thl_ledger_manager=thl_ledger_manager, product=product, amount=usd_cent, - ext_ref_id=ext_ref_id, - skip_wallet_balance_check=True, - skip_one_per_day_check=True, + ext_ref_id=ext_ref_id or uuid4().hex, ) + return bus_pe.bp_payouts[0] return _inner @@ -225,9 +223,9 @@ def bp_payout_event_factory( @pytest.fixture def currency(ledger_manager: LedgerManager) -> LedgerCurrency: # return request.param if hasattr(request, "currency") else LedgerCurrency.TEST - assert ( - ledger_manager.currency - ), "LedgerManager must have a currency specified for these tests" + assert ledger_manager.currency, ( + "LedgerManager must have a currency specified for these tests" + ) return ledger_manager.currency diff --git a/tests/managers/thl/test_cashout_method.py b/tests/managers/thl/test_cashout_method.py index fc364f2..bb978f8 100644 --- a/tests/managers/thl/test_cashout_method.py +++ b/tests/managers/thl/test_cashout_method.py @@ -13,7 +13,6 @@ from generalresearch.models.thl.wallet.cashout_method import ( from generalresearch.models.thl.wallet.definitions import PayoutType if TYPE_CHECKING: - from generalresearch.config import GRLBaseSettings from generalresearch.managers.thl.cashout_method import ( CashoutMethodManager, ) @@ -24,7 +23,6 @@ if TYPE_CHECKING: class TestTangoCashoutMethods: - def test_create_and_get( self, cashout_method_manager: CashoutMethodManager, @@ -51,9 +49,7 @@ class TestTangoCashoutMethods: assert len(res) == 2 - class TestUserCashoutMethods: - def test( self, cashout_method_manager: CashoutMethodManager, diff --git a/tests/managers/thl/test_ipinfo.py b/tests/managers/thl/test_ipinfo.py index f39d50e..d84cbf7 100644 --- a/tests/managers/thl/test_ipinfo.py +++ b/tests/managers/thl/test_ipinfo.py @@ -3,7 +3,7 @@ from generalresearch.models.thl.ipinfo import GeoIPInformation class TestGeoIpInfoManager: - def test_get(self, geoipinfo_manager: GeoIpInfoManager): + def test_get(self, geoip_info_manager: GeoIpInfoManager): result = geoip_info_manager.get("8.8.8.8") assert result == GeoIPInformation( @@ -16,7 +16,7 @@ class TestGeoIpInfoManager: ) geoip_info_manager.grip_mmdb.lookup.assert_called_once_with("8.8.8.8") - def test_get_multi(self, geoipinfo_manager: GeoIpInfoManager): + def test_get_multi(self, geoip_info_manager: GeoIpInfoManager): result = geoip_info_manager.get_multi(["8.8.8.8", "1.1.1.1", "8.8.8.8"]) assert result == { @@ -42,6 +42,6 @@ class TestGeoIpInfoManager: call.args[0] for call in geoip_info_manager.grip_mmdb.lookup.call_args_list } == {"8.8.8.8", "1.1.1.1"} - def test_get_multi_empty(self, geoipinfo_manager: GeoIpInfoManager): + def test_get_multi_empty(self, geoip_info_manager: GeoIpInfoManager): assert geoip_info_manager.get_multi([]) == {} geoip_info_manager.grip_mmdb.lookup.assert_not_called() diff --git a/tests/managers/thl/test_ledger/test_thl_lm_tx.py b/tests/managers/thl/test_ledger/test_thl_lm_tx.py index 96ea0d6..22a7b73 100644 --- a/tests/managers/thl/test_ledger/test_thl_lm_tx.py +++ b/tests/managers/thl/test_ledger/test_thl_lm_tx.py @@ -806,6 +806,24 @@ class TestThlLedgerTxManager: assert thl_ledger_manager.get_session_attempt_credit(session.uuid) == 5 assert thl_ledger_manager.get_user_attempt_credit_balance(user) == 5 + attempt_credit_account = ( + thl_ledger_manager.get_account_or_create_user_attempt_credit(user) + ) + user_txs = thl_ledger_manager.get_user_txs( + user, account_uuid=attempt_credit_account.uuid + ) + assert user_txs.total == 1 + assert user_txs.transactions[0].tx_type == TransactionType.USER_ATTEMPT_CREDIT + assert user_txs.transactions[0].amount == 5 + assert user_txs.transactions[0].balance_after == 5 + assert user_txs.summary.user_attempt_credit.entry_count == 1 + assert user_txs.summary.user_attempt_credit.total_amount == 5 + + user_txs_by_name = thl_ledger_manager.get_user_txs( + user, qualified_name=attempt_credit_account.qualified_name + ) + assert user_txs_by_name == user_txs + with pytest.raises( LedgerTransactionConditionFailedError, match=r"^duplicate tag$" ): @@ -887,6 +905,15 @@ class TestThlLedgerTxManager: assert thl_ledger_manager.get_user_wallet_balance(user) == round(user_pay * 100) assert thl_ledger_manager.check_ledger_balanced() + attempt_credit_account = ( + thl_ledger_manager.get_account_or_create_user_attempt_credit(user) + ) + user_txs = thl_ledger_manager.get_user_txs( + user, account_uuid=attempt_credit_account.uuid + ) + assert [tx.amount for tx in user_txs.transactions] == [5, -5] + assert [tx.balance_after for tx in user_txs.transactions] == [5, 0] + def test_bp_payment_with_zero_attempt_credit_balance( self, session_factory: Callable[..., Session], diff --git a/tests/managers/thl/test_ledger/test_thl_pem.py b/tests/managers/thl/test_ledger/test_thl_pem.py index fb35aa4..4761438 100644 --- a/tests/managers/thl/test_ledger/test_thl_pem.py +++ b/tests/managers/thl/test_ledger/test_thl_pem.py @@ -30,7 +30,6 @@ if TYPE_CHECKING: class TestThlPayoutEventManager: - def test_get_by_uuid( self, brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager ): @@ -50,8 +49,9 @@ class TestThlPayoutEventManager: bp_payout_event_factory: Callable[..., BrokerageProductPayoutEvent], thl_ledger_manager: ThlLedgerManager, brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager, + create_main_accounts, ): - + create_main_accounts() N_PRODUCTS = randint(3, 10) N_PAYOUT_EVENTS = randint(3, 10) amounts = [] @@ -223,7 +223,6 @@ class TestThlPayoutEventManager: class TestBPPayoutEvent: - def test_get_bp_bp_payout_events_for_products( self, product_factory: Callable[..., Product], diff --git a/tests/managers/thl/test_payout.py b/tests/managers/thl/test_payout.py index 52bbbec..69b9737 100644 --- a/tests/managers/thl/test_payout.py +++ b/tests/managers/thl/test_payout.py @@ -163,7 +163,9 @@ class TestPayout: thl_ledger_manager: ThlLedgerManager, utc_now: datetime, pending_bp_pe: BrokerageProductPayoutEvent, + create_main_accounts, ): + create_main_accounts() thl_ledger_manager.get_account_or_create_bp_wallet(product=product) brokerage_product_payout_event_manager.create_tx_bp_payout_from_payout_event( @@ -871,6 +873,7 @@ class TestBusinessPayoutEventManager: bp1 = business_payout_event_manager.create_from_ach_or_wire( business=gr_business, amount=USDCent(bb1.available_balance), + transaction_id=ach_id1, pm=product_manager, thl_lm=thl_ledger_manager, created=start + timedelta(days=1, hours=5), @@ -901,22 +904,22 @@ class TestBusinessPayoutEventManager: ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) - business.prebuild_balance( - thl_pg_config=thl_web_rr, + gr_business.prebuild_balance( + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, pop_ledger=pop_ledger_merge, ) - business.prebuild_payouts( + gr_business.prebuild_payouts( bpem=business_payout_event_manager, ) - assert isinstance(business.payouts, list) - assert len(business.payouts) == 2 - assert len(business.payouts[0].bp_payouts) == 2 - assert len(business.payouts[1].bp_payouts) == 1 + assert isinstance(gr_business.payouts, list) + assert len(gr_business.payouts) == 2 + assert len(gr_business.payouts[0].bp_payouts) == 2 + assert len(gr_business.payouts[1].bp_payouts) == 1 - bb2 = business.balance + bb2 = gr_business.balance # Okay os we have the balance before, and after the Business Payout # of bb1.available_balance worth.. -- cgit v1.2.3 From 438ea05fdb98c847d534ae190179b4766fa96f26 Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 13:40:54 -0600 Subject: test user payout paypal and tango, and fix bugs --- generalresearch/managers/thl/cashout_method.py | 2 +- generalresearch/managers/thl/wallet/tango.py | 6 +- generalresearch/managers/thl/wallet/user_payout.py | 23 +- .../models/thl/wallet/cashout_method.py | 4 +- test_utils/managers/conftest.py | 25 +- test_utils/managers/thl/conftest.py | 11 - tests/managers/thl/test_ledger/test_thl_pem.py | 32 -- tests/managers/thl/test_payout.py | 2 +- tests/managers/thl/test_wallet/__init__.py | 0 tests/managers/thl/test_wallet/test_user_payout.py | 435 +++++++++++++++++++++ 10 files changed, 454 insertions(+), 86 deletions(-) create mode 100644 tests/managers/thl/test_wallet/__init__.py create mode 100644 tests/managers/thl/test_wallet/test_user_payout.py diff --git a/generalresearch/managers/thl/cashout_method.py b/generalresearch/managers/thl/cashout_method.py index 46bde38..90a6876 100644 --- a/generalresearch/managers/thl/cashout_method.py +++ b/generalresearch/managers/thl/cashout_method.py @@ -376,7 +376,7 @@ class CashoutMethodManager(PostgresManager): if user and x["type"] in {PayoutType.PAYPAL, PayoutType.CASH_IN_MAIL}: user = user if isinstance(user, UserRef) else user.to_user_ref() x["user"] = user - x["original_currency"] = x.get("currency") or Currency.USD + x["original_currency"] = x.get("original_currency") or Currency.USD x["currency"] = Currency.USD return CashoutMethod.model_validate(x) diff --git a/generalresearch/managers/thl/wallet/tango.py b/generalresearch/managers/thl/wallet/tango.py index ca993ea..361a885 100644 --- a/generalresearch/managers/thl/wallet/tango.py +++ b/generalresearch/managers/thl/wallet/tango.py @@ -1,6 +1,7 @@ from __future__ import annotations from datetime import timedelta +from decimal import Decimal from threading import Lock from typing import TYPE_CHECKING, Any @@ -69,13 +70,14 @@ class TangoManager: utid = cashout_method.data.utid currency = cashout_method.original_currency if currency and currency != Currency.USD: - amount = round(float(amount) / self.get_exchange_rates()[currency], 2) + amount = round(float(amount) / self.get_exchange_rates()[currency]) + amount_in_currency = Decimal(float(amount) / 100).quantize(Decimal('0.01')) return TangoCashoutMethodRequestData.model_validate( { "accountIdentifier": self.tango_account_id, "customerIdentifier": self.tango_customer_id, "utid": utid, - "amount": str(amount), + "amount": amount_in_currency, "campaign": "300large", "sendEmail": False, "externalRefID": payout_event_id, diff --git a/generalresearch/managers/thl/wallet/user_payout.py b/generalresearch/managers/thl/wallet/user_payout.py index a1c628b..983ab3a 100644 --- a/generalresearch/managers/thl/wallet/user_payout.py +++ b/generalresearch/managers/thl/wallet/user_payout.py @@ -68,8 +68,8 @@ class UserPayoutEventManager(PayoutEventManager): return pe def get_payout_detail(self, pe_uuid: UUIDStr) -> CashoutRequestInfo: - # This gets the payout event, and then extracts information for - # the purposes of returning to the user. + # This gets the payout event and then extracts information to return + # to the user. pe = self.get_by_uuid(pe_uuid=pe_uuid) transaction_info = {} @@ -205,9 +205,6 @@ class UserPayoutEventManager(PayoutEventManager): created: AwareDatetimeISO | None = None, request_data: dict[str, Any] | None = None, # --- Optional: None --- - account_reference_type: str | None = None, - account_reference_uuid: UUIDStr | None = None, - description: str | None = None, ext_ref_id: str | None = None, order_data: dict[str, Any] | CashMailOrderData | None = None, ) -> UserPayoutEvent: @@ -215,10 +212,7 @@ class UserPayoutEventManager(PayoutEventManager): payout_event = UserPayoutEvent( uuid=uuid or uuid4().hex, debit_account_uuid=debit_account_uuid, - account_reference_type=account_reference_type, - account_reference_uuid=account_reference_uuid, cashout_method_uuid=cashout_method_uuid, - description=description, created=created or datetime.now(tz=UTC), amount=amount, status=status or PayoutStatus.PENDING, @@ -250,7 +244,7 @@ class UserPayoutEventManager(PayoutEventManager): assert c.rowcount == 1, f"expected 1 row inserted, got {c.rowcount}" conn.commit() - return payout_event + return self.get_by_uuid(payout_event.uuid) def try_user_request_redeem( self, @@ -261,7 +255,6 @@ class UserPayoutEventManager(PayoutEventManager): cashout_method_manager: CashoutMethodManager, ledger_manager: ThlLedgerManager, user_ip_history_manager: UserIpHistoryManager, - geoip_info_manager: GeoIpInfoManager, redis_client: Redis, slack_client: slack.WebClient | None = None, ) -> tuple[UserPayoutEvent | None, str | None]: @@ -274,7 +267,6 @@ class UserPayoutEventManager(PayoutEventManager): cashout_method_manager=cashout_method_manager, ledger_manager=ledger_manager, user_ip_history_manager=user_ip_history_manager, - geoip_info_manager=geoip_info_manager, redis_client=redis_client, slack_client=slack_client, ), None @@ -290,7 +282,6 @@ class UserPayoutEventManager(PayoutEventManager): cashout_method_manager: CashoutMethodManager, ledger_manager: ThlLedgerManager, user_ip_history_manager: UserIpHistoryManager, - geoip_info_manager: GeoIpInfoManager, redis_client: Redis, slack_client: slack.WebClient | None = None, ) -> UserPayoutEvent: @@ -303,9 +294,7 @@ class UserPayoutEventManager(PayoutEventManager): now = datetime.now(tz=UTC) user.prefetch_product(pg_config=self.pg_config) - country_iso = user_ip_history_manager.get_user_latest_country( - user, geoip_info_manager - ) + country_iso = user_ip_history_manager.get_user_latest_country(user) assert country_iso, "user has no country" usd_exchange_rates = tango_manager.get_exchange_rates() @@ -345,9 +334,7 @@ class UserPayoutEventManager(PayoutEventManager): product = user.product banned_countries = user.product.user_health_config.banned_countries - assert not user_ip_history_manager.is_user_anonymous( - user, geoip_info_manager=geoip_info_manager - ), "Anonymous user requesting redemption" + assert not user_ip_history_manager.is_user_anonymous(user), "Anonymous user requesting redemption" if country_iso in banned_countries: raise AssertionError("Banned country requesting redemption") diff --git a/generalresearch/models/thl/wallet/cashout_method.py b/generalresearch/models/thl/wallet/cashout_method.py index a82fa31..61a8992 100644 --- a/generalresearch/models/thl/wallet/cashout_method.py +++ b/generalresearch/models/thl/wallet/cashout_method.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib import logging from datetime import UTC, datetime +from decimal import Decimal from enum import StrEnum from typing import Any, Literal, Self @@ -240,7 +241,8 @@ class TangoCashoutMethodRequestData(BaseModel): accountIdentifier: str = Field() customerIdentifier: str = Field() utid: str = Field(description="tango utid") - amount: USDCent = Field(description="Amount to be paid out") + # This is not necessarily in USD. It is in whatever currency the card is in + amount: Decimal = Field(description="Amount to be paid out (in USD, EUR, etc)") campaign: Literal["300large"] sendEmail: bool = Field(default=False) externalRefID: str = Field(description="External Ref ID") diff --git a/test_utils/managers/conftest.py b/test_utils/managers/conftest.py index 72be79e..a9f92a2 100644 --- a/test_utils/managers/conftest.py +++ b/test_utils/managers/conftest.py @@ -25,7 +25,6 @@ from generalresearch.managers.thl.user_streak import ( ) from generalresearch.managers.thl.userhealth import ( AuditLogManager, - IPRecordManager, UserIpHistoryManager, ) from generalresearch.models.definitions import Source @@ -56,24 +55,6 @@ def audit_log_manager(thl_web_rw: PostgresConfig) -> AuditLogManager: return AuditLogManager(pg_config=thl_web_rw) -@pytest.fixture -def ip_record_manager( - thl_web_rw: PostgresConfig, - thl_redis_config: RedisConfig, - geoip_info_manager: GeoIpInfoManager, -) -> IPRecordManager: - assert thl_web_rw.dsn.path - assert "/unittest-" in thl_web_rw.dsn.path - - from generalresearch.managers.thl.userhealth import IPRecordManager - - return IPRecordManager( - pg_config=thl_web_rw, - redis_config=thl_redis_config, - geoip_info_manager=geoip_info_manager, - ) - - @pytest.fixture def user_iphistory_manager( thl_web_rw: PostgresConfig, @@ -124,6 +105,11 @@ def grip_lookup_results() -> dict[str, GRIPMMDBLookupResult]: network_operator="Cloudflare", ), ), + "2.2.2.2": GRIPMMDBLookupResult( + country=GRIPCountryRecord(country_iso="IT"), + anonymous=GRIPAnonymousRecord(is_anonymous=False), + asn=GRIPAsnRecord(), + ), }, ) @@ -214,7 +200,6 @@ def random_ext_id_factory(base: str = "U02") -> Callable[..., str]: @pytest.fixture(scope="session") def example_tango_cashout_methods( - random_ext_id_factory: Callable[..., str], ) -> list[CashoutMethod]: return [ CashoutMethod( diff --git a/test_utils/managers/thl/conftest.py b/test_utils/managers/thl/conftest.py index 5b96680..2dce648 100644 --- a/test_utils/managers/thl/conftest.py +++ b/test_utils/managers/thl/conftest.py @@ -40,7 +40,6 @@ if TYPE_CHECKING: UserMetadataManager, ) from generalresearch.managers.thl.userhealth import ( - AuditLogManager, IPRecordManager, ) from generalresearch.managers.thl.wall import ( @@ -162,13 +161,6 @@ def brokerage_product_payout_event_manager( ) -@pytest.fixture() -def audit_log_manager(thl_web_rw: PostgresConfig) -> AuditLogManager: - from generalresearch.managers.thl.userhealth import AuditLogManager - - return AuditLogManager(pg_config=thl_web_rw) - - @pytest.fixture(scope="session") def business_payout_event_manager( thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig @@ -338,9 +330,6 @@ def surveypenalty_manager(thl_redis_config: RedisConfig): return SurveyPenaltyManager(redis_config=thl_redis_config) -# --- IP Record --- - - @pytest.fixture(scope="session") def ip_record_manager( thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig diff --git a/tests/managers/thl/test_ledger/test_thl_pem.py b/tests/managers/thl/test_ledger/test_thl_pem.py index 4761438..2d03b25 100644 --- a/tests/managers/thl/test_ledger/test_thl_pem.py +++ b/tests/managers/thl/test_ledger/test_thl_pem.py @@ -122,38 +122,6 @@ class TestThlPayoutEventManager: assert len(res) == (N_PRODUCTS * N_PAYOUT_EVENTS) assert sum([i.amount for i in res]) == sum(amounts) - @pytest.mark.skip - def test_get_payout_detail(self, user_payout_event_manager: UserPayoutEventManager): - """This fails because the description coming back is None, but then - it tries to return a PayoutEvent which validates that the - description can't be None - """ - from generalresearch.models.thl.payout import ( - PayoutType, - ) - - rand_amount = randint(a=99, b=999) - - pe = user_payout_event_manager.create( - debit_account_uuid=uuid4().hex, - account_reference_type="str-type-random", - account_reference_uuid=uuid4().hex, - cashout_method_uuid=uuid4().hex, - description="Best payout !", - amount=rand_amount, - status=PayoutStatus.PENDING, - ext_ref_id="123", - payout_type=PayoutType.CASH_IN_MAIL, - request_data={"foo": 123}, - order_data={}, - ) - - res = user_payout_event_manager.get_payout_detail(pe_uuid=pe.uuid) - assert isinstance(res, CashoutRequestInfo) - - # def test_filter_by(self): - # raise NotImplementedError - def test_create( self, user_payout_event_factory: Callable[..., UserPayoutEvent], diff --git a/tests/managers/thl/test_payout.py b/tests/managers/thl/test_payout.py index 69b9737..a9cd52a 100644 --- a/tests/managers/thl/test_payout.py +++ b/tests/managers/thl/test_payout.py @@ -34,9 +34,9 @@ if TYPE_CHECKING: BrokerageProductPayoutEventManager, BusinessPayoutEventManager, PayoutEventManager, - UserPayoutEventManager, ) from generalresearch.managers.thl.product import ProductManager + from generalresearch.managers.thl.wallet.user_payout import UserPayoutEventManager from generalresearch.models.gr.business import Business from generalresearch.models.thl.ledger import LedgerAccount from generalresearch.models.thl.payout import ( diff --git a/tests/managers/thl/test_wallet/__init__.py b/tests/managers/thl/test_wallet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/managers/thl/test_wallet/test_user_payout.py b/tests/managers/thl/test_wallet/test_user_payout.py new file mode 100644 index 0000000..bbc97a3 --- /dev/null +++ b/tests/managers/thl/test_wallet/test_user_payout.py @@ -0,0 +1,435 @@ +from datetime import datetime +from decimal import Decimal +from random import randint +from unittest.mock import Mock +from uuid import uuid4 + +import pytest +from faker import Faker + +from generalresearch.currency import USDCent +from generalresearch.managers.thl.cashout_method import CashoutMethodManager +from generalresearch.managers.thl.ipinfo import GeoIpInfoManager +from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager +from generalresearch.managers.thl.userhealth import UserIpHistoryManager +from generalresearch.managers.thl.wallet.tango import TangoManager +from generalresearch.managers.thl.wallet.user_payout import ( + UserPayoutEventManager, + make_request_paypal, +) +from generalresearch.models.thl.definitions import PayoutStatus +from generalresearch.models.thl.payout import UserPayoutEvent +from generalresearch.models.thl.user import User +from generalresearch.models.thl.wallet.cashout_method import ( + CashMailCashoutMethodData, + CashMailCashoutMethodRequestData, + CashoutRequestInfo, + PaypalCashoutMethodData, + PaypalCashoutMethodRequestData, + USDeliveryAddress, +) +from generalresearch.models.thl.wallet.definitions import PayoutType + +fake = Faker() + + +class TestUserPayoutEventManager: + def test_get_by_uuid_and_create( + self, + user: User, + user_payout_event_manager: UserPayoutEventManager, + thl_ledger_manager: ThlLedgerManager, + utc_now: datetime, + ): + user_account = thl_ledger_manager.get_account_or_create_user_wallet(user=user) + request_data = PaypalCashoutMethodRequestData.model_validate( + { + "email": fake.email(), + "interface": "api", + } + ) + pe1: UserPayoutEvent = user_payout_event_manager.create( + debit_account_uuid=user_account.uuid, + payout_type=PayoutType.PAYPAL, + cashout_method_uuid=uuid4().hex, + amount=100, + created=utc_now, + request_data=request_data.model_dump(mode="json"), + ) + # these get added by the query + pe1.account_reference_type = "user" + pe1.account_reference_uuid = user.uuid + + pe2 = user_payout_event_manager.get_by_uuid(pe_uuid=pe1.uuid) + + assert pe1 == pe2 + + def test_get_payout_detail_cash_in_mail( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + ): + user = user_with_wallet + data = CashMailCashoutMethodData( + delivery_address=USDeliveryAddress.model_validate( + { + "name_or_attn": "Josh Ackerman", + "address": "123 Fake St", + "city": "San Francisco", + "state": "CA", + "postal_code": "12345", + } + ) + ) + cm = cashout_method_manager.create_cash_in_mail_cashout_method( + data=data, user=user + ) + user_account = thl_ledger_manager.get_account_or_create_user_wallet(user=user) + + rand_amount = randint(a=99, b=999) + request_data = CashMailCashoutMethodRequestData.model_validate( + cm.data.model_dump() + ) + + pe = user_payout_event_manager.create( + debit_account_uuid=user_account.uuid, + cashout_method_uuid=cm.id, + amount=rand_amount, + ext_ref_id=uuid4().hex, + payout_type=PayoutType.CASH_IN_MAIL, + request_data=request_data.model_dump(mode="json"), + ) + + res = user_payout_event_manager.get_payout_detail(pe_uuid=pe.uuid) + assert isinstance(res, CashoutRequestInfo) + + def test_get_payout_detail_paypal( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + ): + user = user_with_wallet + data = PaypalCashoutMethodData(email=fake.email()) + cm = cashout_method_manager.create_paypal_cashout_method(data=data, user=user) + user_account = thl_ledger_manager.get_account_or_create_user_wallet(user=user) + + rand_amount = randint(a=99, b=999) + request_data = make_request_paypal(cm) + pe = user_payout_event_manager.create( + debit_account_uuid=user_account.uuid, + cashout_method_uuid=cm.id, + amount=rand_amount, + ext_ref_id=uuid4().hex, + payout_type=PayoutType.PAYPAL, + request_data=request_data.model_dump(mode="json"), + ) + + res = user_payout_event_manager.get_payout_detail(pe_uuid=pe.uuid) + assert isinstance(res, CashoutRequestInfo) + + +class TestUserRequestRedeem: + @staticmethod + def request_paypal_payout( + *, + user: User, + amount: USDCent, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + ip_record_factory, + thl_redis_client, + ip: str = "8.8.8.8", + fund_amount: Decimal = Decimal("10.00"), + ) -> UserPayoutEvent: + email = fake.email() + cashout_method = cashout_method_manager.create_paypal_cashout_method( + data=PaypalCashoutMethodData(email=email), + user=user, + ) + ip_record_factory(user=user, ip=ip) + thl_ledger_manager.create_tx_user_bonus( + user=user, + amount=fund_amount, + ref_uuid=uuid4().hex, + description="Fund PayPal redemption test", + ) + + tango_manager = Mock(spec=TangoManager) + tango_manager.get_exchange_rates.return_value = {} + + return user_payout_event_manager.user_request_redeem( + user=user, + cashout_method_id=cashout_method.id, + amount=amount, + tango_manager=tango_manager, + cashout_method_manager=cashout_method_manager, + ledger_manager=thl_ledger_manager, + user_ip_history_manager=user_iphistory_manager, + redis_client=thl_redis_client, + ) + + def test_user_request_redeem_paypal( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + geoip_info_manager: GeoIpInfoManager, + ip_record_factory, + thl_redis_client, + ): + user = user_with_wallet + payout = self.request_paypal_payout( + user=user, + amount=USDCent(600), + user_payout_event_manager=user_payout_event_manager, + cashout_method_manager=cashout_method_manager, + thl_ledger_manager=thl_ledger_manager, + user_iphistory_manager=user_iphistory_manager, + ip_record_factory=ip_record_factory, + thl_redis_client=thl_redis_client, + ) + + assert payout.status == PayoutStatus.PENDING + assert payout.payout_type == PayoutType.PAYPAL + assert payout.amount == 600 + assert payout.request_data["interface"] == "api" + assert thl_ledger_manager.get_user_wallet_balance(user) == 400 + assert user_payout_event_manager.get_by_uuid(payout.uuid) == payout + + detail = user_payout_event_manager.get_payout_detail(payout.uuid) + assert not detail.transaction_info + assert detail.description == "PayPal" + + def test_user_request_redeem_paypal_complete( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + geoip_info_manager: GeoIpInfoManager, + ip_record_factory, + thl_redis_client, + create_main_accounts, + ): + create_main_accounts() + user = user_with_wallet + payout = self.request_paypal_payout( + user=user, + amount=USDCent(600), + user_payout_event_manager=user_payout_event_manager, + cashout_method_manager=cashout_method_manager, + thl_ledger_manager=thl_ledger_manager, + user_iphistory_manager=user_iphistory_manager, + ip_record_factory=ip_record_factory, + thl_redis_client=thl_redis_client, + ) + + user_payout_event_manager.update( + payout_event=payout, + status=PayoutStatus.APPROVED, + ext_ref_id="paypal-batch-id", + ) + thl_ledger_manager.create_tx_user_payout_complete( + user=user, + payout_event=payout, + fee_amount=Decimal("0.25"), + ) + user_payout_event_manager.update( + payout_event=payout, + status=PayoutStatus.COMPLETE, + order_data={"transaction_id": "paypal-transaction-id"}, + ) + + detail = user_payout_event_manager.get_payout_detail(pe_uuid=payout.uuid) + assert detail.status == PayoutStatus.COMPLETE + assert detail.transaction_info == {"transaction_id": "paypal-transaction-id"} + + assert thl_ledger_manager.get_user_wallet_balance(user_with_wallet) == 400 + bp_expense = thl_ledger_manager.get_account_or_create_bp_expense_by_uuid( + product_uuid=user.to_user_ref().product_id, expense_name="paypal" + ) + assert thl_ledger_manager.get_account_balance(bp_expense) == -25 + + def test_user_request_redeem_tango_complete( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + ip_record_factory, + thl_redis_client, + create_main_accounts, + setup_cashoutmethod_db, + example_tango_cashout_methods, + ): + create_main_accounts() + setup_cashoutmethod_db() + user = user_with_wallet + # This is the italian Amazon.it gift card + cashout_method = example_tango_cashout_methods[1] + # This is an IP that we have hard-coded by the geoip_info_manager as Italy + ip_record_factory(user=user, ip="2.2.2.2") + + thl_ledger_manager.create_tx_user_bonus( + user=user, + amount=Decimal("5.00"), + ref_uuid=uuid4().hex, + description="Fund Tango redemption test", + ) + + tango_client = Mock() + tango_client.get_exchange_rates.return_value = { + "exchangeRates": [ + {"rewardCurrency": "USD", "baseCurrency": "EUR", "baseFx": 1.14792} + ] + } + tango_manager = TangoManager( + tango_client=tango_client, + tango_account_id="test-account", + tango_customer_id="test-customer", + cashout_method_manager=cashout_method_manager, + ) + request = tango_manager.make_request( + amount=USDCent(2_00), + cashout_method=cashout_method, + payout_event_id=uuid4().hex, + ) + assert request.amount == Decimal(2.0 / 1.14792).quantize( + Decimal("0.01") + ) # ~ 1.74 EUR + + payout = user_payout_event_manager.user_request_redeem( + user=user, + cashout_method_id=cashout_method.id, + amount=USDCent(2_00), + tango_manager=tango_manager, + cashout_method_manager=cashout_method_manager, + ledger_manager=thl_ledger_manager, + user_ip_history_manager=user_iphistory_manager, + redis_client=thl_redis_client, + ) + assert payout.request_data["amount"] == '1.74' + assert payout.request_data["externalRefID"] == payout.uuid + + user_payout_event_manager.update( + payout_event=payout, + status=PayoutStatus.APPROVED, + ext_ref_id="tango-order-id", + ) + thl_ledger_manager.create_tx_user_payout_complete( + user=user, + payout_event=payout, + ) + user_payout_event_manager.update( + payout_event=payout, + status=PayoutStatus.COMPLETE, + order_data={ + "reward": { + "credentialList": [ + { + "credentialType": "giftCardCode", + "credentialValue": "TEST-CODE", + } + ], + "redemptionInstructions": "Use this code at checkout.", + } + }, + ) + + detail = user_payout_event_manager.get_payout_detail(pe_uuid=payout.uuid) + assert payout.payout_type == PayoutType.TANGO + assert detail.status == PayoutStatus.COMPLETE + assert detail.transaction_info == { + "credential_list": [ + { + "credentialType": "giftCardCode", + "credentialValue": "TEST-CODE", + } + ], + "redemption_instructions": "Use this code at checkout.", + } + assert thl_ledger_manager.get_user_wallet_balance(user) == 300 + bp_expense = thl_ledger_manager.get_account_or_create_bp_expense_by_uuid( + product_uuid=user.to_user_ref().product_id, + expense_name="tango", + ) + assert thl_ledger_manager.get_account_balance(bp_expense) == -7 + + def test_user_request_redeem_paypal_insufficient_balance( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + ip_record_factory, + thl_redis_client, + ): + with pytest.raises(AssertionError, match="more than their redeemable balance"): + self.request_paypal_payout( + user=user_with_wallet, + amount=USDCent(1_100), + user_payout_event_manager=user_payout_event_manager, + cashout_method_manager=cashout_method_manager, + thl_ledger_manager=thl_ledger_manager, + user_iphistory_manager=user_iphistory_manager, + ip_record_factory=ip_record_factory, + thl_redis_client=thl_redis_client, + ) + + def test_user_request_redeem_paypal_blocked_user( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + ip_record_factory, + thl_redis_client, + ): + user_with_wallet.blocked = True + + with pytest.raises(AssertionError, match="Blocked user"): + self.request_paypal_payout( + user=user_with_wallet, + amount=USDCent(600), + user_payout_event_manager=user_payout_event_manager, + cashout_method_manager=cashout_method_manager, + thl_ledger_manager=thl_ledger_manager, + user_iphistory_manager=user_iphistory_manager, + ip_record_factory=ip_record_factory, + thl_redis_client=thl_redis_client, + ) + + def test_user_request_redeem_paypal_anonymous_user( + self, + user_with_wallet: User, + user_payout_event_manager: UserPayoutEventManager, + cashout_method_manager: CashoutMethodManager, + thl_ledger_manager: ThlLedgerManager, + user_iphistory_manager: UserIpHistoryManager, + ip_record_factory, + thl_redis_client, + ): + with pytest.raises(AssertionError, match="Anonymous user"): + self.request_paypal_payout( + user=user_with_wallet, + amount=USDCent(600), + user_payout_event_manager=user_payout_event_manager, + cashout_method_manager=cashout_method_manager, + thl_ledger_manager=thl_ledger_manager, + user_iphistory_manager=user_iphistory_manager, + ip_record_factory=ip_record_factory, + thl_redis_client=thl_redis_client, + ip="1.1.1.1", + ) -- cgit v1.2.3 From f4909f0fafd6161dd423859afadb72b916e90564 Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 14:56:36 -0600 Subject: 1 single broken managers test --- generalresearch/cacheing.py | 72 ++++++++-------------- generalresearch/managers/thl/cashout_method.py | 19 +++--- .../managers/thl/ledger_manager/conditions.py | 17 +++-- .../managers/thl/ledger_manager/thl_ledger.py | 2 +- generalresearch/managers/thl/payout.py | 2 +- .../managers/thl/user_manager/user_manager.py | 9 ++- test_utils/models/thl/conftest.py | 7 --- test_utils/models/upk/conftest.py | 7 +++ tests/managers/thl/test_ledger/test_lm_tx.py | 4 +- tests/managers/thl/test_ledger/test_lm_tx_locks.py | 21 ++++--- tests/managers/thl/test_ledger/test_thl_pem.py | 14 ++--- tests/managers/thl/test_payout.py | 30 ++++----- tests/managers/thl/test_session_manager.py | 6 +- tests/managers/thl/test_survey_penalty.py | 2 +- tests/managers/thl/test_user_manager/test_base.py | 28 ++++----- tests/managers/thl/test_user_manager/test_redis.py | 15 +++-- tests/managers/thl/test_userhealth.py | 2 - 17 files changed, 121 insertions(+), 136 deletions(-) diff --git a/generalresearch/cacheing.py b/generalresearch/cacheing.py index 34df267..55d9c9c 100644 --- a/generalresearch/cacheing.py +++ b/generalresearch/cacheing.py @@ -1,47 +1,25 @@ -from generalresearch import retry - - -class RetryCache: - # Simple pylibmc.Client wrapper that implements a retry on each method - - def __init__(self, client, tries: int = 4, delay: int = 1, backoff: float = 1.5): - import pylibmc - - self.client = client - self.f = retry(pylibmc.Error, tries=tries, delay=delay, backoff=backoff) - - def get(self, key): - @self.f - def _get(key): - return self.client.get(key) - - return _get(key) - - def set(self, key, value, timeout=0): - @self.f - def _set(key, value, timeout): - return self.client.set(key, value, time=timeout) - - return _set(key, value, timeout) - - def delete_multi(self, keys): - @self.f - def _delete_multi(keys): - return self.client.delete_multi(keys) - - return _delete_multi(keys) - - def delete(self, key): - @self.f - def _delete(key): - return self.client.delete(key) - - return _delete(key) - - -if __name__ == "__main__": - import pylibmc - - CACHE = RetryCache(pylibmc.Client(["127.0.0.1:11211"], binary=True)) - CACHE.set("foo", "bar") - print(CACHE.get("foo")) +from cachetools import TTLCache + + +class InstrumentedTTLCache(TTLCache): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.hits = 0 + self.misses = 0 + + def __getitem__(self, key): + try: + value = super().__getitem__(key) + self.hits += 1 + return value + except KeyError: + self.misses += 1 + raise + + def cache_info(self): + return { + "hits": self.hits, + "misses": self.misses, + "currsize": self.currsize, + "maxsize": self.maxsize, + } diff --git a/generalresearch/managers/thl/cashout_method.py b/generalresearch/managers/thl/cashout_method.py index 90a6876..e83b16a 100644 --- a/generalresearch/managers/thl/cashout_method.py +++ b/generalresearch/managers/thl/cashout_method.py @@ -329,6 +329,16 @@ class CashoutMethodManager(PostgresManager): cms = self.get_cashout_methods(user=user) + # Filter out by country before bothering w exchange rates + cms = [ + cm + for cm in cms + if ( + cm.type == PayoutType.TANGO and country_iso.lower() in cm.data.countries + ) + or cm.type != PayoutType.TANGO + ] + for x in cms: # assets in non-USD need to be converted to USD here if x.original_currency is not None: @@ -342,15 +352,6 @@ class CashoutMethodManager(PostgresManager): # Adjust min_value for BP x.min_value = max(x.min_value_usd, min_value) - cms = [ - cm - for cm in cms - if ( - cm.type == PayoutType.TANGO and country_iso.lower() in cm.data.countries - ) - or cm.type != PayoutType.TANGO - ] - return {x.id: x for x in cms} @staticmethod diff --git a/generalresearch/managers/thl/ledger_manager/conditions.py b/generalresearch/managers/thl/ledger_manager/conditions.py index 21a99b5..98502d0 100644 --- a/generalresearch/managers/thl/ledger_manager/conditions.py +++ b/generalresearch/managers/thl/ledger_manager/conditions.py @@ -35,7 +35,10 @@ def generate_condition_mp_payment(wall: Wall) -> Callable[..., tuple[bool, str]] def _condition(lm: LedgerManager) -> tuple[bool, str]: tag = f"{lm.currency.value}:mp_payment:{wall_uuid}" txs = lm.get_tx_ids_by_tag(tag=tag) - return len(txs) == 0, "duplicate tag" + if len(txs) != 0: + return False, "duplicate tag" + else: + return True, "" return _condition @@ -49,8 +52,11 @@ def generate_condition_bp_payment(session: Session) -> Callable[..., tuple[bool, def _condition(lm: LedgerManager) -> tuple[bool, str]: tag = f"{lm.currency.value}:bp_payment:{session_uuid}" - txs_ids = lm.get_tx_ids_by_tag(tag=tag) - return len(txs_ids) == 0, "duplicate tag" + txs = lm.get_tx_ids_by_tag(tag=tag) + if len(txs) != 0: + return False, "duplicate tag" + else: + return True, "" return _condition @@ -62,7 +68,10 @@ def generate_condition_tag_exists(tag: str) -> Callable[..., tuple[bool, str]]: def _condition(lm: LedgerManager) -> tuple[bool, str]: txs_ids = lm.get_tx_ids_by_tag(tag=tag) - return len(txs_ids) == 0, "duplicate tag" + if len(txs_ids) != 0: + return False, "duplicate tag" + else: + return True, "" return _condition diff --git a/generalresearch/managers/thl/ledger_manager/thl_ledger.py b/generalresearch/managers/thl/ledger_manager/thl_ledger.py index 5210189..de7cae9 100644 --- a/generalresearch/managers/thl/ledger_manager/thl_ledger.py +++ b/generalresearch/managers/thl/ledger_manager/thl_ledger.py @@ -303,7 +303,7 @@ class ThlLedgerManager(LedgerManager): user: User, created: datetime | None = None, force: bool = False, - ) -> PositiveInt: + ) -> LedgerTransaction: """ Create a transaction when we complete a task from a marketplace, showing the marketplace paying us for the task complete. diff --git a/generalresearch/managers/thl/payout.py b/generalresearch/managers/thl/payout.py index 3968ea7..af4f712 100644 --- a/generalresearch/managers/thl/payout.py +++ b/generalresearch/managers/thl/payout.py @@ -145,7 +145,7 @@ class BrokerageProductPayoutEventManager(PayoutEventManager): amount = USDCent(payout_event.amount) product_id = payout_event.product_id - txs = thl_ledger_manager.get_tx_by_tag(tag) + txs = thl_ledger_manager.get_txs_by_tag(tag) if not txs: return False diff --git a/generalresearch/managers/thl/user_manager/user_manager.py b/generalresearch/managers/thl/user_manager/user_manager.py index df3fb6b..15fd291 100644 --- a/generalresearch/managers/thl/user_manager/user_manager.py +++ b/generalresearch/managers/thl/user_manager/user_manager.py @@ -7,9 +7,10 @@ from datetime import datetime from threading import Lock from typing import TYPE_CHECKING -from cachetools import TTLCache, cachedmethod +from cachetools import cachedmethod from pydantic import RedisDsn +from generalresearch.cacheing import InstrumentedTTLCache from generalresearch.managers.base import Permission from generalresearch.managers.thl.product import ProductManager from generalresearch.managers.thl.user_manager.exceptions import ( @@ -87,7 +88,7 @@ class UserManager: self.product_manager = ProductManager( pg_config=pg_config, permissions=[Permission.READ] ) - self.get_user_cache = TTLCache(maxsize=10000, ttl=30) + self.get_user_cache = InstrumentedTTLCache(maxsize=10000, ttl=30) self.get_user_cache_lock = Lock() def set_last_seen(self, user: User) -> None: @@ -147,6 +148,10 @@ class UserManager: with self.get_user_cache_lock: self.get_user_cache.clear() + def cache_info(self): + with self.get_user_cache_lock: + return self.get_user_cache.cache_info() + @deepcopy_return @cachedmethod( operator.attrgetter("get_user_cache"), diff --git a/test_utils/models/thl/conftest.py b/test_utils/models/thl/conftest.py index 433003c..a8ad037 100644 --- a/test_utils/models/thl/conftest.py +++ b/test_utils/models/thl/conftest.py @@ -628,13 +628,9 @@ def user_payout_event_factory( debit_account_uuid = debit_account_uuid or uuid4().hex cashout_method_uuid = cashout_method_uuid or uuid4().hex - # account_reference_type = account_reference_type or f"acct-ref-{uuid4().hex}" - # account_reference_uuid = account_reference_uuid or uuid4().hex - # cashout_method_uuid = cashout_method_uuid or uuid4().hex amount = amount or randint(a=99, b=9_999) status = status or rand_choice(list(PayoutStatus)) - description = description or f"desc-{uuid4().hex[:12]}" # ext_ref_id = ext_ref_id or f"ext-ref-{uuid4().hex[:8]}" payout_type = payout_type or rand_choice(list(PayoutType)) request_data = request_data or {} @@ -643,10 +639,7 @@ def user_payout_event_factory( return user_payout_event_manager.create( uuid=uuid, debit_account_uuid=debit_account_uuid, - account_reference_type=account_reference_type, - account_reference_uuid=account_reference_uuid, cashout_method_uuid=cashout_method_uuid, - description=description, created=created, amount=amount, status=status, diff --git a/test_utils/models/upk/conftest.py b/test_utils/models/upk/conftest.py index ad96bbb..520d566 100644 --- a/test_utils/models/upk/conftest.py +++ b/test_utils/models/upk/conftest.py @@ -33,6 +33,13 @@ def insert_data_from_csv( df["id"] = df["id"].map(lambda x: UUID(x).hex) args = df.to_dict("tight")["data"] + # Fix issue with nullable cols + if table_name == 'marketplace_question': + for arg in args: + for idx, col in enumerate(arg): + if pd.isnull(col): + arg[idx] = None + with thl_web_rw.make_connection() as conn: with conn.cursor() as c: if disable_fk_checks: diff --git a/tests/managers/thl/test_ledger/test_lm_tx.py b/tests/managers/thl/test_ledger/test_lm_tx.py index 025f6ac..d98b838 100644 --- a/tests/managers/thl/test_ledger/test_lm_tx.py +++ b/tests/managers/thl/test_ledger/test_lm_tx.py @@ -206,12 +206,12 @@ class TestLedgerManagerGetTx: self, ledger_tx: LedgerTransaction, tag: str, ledger_manager: LedgerManager ): # (1) search for a random tag - res = ledger_manager.get_tx_by_tag(tag="aaa:bbb") + res = ledger_manager.get_txs_by_tag(tag="aaa:bbb") assert isinstance(res, list) assert len(res) == 0 # (2) search for the tag that was used during ledger_transaction creation - res = ledger_manager.get_tx_by_tag(tag=tag) + res = ledger_manager.get_txs_by_tag(tag=tag) assert isinstance(res, list) assert len(res) == 1 diff --git a/tests/managers/thl/test_ledger/test_lm_tx_locks.py b/tests/managers/thl/test_ledger/test_lm_tx_locks.py index 166598e..91d9cbc 100644 --- a/tests/managers/thl/test_ledger/test_lm_tx_locks.py +++ b/tests/managers/thl/test_ledger/test_lm_tx_locks.py @@ -38,7 +38,6 @@ logger = logging.getLogger("LedgerManager") class TestLedgerLocks: - def test_a( self, user_factory: Callable[..., User], @@ -108,16 +107,16 @@ class TestLedgerLocks: f1 = generate_condition_mp_payment(wall=w1) f2 = generate_condition_mp_payment(wall=w2) f3 = generate_condition_mp_payment(wall=w3) - assert f1(ledger_manager) is False - assert f2(lm=ledger_manager) is False - assert f3(lm=ledger_manager) is True + assert f1(ledger_manager) == (False, "duplicate tag") + assert f2(lm=ledger_manager) == (False, "duplicate tag") + assert f3(lm=ledger_manager)[0] == True condition = f3 create_tx_func = lambda: thl_ledger_manager.create_tx_task_complete_( wall=w3, user=user ) assert isinstance(create_tx_func, Callable) - assert f3(ledger_manager) is True + assert f3(ledger_manager)[0] is True ledger_manager.redis_client.delete(flag_name) ledger_manager.redis_client.delete(lock_name) @@ -125,13 +124,14 @@ class TestLedgerLocks: tx = thl_ledger_manager.create_tx_protected( lock_key=lock_key, condition=condition, create_tx_func=create_tx_func ) - assert f3(ledger_manager) is False + assert f3(ledger_manager) == (False, "duplicate tag") # purposely hold the lock open tx = None ledger_manager.redis_client.set(lock_name, "1") - with caplog.at_level(logging.ERROR), pytest.raises( - expected_exception=LedgerTransactionCreateLockError + with ( + caplog.at_level(logging.ERROR), + pytest.raises(expected_exception=LedgerTransactionCreateLockError), ): tx = thl_ledger_manager.create_tx_protected( lock_key=lock_key, @@ -231,8 +231,9 @@ class TestLedgerLocks: # Purposely hold the lock open ledger_manager.redis_client.set(name=lock_name, value="1") - with caplog.at_level(logging.DEBUG), pytest.raises( - expected_exception=LedgerTransactionCreateLockError + with ( + caplog.at_level(logging.DEBUG), + pytest.raises(expected_exception=LedgerTransactionCreateLockError), ): tx = thl_ledger_manager.create_tx_task_complete( wall=wall3, user=user, created=wall3.started diff --git a/tests/managers/thl/test_ledger/test_thl_pem.py b/tests/managers/thl/test_ledger/test_thl_pem.py index 2d03b25..348a02d 100644 --- a/tests/managers/thl/test_ledger/test_thl_pem.py +++ b/tests/managers/thl/test_ledger/test_thl_pem.py @@ -8,13 +8,9 @@ from uuid import UUID, uuid4 import pytest from generalresearch.currency import USDCent -from generalresearch.models.thl.definitions import PayoutStatus from generalresearch.models.thl.payout import ( BrokerageProductPayoutEvent, ) -from generalresearch.models.thl.wallet.cashout_method import ( - CashoutRequestInfo, -) if TYPE_CHECKING: from generalresearch.managers.thl.ledger_manager.ledger import LedgerManager @@ -23,6 +19,7 @@ if TYPE_CHECKING: ) from generalresearch.managers.thl.payout import ( BrokerageProductPayoutEventManager, + BusinessPayoutEventManager, UserPayoutEventManager, ) from generalresearch.models.thl.payout import UserPayoutEvent @@ -153,6 +150,7 @@ class TestThlPayoutEventManager: create_main_accounts: Callable[..., None], thl_ledger_manager: ThlLedgerManager, brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager, + business_payout_event_manager: BusinessPayoutEventManager, ledger_manager: LedgerManager, ): @@ -166,13 +164,13 @@ class TestThlPayoutEventManager: # Save a Brokerage Product Payout, so we have something in the # Payout Event table and the respective ledger TX and Entry rows for it - pe = brokerage_product_payout_event_manager.create_bp_payout_event( + bus_pe = business_payout_event_manager.create_bp_payout_event( thl_ledger_manager=thl_ledger_manager, product=product, amount=USDCent(rand_amount), - skip_wallet_balance_check=True, - skip_one_per_day_check=True, + ext_ref_id=uuid4().hex ) + pe = bus_pe.bp_payouts[0] assert isinstance(pe, BrokerageProductPayoutEvent) # Now try to query for it! @@ -183,7 +181,7 @@ class TestThlPayoutEventManager: res = thl_ledger_manager.get_tx_bp_payouts(account_uuids=[uuid4().hex]) assert len(res) == 0 - # Confirm it added to the users balance. The amount is negative because + # Confirm it added to the user's balance. The amount is negative because # money was sent to the Brokerage product: Product, but they didn't have # any activity that earned them money bal = ledger_manager.get_account_balance(account=account_bp_wallet) diff --git a/tests/managers/thl/test_payout.py b/tests/managers/thl/test_payout.py index a9cd52a..be121d8 100644 --- a/tests/managers/thl/test_payout.py +++ b/tests/managers/thl/test_payout.py @@ -388,7 +388,7 @@ class TestBusinessPayoutEventManager: ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=thl_ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -472,6 +472,7 @@ class TestBusinessPayoutEventManager: self, business_payout_event_manager: BusinessPayoutEventManager, request ): # TODO: Generate this file at random + # @max file is not commited fp = os.path.join( request.config.rootpath, "data/pytest_recoup_proportional.csv" ) @@ -690,7 +691,7 @@ class TestBusinessPayoutEventManager: ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -862,18 +863,18 @@ class TestBusinessPayoutEventManager: assert gr_business.payouts is None gr_business.prebuild_payouts( - thl_pg_config=thl_web_rr, - thl_lm=thl_ledger_manager, bpem=business_payout_event_manager, ) assert isinstance(gr_business.payouts, list) assert len(gr_business.payouts) == 1 assert gr_business.payouts[0].ext_ref_id == ach_id1 + ach_id2 = uuid4().hex + bp1 = business_payout_event_manager.create_from_ach_or_wire( business=gr_business, amount=USDCent(bb1.available_balance), - transaction_id=ach_id1, + transaction_id=ach_id2, pm=product_manager, thl_lm=thl_ledger_manager, created=start + timedelta(days=1, hours=5), @@ -885,16 +886,12 @@ class TestBusinessPayoutEventManager: bp1_tx = brokerage_product_payout_event_manager.check_for_ledger_tx( thl_ledger_manager=thl_ledger_manager, payout_event=bp1.bp_payouts[0], - product_id=bp1.bp_payouts[0].product_id, - amount=bp1.bp_payouts[0].amount, ) assert bp1_tx bp2_tx = brokerage_product_payout_event_manager.check_for_ledger_tx( thl_ledger_manager=thl_ledger_manager, payout_event=bp1.bp_payouts[1], - product_id=bp1.bp_payouts[1].product_id, - amount=bp1.bp_payouts[1].amount, ) assert bp2_tx @@ -996,13 +993,12 @@ class TestBusinessPayoutEventManager: wall_req_cpi=Decimal("50.00"), started=start + timedelta(days=1, hours=2, minutes=1 + idx), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) # Now that we paid out the business: Business, let's confirm the updated balances ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -1042,6 +1038,7 @@ class TestBusinessPayoutEventManager: pm=product_manager, thl_lm=thl_ledger_manager, created=start + timedelta(days=1, hours=3), + transaction_id=uuid4().hex, ) assert isinstance(bp1, BusinessPayoutEvent) assert len(bp1.bp_payouts) == 3 @@ -1057,7 +1054,7 @@ class TestBusinessPayoutEventManager: # balance.available_balance are reflective of having a $250 ACH/Wire # sent to the Business gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -1123,14 +1120,13 @@ class TestBusinessPayoutEventManager: wall_req_cpi=Decimal("7.50"), started=start + timedelta(days=1, hours=1 + iidx, minutes=1 + idx), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) rm_ledger_collection() rm_pop_ledger_merge() ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -1151,7 +1147,7 @@ class TestBusinessPayoutEventManager: ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, @@ -1175,12 +1171,10 @@ class TestBusinessPayoutEventManager: ledger_collection.initial_load(client=None, sync=True) pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) gr_business.prebuild_payouts( - thl_pg_config=thl_web_rr, - thl_lm=thl_ledger_manager, bpem=business_payout_event_manager, ) gr_business.prebuild_balance( - thl_pg_config=thl_web_rr, + product_manager=product_manager, lm=ledger_manager, ds=mnt_filepath, client=client_no_amm, diff --git a/tests/managers/thl/test_session_manager.py b/tests/managers/thl/test_session_manager.py index 60edcb9..4b122ec 100644 --- a/tests/managers/thl/test_session_manager.py +++ b/tests/managers/thl/test_session_manager.py @@ -117,6 +117,7 @@ class TestSessionManagerFilter: def test_team( self, product_factory: Callable[..., Product], + product_manager, user_factory: Callable[..., User], gr_team: Team, session_manager: SessionManager, @@ -129,7 +130,7 @@ class TestSessionManagerFilter: u = user_factory(product=p1) session_manager.create(started=utc_hour_ago, user=u, uuid_id=uuid4().hex) - gr_team.prefetch_products(thl_pg_config=thl_web_rr) + gr_team.prefetch_products(product_manager=product_manager) assert len(gr_team.product_uuids) == 1 res = session_manager.filter(product_uuids=gr_team.product_uuids) assert len(res) == 5 @@ -142,6 +143,7 @@ class TestSessionManagerFilter: session_manager: SessionManager, utc_hour_ago: datetime, thl_web_rr: PostgresConfig, + product_manager, ): p1 = product_factory(business=gr_business) @@ -149,7 +151,7 @@ class TestSessionManagerFilter: u = user_factory(product=p1) session_manager.create(started=utc_hour_ago, user=u, uuid_id=uuid4().hex) - gr_business.prefetch_products(thl_pg_config=thl_web_rr) + gr_business.prefetch_products(product_manager=product_manager) assert len(gr_business.product_uuids) == 1 res = session_manager.filter(product_uuids=gr_business.product_uuids) assert len(res) == 5 diff --git a/tests/managers/thl/test_survey_penalty.py b/tests/managers/thl/test_survey_penalty.py index 04f69d2..71cea16 100644 --- a/tests/managers/thl/test_survey_penalty.py +++ b/tests/managers/thl/test_survey_penalty.py @@ -103,7 +103,7 @@ class TestSurveyPenalty: assert res == {"t:a": 0.1, "t:b": 0.2, "u:b": 0.1} assert surveypenalty_manager.cache.currsize == 1 cached_key = tuple(list(next(iter(surveypenalty_manager.cache.keys())))[1:]) - assert cached_key == ("product_id", product_uuid, "team_id", team_id_random) + assert cached_key == (("product_id", product_uuid), ("team_id", team_id_random)) # Both don't exist, return nothing res = surveypenalty_manager.get_penalties_for( diff --git a/tests/managers/thl/test_user_manager/test_base.py b/tests/managers/thl/test_user_manager/test_base.py index 5d12052..2a7e879 100644 --- a/tests/managers/thl/test_user_manager/test_base.py +++ b/tests/managers/thl/test_user_manager/test_base.py @@ -56,43 +56,43 @@ class TestUserManager: def test_get_user_no_inmemory(self): self.user_manager.clear_user_inmemory_cache(self.user) - self.user_manager.get_user.__wrapped__.cache_clear() + self.user_manager.cache_clear() u = self.user_manager.get_user(user_id=self.user.user_id) # this should hit mysql assert u == self.user - cache_info = self.user_manager.get_user.__wrapped__.cache_info() - assert cache_info.hits == 0, cache_info - assert cache_info.misses == 1, cache_info + cache_info = self.user_manager.cache_info() + assert cache_info['hits'] == 0, cache_info + assert cache_info['misses'] == 1, cache_info # this should hit the lru cache u = self.user_manager.get_user(user_id=self.user.user_id) assert u == self.user - cache_info = self.user_manager.get_user.__wrapped__.cache_info() - assert cache_info.hits == 1, cache_info - assert cache_info.misses == 1, cache_info + cache_info = self.user_manager.cache_info() + assert cache_info['hits'] == 1, cache_info + assert cache_info['misses'] == 1, cache_info def test_get_user_with_inmemory(self): # user_manager = self.get_user_manager() self.user_manager.set_user_inmemory_cache(self.user) - self.user_manager.get_user.__wrapped__.cache_clear() + self.user_manager.cache_clear() u = self.user_manager.get_user(user_id=self.user.user_id) # this should hit inmemory cache assert u == self.user - cache_info = self.user_manager.get_user.__wrapped__.cache_info() - assert cache_info.hits == 0, cache_info - assert cache_info.misses == 1, cache_info + cache_info = self.user_manager.cache_info() + assert cache_info['hits'] == 0, cache_info + assert cache_info['misses'] == 1, cache_info # this should hit the lru cache u = self.user_manager.get_user(user_id=self.user.user_id) assert u == self.user - cache_info = self.user_manager.get_user.__wrapped__.cache_info() - assert cache_info.hits == 1, cache_info - assert cache_info.misses == 1, cache_info + cache_info = self.user_manager.cache_info() + assert cache_info['hits'] == 1, cache_info + assert cache_info['misses'] == 1, cache_info class TestBlockUserManager: diff --git a/tests/managers/thl/test_user_manager/test_redis.py b/tests/managers/thl/test_user_manager/test_redis.py index f6b59c9..89d0a7a 100644 --- a/tests/managers/thl/test_user_manager/test_redis.py +++ b/tests/managers/thl/test_user_manager/test_redis.py @@ -8,15 +8,14 @@ from generalresearch.managers.base import Permission from generalresearch.managers.thl.user_manager.redis_user_manager import ( RedisUserManager, ) +from generalresearch.redis_helper import RedisConfig if TYPE_CHECKING: - from generalresearch.config import GRLBaseSettings from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig class TestUserManagerRedis: - def test_get_notset(self, redis_user_manager: RedisUserManager, user: User): redis_user_manager.clear_user(user=user) assert redis_user_manager.get_user(user_id=user.user_id) is None @@ -48,10 +47,10 @@ class TestUserManagerRedis: def test_get_with_cache_prefix( self, - settings: GRLBaseSettings, user: User, thl_web_rw: PostgresConfig, thl_web_rr: PostgresConfig, + thl_redis_config: RedisConfig, ): """ Confirm the prefix functionality is working; we do this so it @@ -67,19 +66,19 @@ class TestUserManagerRedis: pg_config=thl_web_rw, pg_config_rr=thl_web_rr, sql_permissions=[Permission.UPDATE, Permission.CREATE], - redis=settings.redis, - redis_timeout=settings.redis_timeout, + redis=thl_redis_config.dsn, + redis_timeout=1, ) um2 = UserManager( pg_config=thl_web_rw, pg_config_rr=thl_web_rr, sql_permissions=[Permission.UPDATE, Permission.CREATE], - redis=settings.redis, - redis_timeout=settings.redis_timeout, + redis=thl_redis_config.dsn, + redis_timeout=1, cache_prefix="user-lookup-v2", ) - + user = user.to_user_ref() um1.get_or_create_user( product_id=user.product_id, product_user_id=user.product_user_id ) diff --git a/tests/managers/thl/test_userhealth.py b/tests/managers/thl/test_userhealth.py index 8256185..bd2152a 100644 --- a/tests/managers/thl/test_userhealth.py +++ b/tests/managers/thl/test_userhealth.py @@ -226,13 +226,11 @@ class TestIPRecordManager: self, thl_web_rr: PostgresConfig, thl_redis_config: RedisConfig, - geoip_info_manager: GeoIpInfoManager, ip_record_manager: IPRecordManager, ): instance = IPRecordManager( pg_config=thl_web_rr, redis_config=thl_redis_config, - geoip_info_manager=geoip_info_manager, ) assert isinstance(instance, IPRecordManager) assert isinstance(ip_record_manager, IPRecordManager) -- cgit v1.2.3 From 83e587c8f5431d9370429f623709092a60e006fd Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 15:15:55 -0600 Subject: fix some more tests --- tests/models/gr/test_business.py | 6 +----- tests/models/gr/test_team.py | 4 ++-- tests/models/test_finance.py | 11 +++++++++-- tests/models/thl/test_user_iphistory.py | 7 ++++++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/tests/models/gr/test_business.py b/tests/models/gr/test_business.py index e38850d..8de2a47 100644 --- a/tests/models/gr/test_business.py +++ b/tests/models/gr/test_business.py @@ -11,11 +11,6 @@ from uuid import uuid4 import pandas as pd import pytest from dask.distributed import Client as DaskClient - -# noinspection PyUnresolvedReferences -from distributed.utils_test import ( - client_no_amm, -) from pytest import approx from generalresearch.currency import USDCent @@ -699,6 +694,7 @@ class TestBusinessBalance: pop_ledger_merge: PopLedgerMerge, adj_to_fail_with_tx_factory: Callable[..., None], delete_df_collection: Callable[..., None], + client_no_amm: DaskClient, ): """ - Product 1 $2.50 Complete diff --git a/tests/models/gr/test_team.py b/tests/models/gr/test_team.py index e853817..7c93209 100644 --- a/tests/models/gr/test_team.py +++ b/tests/models/gr/test_team.py @@ -107,7 +107,7 @@ class TestTeam: self, gr_team: Team, gr_business: Business, - team_manager: TeamManager, + gr_team_manager: TeamManager, gr_business_manager: BusinessManager, ): @@ -117,7 +117,7 @@ class TestTeam: assert isinstance(gr_team.businesses, list) assert len(gr_team.businesses) == 0 - team_manager.add_business(team=gr_team, business=gr_business) + gr_team_manager.add_business(team=gr_team, business=gr_business) assert len(gr_team.businesses) == 0 gr_team.prefetch_businesses(gr_business_manager=gr_business_manager) assert len(gr_team.businesses) == 1 diff --git a/tests/models/test_finance.py b/tests/models/test_finance.py index a1da961..53df014 100644 --- a/tests/models/test_finance.py +++ b/tests/models/test_finance.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections.abc import Callable from datetime import UTC, datetime, timedelta +from decimal import Decimal from itertools import product as iter_product from random import randint from typing import TYPE_CHECKING @@ -839,16 +840,22 @@ class TestBusinessBalanceData: delete_df_collection(coll=ledger_collection) rm_ledger_collection() + # We're expecting 5 time periods x 5 $.50 complete (minus 5% commission = $0.48) + # = $12.00 for _ in range(5): u: User = user_factory(product=product, created=ledger_collection.start) for item in ledger_collection.items: + # A time may be randomly outside the date range, as the + # session_with_tx_factory adds time to each wall event it creates item_time = fake.date_time_between( start_date=item.start, - end_date=item.finish, + end_date=item.finish - timedelta(minutes=10), tzinfo=UTC, ) - session_with_tx_factory(started=item_time, user=u) + session_with_tx_factory( + started=item_time, user=u, wall_req_cpi=Decimal("0.50") + ) item.initial_load(overwrite=True) # Confirm any of the items are archived diff --git a/tests/models/thl/test_user_iphistory.py b/tests/models/thl/test_user_iphistory.py index b8a0be3..efd5168 100644 --- a/tests/models/thl/test_user_iphistory.py +++ b/tests/models/thl/test_user_iphistory.py @@ -1,11 +1,13 @@ from __future__ import annotations from datetime import UTC, datetime, timedelta +from uuid import uuid4 from generalresearch.models.thl.user_iphistory import ( UserIPHistory, UserIPRecord, ) +from generalresearch.models.thl.user_ref import UserRef def test_collapse_ip_records(): @@ -33,7 +35,10 @@ def test_collapse_ip_records(): ), UserIPRecord(ip="1.2.3.6", created=now + timedelta(minutes=7)), ] - iph = UserIPHistory(user_id=1, ips=records) + iph = UserIPHistory( + user=UserRef(user_id=1, product_user_id=uuid4().hex, product_id=uuid4().hex), + ips=records, + ) res = iph.collapse_ip_records() # We should be left with one of the 1.2.3.5 ipv4s, -- cgit v1.2.3 From e3c6a824f4176402e40217c5ce70f837dedc5c3d Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 18 Sep 2026 15:19:18 -0600 Subject: bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3102e39..ee1d222 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "generalresearch" -version = "3.4.7" +version = "3.5.0" description = "Python Utilities for General Research" readme = "README.md" requires-python = ">=3.14" -- cgit v1.2.3