handle LedgerTransactionReleaseLockError in business_payout_event_manager.create_bp_payout_event. Fix related tests.

3 files changed, 86 insertions, 101 deletions

diff --git a/generalresearch/managers/thl/ledger_manager/thl_ledger.py b/generalresearch/managers/thl/ledger_manager/thl_ledger.py
index 0f8c6d3..e5a308f 100644
--- a/generalresearch/managers/thl/ledger_manager/thl_ledger.py
+++ b/generalresearch/managers/thl/ledger_manager/thl_ledger.py
@@ -830,12 +830,13 @@ class ThlLedgerManager(LedgerManager):
         created: datetime,
     ) -> LedgerTransaction:
 
+        tx_type = TransactionType.BP_PAYOUT
         metadata = {
-            tmc.TX_TYPE: TransactionType.BP_PAYOUT,
+            tmc.TX_TYPE: tx_type,
             tmc.EVENT: payoutevent_uuid,
         }
         # This tag will uniquely identify this tx
-        tag = f"{self.currency.value}:bp_payout:{payoutevent_uuid}"
+        tag = f"{self.currency.value}:{tx_type.value}:{payoutevent_uuid}"
         cash_account = self.get_account_cash()
         bp_wallet_account = self.get_account_or_create_bp_wallet(product)
 
diff --git a/generalresearch/managers/thl/payout.py b/generalresearch/managers/thl/payout.py
index 5397ff4..59dc2e9 100644
--- a/generalresearch/managers/thl/payout.py
+++ b/generalresearch/managers/thl/payout.py
@@ -18,12 +18,14 @@ from generalresearch.managers.base import (
 )
 from generalresearch.managers.thl.ledger_manager.exceptions import (
     LedgerTransactionConditionFailedError,
+    LedgerTransactionReleaseLockError,
 )
 from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
 from generalresearch.models.thl.definitions import PayoutStatus
 from generalresearch.models.thl.ledger import (
     Direction,
     OrderBy,
+    TransactionType,
 )
 from generalresearch.models.thl.payout import (
     BrokerageProductPayoutEvent,
@@ -97,9 +99,9 @@ class PayoutEventManager(PostgresManagerWithRedis):
         with self.pg_config.make_connection() as conn:
             with conn.cursor() as c:
                 c.execute(query=query, params=d)
-                assert (
-                    c.rowcount == 1
-                ), "Nothing was updated! Are you sure this payout_event exists?"
+                assert c.rowcount == 1, (
+                    "Nothing was updated! Are you sure this payout_event exists?"
+                )
             conn.commit()
 
 
@@ -567,6 +569,13 @@ class BrokerageProductPayoutEventManager(PayoutEventManager):
                 """) from e
             self.update(payout_event=bp_pe, status=PayoutStatus.FAILED)
             raise
+        except LedgerTransactionReleaseLockError as e:
+            # Redis error upon lock release. The tx was most likely created.
+            LOG.warning(e)
+            tag = f"{thl_ledger_manager.currency.value}:{TransactionType.BP_PAYOUT.value}:{bp_pe.uuid}"
+            # Check if it was created, and if so, swallow error.
+            if not thl_ledger_manager.get_tx_ids_by_tag(tag=tag):
+                self.update(payout_event=bp_pe, status=PayoutStatus.FAILED)
         except Exception:
             self.update(payout_event=bp_pe, status=PayoutStatus.FAILED)
             raise
@@ -632,9 +641,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
         for bp_payout in d["bp_payouts"]:
             bp_payout["created"] = datetime.fromisoformat(bp_payout["created"])
         bpe = BusinessPayoutEvent.model_validate(d)
-        assert (
-            bpe.bp_payouts is not None and len(bpe.bp_payouts) > 0
-        ), "No BP payouts found for this Business Payout Event. This shouldn't happen!"
+        assert bpe.bp_payouts is not None and len(bpe.bp_payouts) > 0, (
+            "No BP payouts found for this Business Payout Event. This shouldn't happen!"
+        )
         return bpe
 
     def filter_by(
@@ -679,9 +688,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
             for bp_payout in row["bp_payouts"]:
                 bp_payout["created"] = datetime.fromisoformat(bp_payout["created"])
             bpe = BusinessPayoutEvent.model_validate(row)
-            assert (
-                bpe.bp_payouts is not None and len(bpe.bp_payouts) > 0
-            ), "No BP payouts found for this Business Payout Event. This shouldn't happen!"
+            assert bpe.bp_payouts is not None and len(bpe.bp_payouts) > 0, (
+                "No BP payouts found for this Business Payout Event. This shouldn't happen!"
+            )
             bpes.append(bpe)
         return bpes
 
@@ -698,9 +707,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
             for bp_pe in bpe.bp_payouts
         ]
         txs = thl_lm.get_tx_ids_by_tags(tags=tags)
-        assert len(txs) == len(
-            bpe.bp_payouts
-        ), f"Expected {len(bpe.bp_payouts)} BP payouts but found {len(txs)}!"
+        assert len(txs) == len(bpe.bp_payouts), (
+            f"Expected {len(bpe.bp_payouts)} BP payouts but found {len(txs)}!"
+        )
         return True
 
     def resume_failed_business_payout(
@@ -826,9 +835,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
             shortfall: int = int(target_amount) - w_df["deduction"].sum()
             w_df["remaining_balance"] = w_df["available_balance"] - w_df["deduction"]
 
-        assert w_df[
-            w_df["deduction"] > w_df["available_balance"]
-        ].empty, "Trying to deduct more from an Product than what is available"
+        assert w_df[w_df["deduction"] > w_df["available_balance"]].empty, (
+            "Trying to deduct more from an Product than what is available"
+        )
 
         return w_df
 
@@ -900,9 +909,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
     ) -> BusinessPayoutEvent:
         assert isinstance(bpe, BusinessPayoutEventCreate)
         assert bpe.bp_payouts, "Must provide at least one BP Payout"
-        assert {bp_pe.status for bp_pe in bpe.bp_payouts} == {
-            PayoutStatus.PENDING
-        }, "All BP Payouts must be PENDING"
+        assert {bp_pe.status for bp_pe in bpe.bp_payouts} == {PayoutStatus.PENDING}, (
+            "All BP Payouts must be PENDING"
+        )
         INSERT_SUPPLIER_PAYOUT = """
         INSERT INTO supplier_payout (
             business_id, created, amount,
@@ -995,9 +1004,9 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
         # Can't pay any Products that don't have a remaining balance
         df = df[df["remaining_balance"] > 0].copy()
 
-        assert (
-            df.deduction.sum() == business.balance.recoup
-        ), "recoup_proportional failure"
+        assert df.deduction.sum() == business.balance.recoup, (
+            "recoup_proportional failure"
+        )
 
         df["issue_amount"] = BusinessPayoutEventManager.distribute_amount(
             df=df, amount=amount
@@ -1103,8 +1112,7 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
         account = thl_ledger_manager.get_account(
             f"{thl_ledger_manager.currency.value}:bp_wallet:{product.uuid}"
         )
-        bpe = BusinessPayoutEvent(
-            id=None,
+        bpe = BusinessPayoutEventCreate(
             business_id=product.business_uuid,
             payout_type=PayoutType.ACH,
             amount=amount,
@@ -1125,7 +1133,7 @@ class BusinessPayoutEventManager(PostgresManagerWithRedis):
                 )
             ],
         )
-        self.create_business_payout_event(bpe=bpe)
+        bpe = self.create_business_payout_event(bpe=bpe)
         self.bp_pe_manager.create_tx_bp_payout_from_payout_event(
             thl_ledger_manager=thl_ledger_manager,
             bp_pe=bpe.bp_payouts[0],
diff --git a/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py b/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py
index de368d8..27ddc29 100644
--- a/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py
+++ b/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py
@@ -32,13 +32,13 @@ from generalresearch.models.thl.session import (
     StatusCode1,
     Wall,
 )
-from generalresearch.models.thl.wallet.definitions import PayoutType
 from generalresearch.redis_helper import RedisConfig
 
 if TYPE_CHECKING:
     from generalresearch.currency import LedgerCurrency
     from generalresearch.managers.thl.payout import (
         BrokerageProductPayoutEventManager,
+        BusinessPayoutEventManager,
     )
     from generalresearch.models.thl.product import Product
     from generalresearch.models.thl.user import User
@@ -353,14 +353,13 @@ class TestThlLedgerManagerBPPayout:
         # Create TX will fail on lock exit, after the tx was created!
         with monkeypatch.context() as m:
             m.setattr(Lock, "release", broken_release)
-            with pytest.raises(expected_exception=Exception) as e:
+            with pytest.raises(LedgerTransactionReleaseLockError) as e:
                 thl_ledger_manager.create_tx_bp_payout(
                     product=product,
                     amount=rand_amount,
                     payoutevent_uuid=payoutevent_uuid,
                     created=datetime.now(tz=UTC),
                 )
-        assert e.type is LedgerTransactionReleaseLockError
         assert str(e.value) == "Redis error: Simulated timeout during release"
 
         # Transaction was still created!
@@ -372,11 +371,16 @@ class TestThlLedgerManagerBPPayout:
 
 
 class TestPayoutEventManagerBPPayout:
+    @pytest.fixture(autouse=True)
+    def setup(self, create_main_accounts):
+        create_main_accounts()
+
     def test_create(
         self,
         product: Product,
         thl_ledger_manager: ThlLedgerManager,
         brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager,
+        business_payout_event_manager: BusinessPayoutEventManager,
     ):
         rand_amount: USDCent = USDCent(randint(100, 1_000))
         now = datetime.now(tz=UTC)
@@ -389,18 +393,17 @@ class TestPayoutEventManagerBPPayout:
         )
         assert thl_ledger_manager.get_account_balance(bp_wallet_account) == rand_amount
 
-        pe = brokerage_product_payout_event_manager.create_bp_payout_event(
+        bpe = business_payout_event_manager.create_bp_payout_event(
             thl_ledger_manager=thl_ledger_manager,
             product=product,
             created=now,
             amount=rand_amount,
-            payout_type=PayoutType.ACH,
+            ext_ref_id=uuid4().hex,
         )
+        bp_pe = bpe.bp_payouts[0]
         assert brokerage_product_payout_event_manager.check_for_ledger_tx(
             thl_ledger_manager=thl_ledger_manager,
-            product_id=product.id,
-            amount=rand_amount,
-            payout_event=pe,
+            payout_event=bp_pe,
         )
         assert thl_ledger_manager.get_account_balance(bp_wallet_account) == 0
 
@@ -410,10 +413,11 @@ class TestPayoutEventManagerBPPayout:
         caplog,
         thl_ledger_manager: ThlLedgerManager,
         brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager,
+        business_payout_event_manager: BusinessPayoutEventManager,
+        monkeypatch: pytest.MonkeyPatch,
     ):
         caplog.set_level("WARNING")
-        original_acquire = Lock.acquire
-        original_release = Lock.release
+        ext_ref_id = uuid4().hex
 
         rand_amount: USDCent = USDCent(randint(100, 1_000))
         now = datetime.now(tz=UTC)
@@ -427,21 +431,17 @@ class TestPayoutEventManagerBPPayout:
         assert thl_ledger_manager.get_account_balance(bp_wallet_account) == rand_amount
 
         # Will fail on lock enter, no tx will actually get created
-        Lock.acquire = broken_acquire
-        with pytest.raises(expected_exception=Exception) as e:
-            pe = brokerage_product_payout_event_manager.create_bp_payout_event(
-                thl_ledger_manager=thl_ledger_manager,
-                product=product,
-                created=now,
-                amount=rand_amount,
-                payout_type=PayoutType.ACH,
-            )
-        assert e.type is LedgerTransactionCreateError
+        with monkeypatch.context() as m:
+            m.setattr(Lock, "acquire", broken_acquire)
+            with pytest.raises(LedgerTransactionCreateError) as e:
+                business_payout_event_manager.create_bp_payout_event(
+                    thl_ledger_manager=thl_ledger_manager,
+                    product=product,
+                    created=now,
+                    amount=rand_amount,
+                    ext_ref_id=ext_ref_id,
+                )
         assert str(e.value) == "Redis error: Simulated timeout during acquire"
-        assert any(
-            "Simulated timeout during acquire. No ledger tx was created" in m
-            for m in caplog.messages
-        )
 
         txs = thl_ledger_manager.get_tx_filtered_by_account(
             account_uuid=bp_wallet_account.uuid
@@ -451,21 +451,18 @@ class TestPayoutEventManagerBPPayout:
         assert len(txs) == 0
         pes = (
             brokerage_product_payout_event_manager.get_bp_bp_payout_events_for_products(
-                thl_ledger_manager=thl_ledger_manager, product_uuids=[product.id]
+                product_uuids=[product.id]
             )
         )
         assert len(pes) == 1
         assert pes[0].status == PayoutStatus.FAILED
         pe = pes[0]
 
-        # Fix the redis method
-        Lock.acquire = original_acquire
-
         # Try to fix the failed payout, by trying ledger tx again
         brokerage_product_payout_event_manager.retry_create_bp_payout_event_tx(
             product=product,
             thl_ledger_manager=thl_ledger_manager,
-            payout_event_uuid=pe.uuid,
+            bp_pe=pe,
         )
         txs = thl_ledger_manager.get_tx_filtered_by_account(
             account_uuid=bp_wallet_account.uuid
@@ -475,44 +472,28 @@ class TestPayoutEventManagerBPPayout:
         assert thl_ledger_manager.get_account_balance(bp_wallet_account) == 0
 
         # And then try to run it again, it'll fail because a payout event with the same info exists
-        with pytest.raises(expected_exception=Exception) as e:
-            pe = brokerage_product_payout_event_manager.create_bp_payout_event(
+        with pytest.raises(expected_exception=ValueError) as e:
+            pe = business_payout_event_manager.create_bp_payout_event(
                 thl_ledger_manager=thl_ledger_manager,
                 product=product,
                 created=now,
                 amount=rand_amount,
-                payout_type=PayoutType.ACH,
+                ext_ref_id=ext_ref_id,
             )
-        assert e.type is ValueError
-        assert "Payout event already exists!" in str(e.value)
+        assert (
+            "Cannot create a BusinessPayoutEvent with an existing transaction_id"
+            in str(e.value)
+        )
 
         # We wouldn't do this in practice, because this is paying out the BP again, but
         #   we can if want to.
-        # Change the timestamp so it'll create a new payout event
-        now = datetime.now(tz=UTC)
-        with pytest.raises(LedgerTransactionConditionFailedError) as e:
-            pe = brokerage_product_payout_event_manager.create_bp_payout_event(
-                thl_ledger_manager=thl_ledger_manager,
-                product=product,
-                created=now,
-                amount=rand_amount,
-                payout_type=PayoutType.ACH,
-            )
-        # But it will fail due to 1 per day check
-        assert str(e.value) == ">1 tx per day"
-        pe = brokerage_product_payout_event_manager.get_by_uuid(e.value.pe_uuid)
-        assert pe.status == PayoutStatus.FAILED
-
-        # And if we really want to, we can make it again
-        now = datetime.now(tz=UTC)
-        pe = brokerage_product_payout_event_manager.create_bp_payout_event(
+        # Change the ext_ref_id so it'll create a new payout event
+        pe = business_payout_event_manager.create_bp_payout_event(
             thl_ledger_manager=thl_ledger_manager,
             product=product,
             created=now,
             amount=rand_amount,
-            payout_type=PayoutType.ACH,
-            skip_one_per_day_check=True,
-            skip_wallet_balance_check=True,
+            ext_ref_id=uuid4().hex,
         )
 
         txs = thl_ledger_manager.get_tx_filtered_by_account(
@@ -525,20 +506,17 @@ class TestPayoutEventManagerBPPayout:
             thl_ledger_manager.get_account_balance(bp_wallet_account) == 0 - rand_amount
         )
 
-        Lock.release = original_release
-        Lock.acquire = original_acquire
-
     def test_create_with_redis_error_release(
         self,
         product: Product,
-        caplog,
         thl_ledger_manager: ThlLedgerManager,
+        business_payout_event_manager: BusinessPayoutEventManager,
         brokerage_product_payout_event_manager: BrokerageProductPayoutEventManager,
+        monkeypatch: pytest.MonkeyPatch,
+        caplog: pytest.LogCaptureFixture,
     ):
         caplog.set_level("WARNING")
 
-        original_release = Lock.release
-
         rand_amount: USDCent = USDCent(randint(100, 1_000))
         now = datetime.now(tz=UTC)
         bp_wallet_account = thl_ledger_manager.get_account_or_create_bp_wallet(
@@ -553,18 +531,17 @@ class TestPayoutEventManagerBPPayout:
 
         # Will fail on lock exit, after the tx was created!
         # But it'll see that the tx was created and so everything will be fine
-        Lock.release = broken_release
-        brokerage_product_payout_event_manager.create_bp_payout_event(
-            thl_ledger_manager=thl_ledger_manager,
-            product=product,
-            created=now,
-            amount=rand_amount,
-            payout_type=PayoutType.ACH,
-        )
-        assert any(
-            "Simulated timeout during release but ledger tx exists" in m
-            for m in caplog.messages
-        )
+        caplog.clear()
+        with monkeypatch.context() as m, caplog.at_level("WARNING"):
+            m.setattr(Lock, "release", broken_release)
+            business_payout_event_manager.create_bp_payout_event(
+                thl_ledger_manager=thl_ledger_manager,
+                product=product,
+                created=now,
+                amount=rand_amount,
+                ext_ref_id=uuid4().hex,
+            )
+        assert "Redis error: Simulated timeout during release" in caplog.messages
 
         txs = thl_ledger_manager.get_tx_filtered_by_account(
             account_uuid=bp_wallet_account.uuid
@@ -573,9 +550,8 @@ class TestPayoutEventManagerBPPayout:
         assert len(txs) == 1
         pes = (
             brokerage_product_payout_event_manager.get_bp_bp_payout_events_for_products(
-                thl_ledger_manager=thl_ledger_manager, product_uuids=[product.uuid]
+                product_uuids=[product.uuid]
             )
         )
         assert len(pes) == 1
         assert pes[0].status == PayoutStatus.COMPLETE
-        Lock.release = original_release