Merge branch 'master' into dev-greg

1 files changed, 236 insertions, 19 deletions

diff --git a/test_utils/spectrum/conftest.py b/test_utils/spectrum/conftest.py
index 0afc3f5..cc91cff 100644
--- a/test_utils/spectrum/conftest.py
+++ b/test_utils/spectrum/conftest.py
@@ -1,7 +1,9 @@
-import logging
+from __future__ import annotations
+
 import time
-from datetime import datetime, timezone
-from typing import TYPE_CHECKING
+from datetime import UTC, datetime
+from decimal import Decimal
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
@@ -9,21 +11,24 @@ from generalresearch.managers.spectrum.survey import (
     SpectrumCriteriaManager,
     SpectrumSurveyManager,
 )
-from generalresearch.models.spectrum.survey import SpectrumSurvey
+from generalresearch.models.definitions import (
+    LogicalOperator,
+)
+from generalresearch.models.spectrum.survey import (
+    SpectrumCondition,
+    SpectrumSurvey,
+)
+from generalresearch.models.thl.survey.condition import ConditionValueType
 from generalresearch.sql_helper import SqlHelper
 
-from .surveys_json import CONDITIONS, SURVEYS_JSON
-
 if TYPE_CHECKING:
     from generalresearch.config import GRLBaseSettings
 
 
 @pytest.fixture(scope="session")
-def spectrum_rw(settings: "GRLBaseSettings") -> SqlHelper:
-    logging.info(f"{settings.spectrum_rw_db=}")
-
+def spectrum_rw(settings: GRLBaseSettings) -> SqlHelper:
     assert settings.spectrum_rw_db is not None
-    assert "/unittest-" in settings.spectrum_rw_db.path
+    assert "/unittest-" in str(settings.spectrum_rw_db.path)
 
     return SqlHelper(
         dsn=settings.spectrum_rw_db,
@@ -35,27 +40,36 @@ def spectrum_rw(settings: "GRLBaseSettings") -> SqlHelper:
 
 @pytest.fixture(scope="session")
 def spectrum_criteria_manager(spectrum_rw: SqlHelper) -> SpectrumCriteriaManager:
+    assert spectrum_rw.dsn
+    assert spectrum_rw.dsn.path
     assert "/unittest-" in spectrum_rw.dsn.path
     return SpectrumCriteriaManager(spectrum_rw)
 
 
 @pytest.fixture(scope="session")
 def spectrum_survey_manager(spectrum_rw: SqlHelper) -> SpectrumSurveyManager:
+    assert spectrum_rw.dsn
+    assert spectrum_rw.dsn.path
     assert "/unittest-" in spectrum_rw.dsn.path
     return SpectrumSurveyManager(spectrum_rw)
 
 
 @pytest.fixture(scope="session")
 def setup_spectrum_surveys(
-    spectrum_rw: SqlHelper, spectrum_survey_manager, spectrum_criteria_manager
+    spectrum_rw: SqlHelper,
+    spectrum_survey_manager: SpectrumSurveyManager,
+    spectrum_criteria_manager: SpectrumCriteriaManager,
+    spectrum_conditions: list[SpectrumCondition],
+    spectrum_api_surveys_json: list[str],
 ) -> None:
-    now = datetime.now(timezone.utc)
+    now = datetime.now(UTC)
     # make sure these example surveys exist in db
-    surveys = [SpectrumSurvey.model_validate_json(x) for x in SURVEYS_JSON]
+    surveys = [SpectrumSurvey.model_validate_json(x) for x in spectrum_api_surveys_json]
     for s in surveys:
-        s.modified_api = datetime.now(tz=timezone.utc)
+        s.modified_api = datetime.now(tz=UTC)
+
     spectrum_survey_manager.create_or_update(surveys)
-    spectrum_criteria_manager.update(CONDITIONS)
+    spectrum_criteria_manager.update(spectrum_conditions)
 
     # and make sure they have allocation for 687
     spectrum_rw.execute_sql_query(
@@ -66,10 +80,10 @@ def setup_spectrum_surveys(
         ["687", "GRL", "x", "x", "x", "x"],
         commit=True,
     )
-    supplier687_pk = spectrum_rw.execute_sql_query(
-        f"""
-    select id from `{spectrum_rw.db}`.spectrum_supplier where supplier_id = '687'"""
-    )[0]["id"]
+    supplier687_pk = spectrum_rw.execute_sql_query(f"""
+    select id from `{spectrum_rw.db}`.spectrum_supplier where supplier_id = '687'""")[
+        0
+    ]["id"]
     conn = spectrum_rw.make_connection()
     c = conn.cursor()
     c.executemany(
@@ -83,3 +97,206 @@ def setup_spectrum_surveys(
     conn.commit()
     # Wait a second to make sure the spectrum-grpc pulls these from the db into global-vars
     time.sleep(1)
+
+
+@pytest.fixture(scope="session")
+def spectrum_api_surveys_json() -> list[str]:
+    return [
+        (
+            '{"cpi":"3.90","country_isos":["us"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":["1235","212"],"survey_id":"111111","survey_name":"Exciting New Survey #14472374",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261,14126487,14361592,14376811,14385771,14387789,14472374",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"us","language_iso":"eng",'
+            '"include_psids":null,"exclude_psids":null'
+            ',"qualifications":["ee5e842","e6e0b0b"],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":["32cbf31"]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+        (
+            '{"cpi":"3.90","country_isos":["us"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":["1235","212"],"survey_id":"14472374","survey_name":"Exciting New Survey #14472374",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261,14126487,14361592,14376811,14385771,14387789,14472374",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"us","language_iso":"eng",'
+            '"include_psids":null,"exclude_psids":"0408319875e9dbffdc09e86671ad5636,23c4c66ecbc465906d0b0fd798740e64,'
+            '861df4603df3b7f754b8d4b89cbdb313","qualifications":["ee5e842","e6e0b0b"],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":["32cbf31"]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+        (
+            '{"cpi":"3.90","country_isos":["us"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":["1235","212"],"survey_id":"12345","survey_name":"Exciting New Survey #14472374",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261,14126487,14361592,14376811,14385771,14387789,14472374",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"us","language_iso":"eng",'
+            '"include_psids":"7d043991b1494dbbb57786b11c88239c","exclude_psids":null'
+            ',"qualifications":["ee5e842","e6e0b0b"],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":["32cbf31"]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+        (
+            '{"cpi":"1.40","country_isos":["us"],"language_isos":["eng"],"buyer_id":"233","bid_loi":null,"source":"s",'
+            '"used_question_ids":["245","244","212","211","225"],"survey_id":"14970164","survey_name":"Exciting New Survey '
+            '#14970164","status":22,"field_end_date":"2024-05-07T16:18:33.000000Z","category_code":"232",'
+            '"calculation_type":"COMPLETES","requires_pii":false,"survey_exclusions":"14970164,29690277",'
+            '"exclusion_period":30,"bid_ir":null,"overall_loi":900,"overall_ir":0.56,"last_block_loi":600,'
+            '"last_block_ir":0.01,"project_last_complete_date":"2024-05-28T04:12:56.297000Z","country_iso":"us",'
+            '"language_iso":"eng","include_psids":null,"exclude_psids":"01c7156fd9639737effbbdebd7fd66f6,'
+            "0508b88f4991bac8b10e9de74ce80194,0a51c627d77cef41f802e51a00126697,15b888176ac4781c2c978a9a05c396f8,"
+            "17bc146b4f7fb05c7058d25da70c6a44,29935289c1f86a4144aab2e12652f305,2fe9d1d451efca10eba4fa4e5e2b74c9,"
+            "c3527b7ef570a1571ea19870f3c25600,cdf2771d57cda9f1bf334382b2b7afd8,cebf3ec50395d973310ea526457dd5a0,"
+            "cf3877cfc15e2e6ef2a56a7a7a37f3d3,dfa691e6d060e3643d5731df30be9f69,e0cb49537182660826aa351e1187809f,"
+            'edb6d280113ca49561f25fdcb500fde6,fbfba66cfad602f1c26e61e6174eb1f7,fd4307b16fd15e8534a4551c9b6872fc",'
+            '"qualifications":["1ab337d","a01aa68","437774f","dc6065b","82b6ad6"],"quotas":[{"remaining_count":242,'
+            '"condition_hashes":["c23c0b9"]},{"remaining_count":0,"condition_hashes":["5b8c6cf"]},{"remaining_count":126,'
+            '"condition_hashes":["ac35a6e"]},{"remaining_count":110,"condition_hashes":["5e7e5aa"]},{"remaining_count":108,'
+            '"condition_hashes":["9a7aef3"]},{"remaining_count":127,"condition_hashes":["4f75127"]},{"remaining_count":0,'
+            '"condition_hashes":["95437ed"]},{"remaining_count":17,"condition_hashes":["b4b7b95"]},{"remaining_count":16,'
+            '"condition_hashes":["0ab0ae6"]},{"remaining_count":8,"condition_hashes":["6e86fb5"]},{"remaining_count":12,'
+            '"condition_hashes":["24de31e"]},{"remaining_count":69,"condition_hashes":["6bdf350"]},{"remaining_count":411,'
+            '"condition_hashes":["c94d422"]}],"conditions":null,"created_api":"2023-03-30T22:47:36.324000Z",'
+            '"modified_api":"2024-05-30T13:07:16.489000Z","updated":"2024-05-30T21:52:37.493282Z","is_live":true,'
+            '"all_hashes":["c94d422","b4b7b95","6bdf350","6e86fb5","82b6ad6","24de31e","1ab337d","c23c0b9","9a7aef3",'
+            '"ac35a6e","95437ed","5b8c6cf","437774f","a01aa68","5e7e5aa","4f75127","0ab0ae6","dc6065b"]}'
+        ),
+        (
+            '{"cpi":"1.23","country_isos":["au"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":[],"survey_id":"69420","survey_name":"Everyone is eligible AU",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261,14126487,14361592,14376811,14385771,14387789,14472374",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"au","language_iso":"eng",'
+            '"include_psids":null,"exclude_psids":null'
+            ',"qualifications":[],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":[]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+        (
+            '{"cpi":"1.23","country_isos":["us"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":[],"survey_id":"69421","survey_name":"Everyone is eligible US",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261,14126487,14361592,14376811,14385771,14387789,14472374",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"us","language_iso":"eng",'
+            '"include_psids":null,"exclude_psids":null'
+            ',"qualifications":[],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":[]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+        # For partial eligibility
+        (
+            '{"cpi":"1.23","country_isos":["us"],"language_isos":["eng"],"buyer_id":"215","bid_loi":780,"source":"s",'
+            '"used_question_ids":["1031", "212"],"survey_id":"999000","survey_name":"Pet owners",'
+            '"status":22,"field_end_date":"2023-03-02T07:05:36.261000Z","category_code":"232","calculation_type":"COMPLETES",'
+            '"requires_pii":false,"survey_exclusions":"13947261",'
+            '"exclusion_period":30,"bid_ir":0.2,"overall_loi":null,"overall_ir":null,"last_block_loi":null,'
+            '"last_block_ir":null,"project_last_complete_date":null,"country_iso":"us","language_iso":"eng",'
+            '"include_psids":null,"exclude_psids":null'
+            ',"qualifications":["0039b0c", "00f60a8"],"quotas":[{"remaining_count":100,'
+            '"condition_hashes":[]}],"conditions":null,"created_api":"2023-02-28T07:05:36.698000Z",'
+            '"modified_api":"2024-03-10T09:43:40.030000Z","updated":"2024-05-30T21:52:46.431612Z","is_live":true'
+            "}"
+        ),
+    ]
+
+
+def spectrum_conditions() -> list[SpectrumCondition]:
+    # make sure hashes for 111111 are in db
+    c1 = SpectrumCondition(
+        question_id="1001",
+        value_type=ConditionValueType.LIST,
+        values=["a", "b", "c"],
+        negate=False,
+        logical_operator=LogicalOperator.OR,
+    )
+    c2 = SpectrumCondition(
+        question_id="1001",
+        value_type=ConditionValueType.LIST,
+        values=["a"],
+        negate=False,
+        logical_operator=LogicalOperator.OR,
+    )
+    c3 = SpectrumCondition(
+        question_id="1002",
+        value_type=ConditionValueType.RANGE,
+        values=["18-24", "30-32"],
+        negate=False,
+        logical_operator=LogicalOperator.OR,
+    )
+    c4 = SpectrumCondition(
+        question_id="212",
+        value_type=ConditionValueType.LIST,
+        values=["23", "24"],
+        negate=False,
+        logical_operator=LogicalOperator.OR,
+    )
+    c5 = SpectrumCondition(
+        question_id="1031",
+        value_type=ConditionValueType.LIST,
+        values=["113", "114", "121"],
+        negate=False,
+        logical_operator=LogicalOperator.OR,
+    )
+    return [c1, c2, c3, c4, c5]
+
+
+@pytest.fixture(scope="session")
+def spectrum_api_survey_json() -> dict[str, Any]:
+    return {
+        "survey_id": 29333264,
+        "survey_name": "#29333264",
+        "survey_status": 22,
+        "field_end_date": datetime(2024, 5, 23, 18, 18, 31, tzinfo=UTC),
+        "category": "Exciting New",
+        "category_code": 232,
+        "crtd_on": datetime(2024, 5, 20, 17, 48, 13, tzinfo=UTC),
+        "mod_on": datetime(2024, 5, 20, 18, 18, 31, tzinfo=UTC),
+        "soft_launch": False,
+        "click_balancing": 0,
+        "price_type": 1,
+        "pii": False,
+        "buyer_message": "",
+        "buyer_id": 4726,
+        "incl_excl": 0,
+        "cpi": Decimal("1.20"),
+        "last_complete_date": None,
+        "project_last_complete_date": None,
+        "quotas": [
+            {
+                "quota_id": "c2bc961e-4f26-4223-b409-ebe9165cfdf5",
+                "quantities": {"currently_open": 491, "remaining": 495, "achieved": 0},
+                "criteria": [
+                    {
+                        "qualification_code": 214,
+                        "range_sets": [{"units": 311, "to": 64, "from": 18}],
+                    }
+                ],
+            }
+        ],
+        "qualifications": [
+            {
+                "range_sets": [{"units": 311, "to": 64, "from": 18}],
+                "qualification_code": 212,
+            },
+            {"condition_codes": ["111", "117", "112"], "qualification_code": 1202},
+        ],
+        "country_iso": "fr",
+        "language_iso": "fre",
+        "bid_ir": 0.4,
+        "bid_loi": 600,
+        "overall_ir": None,
+        "overall_loi": None,
+        "last_block_ir": None,
+        "last_block_loi": None,
+        "survey_exclusions": set(),
+        "exclusion_period": 0,
+    }