moving create_dummy, organizing test_utils, working on django config for any repos

9 files changed, 42 insertions, 146 deletions

diff --git a/test_utils/managers/upk/conftest.py b/test_utils/managers/upk/conftest.py
index e28d085..d8f956c 100644
--- a/test_utils/managers/upk/conftest.py
+++ b/test_utils/managers/upk/conftest.py
@@ -1,173 +1,69 @@
-import os
-import time
-from typing import TYPE_CHECKING, Optional
-from uuid import UUID
+from typing import Callable, Generator
 
-import pandas as pd
 import pytest
 
+from generalresearch.managers.thl.profiling.question import (
+    QuestionManager,
+)
+from generalresearch.managers.thl.profiling.schema import (
+    UpkSchemaManager,
+)
+from generalresearch.managers.thl.profiling.uqa import UQAManager
+from generalresearch.managers.thl.profiling.user_upk import (
+    UserUpkManager,
+)
+from generalresearch.models.thl.user import User
 from generalresearch.pg_helper import PostgresConfig
-
-if TYPE_CHECKING:
-    from generalresearch.managers.thl.category import CategoryManager
-
-
-def insert_data_from_csv(
-    thl_web_rw: PostgresConfig,
-    table_name: str,
-    fp: Optional[str] = None,
-    disable_fk_checks: bool = False,
-    df: Optional[pd.DataFrame] = None,
-):
-    assert fp is not None or df is not None and not (fp is not None and df is not None)
-    if fp:
-        df = pd.read_csv(fp, dtype=str)
-    df = df.where(pd.notnull(df), None)
-    cols = list(df.columns)
-    col_str = ", ".join(cols)
-    values_str = ", ".join(["%s"] * len(cols))
-    if "id" in df.columns and len(df["id"].iloc[0]) == 36:
-        df["id"] = df["id"].map(lambda x: UUID(x).hex)
-    args = df.to_dict("tight")["data"]
-
-    with thl_web_rw.make_connection() as conn:
-        with conn.cursor() as c:
-            if disable_fk_checks:
-                c.execute("SET CONSTRAINTS ALL DEFERRED")
-            c.executemany(
-                f"INSERT INTO {table_name} ({col_str}) VALUES ({values_str})",
-                params_seq=args,
-            )
-        conn.commit()
+from generalresearch.redis_helper import RedisConfig
 
 
 @pytest.fixture(scope="session")
-def category_data(
-    thl_web_rw: PostgresConfig, category_manager: "CategoryManager"
-) -> None:
-    fp = os.path.join(os.path.dirname(__file__), "marketplace_category.csv.gz")
-    insert_data_from_csv(
-        thl_web_rw,
-        fp=fp,
-        table_name="marketplace_category",
-        disable_fk_checks=True,
-    )
-    # Don't strictly need to do this, but probably we should
-    category_manager.populate_caches()
-    cats = category_manager.categories.values()
-    path_id = {c.path: c.id for c in cats}
-    data = [
-        {"id": c.id, "parent_id": path_id[c.parent_path]} for c in cats if c.parent_path
-    ]
-    query = """
-    UPDATE marketplace_category
-    SET parent_id = %(parent_id)s
-    WHERE id = %(id)s;
-    """
-    with thl_web_rw.make_connection() as conn:
-        with conn.cursor() as c:
-            c.executemany(query=query, params_seq=data)
-        conn.commit()
+def upk_schema_manager(thl_web_rw: PostgresConfig) -> UpkSchemaManager:
+    return UpkSchemaManager(pg_config=thl_web_rw)
 
 
 @pytest.fixture(scope="session")
-def property_data(thl_web_rw: PostgresConfig) -> None:
-    fp = os.path.join(os.path.dirname(__file__), "marketplace_property.csv.gz")
-    insert_data_from_csv(thl_web_rw, fp=fp, table_name="marketplace_property")
-
+def user_upk_manager(
+    thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig
+) -> UserUpkManager:
 
-@pytest.fixture(scope="session")
-def item_data(thl_web_rw: PostgresConfig) -> None:
-    fp = os.path.join(os.path.dirname(__file__), "marketplace_item.csv.gz")
-    insert_data_from_csv(thl_web_rw, fp=fp, table_name="marketplace_item")
+    return UserUpkManager(pg_config=thl_web_rw, redis_config=thl_redis_config)
 
 
 @pytest.fixture(scope="session")
-def propertycategoryassociation_data(
+def question_manager(
     thl_web_rw: PostgresConfig,
-    category_data,
-    property_data,
-    category_manager: "CategoryManager",
-) -> None:
-    table_name = "marketplace_propertycategoryassociation"
-    fp = os.path.join(os.path.dirname(__file__), f"{table_name}.csv.gz")
-    # Need to lookup category pk from uuid
-    category_manager.populate_caches()
-    df = pd.read_csv(fp, dtype=str)
-    df["category_id"] = df["category_id"].map(
-        lambda x: category_manager.categories[x].id
-    )
-    insert_data_from_csv(thl_web_rw, df=df, table_name=table_name)
+) -> QuestionManager:
+    return QuestionManager(pg_config=thl_web_rw)
 
 
 @pytest.fixture(scope="session")
-def propertycountry_data(thl_web_rw: PostgresConfig, property_data) -> None:
-    fp = os.path.join(os.path.dirname(__file__), "marketplace_propertycountry.csv.gz")
-    insert_data_from_csv(thl_web_rw, fp=fp, table_name="marketplace_propertycountry")
+def uqa_manager(
+    thl_web_rw: PostgresConfig, thl_redis_config: RedisConfig
+) -> UQAManager:
 
+    return UQAManager(redis_config=thl_redis_config, pg_config=thl_web_rw)
 
-@pytest.fixture(scope="session")
-def propertymarketplaceassociation_data(
-    thl_web_rw: PostgresConfig, property_data
-) -> None:
-    table_name = "marketplace_propertymarketplaceassociation"
-    fp = os.path.join(os.path.dirname(__file__), f"{table_name}.csv.gz")
-    insert_data_from_csv(thl_web_rw, fp=fp, table_name=table_name)
 
+@pytest.fixture(scope="function")
+def uqa_manager_clear_cache_factory(
+    uqa_manager: UQAManager,
+) -> Callable[..., Generator[None]]:
 
-@pytest.fixture(scope="session")
-def propertyitemrange_data(
-    thl_web_rw: PostgresConfig, property_data, item_data
-) -> None:
-    table_name = "marketplace_propertyitemrange"
-    fp = os.path.join(os.path.dirname(__file__), f"{table_name}.csv.gz")
-    insert_data_from_csv(thl_web_rw, fp=fp, table_name=table_name)
+    def _inner(user: User) -> Generator[None]:
+        # On successive py-test/jenkins runs, the cache may contain
+        #   the previous run's info (keyed under the same user_id)
+        uqa_manager.clear_cache(user)
 
+        yield
 
-@pytest.fixture(scope="session")
-def question_data(thl_web_rw: PostgresConfig) -> None:
-    table_name = "marketplace_question"
-    fp = os.path.join(os.path.dirname(__file__), f"{table_name}.csv.gz")
-    insert_data_from_csv(
-        thl_web_rw, fp=fp, table_name=table_name, disable_fk_checks=True
-    )
+        uqa_manager.clear_cache(user)
 
+    return _inner
 
-@pytest.fixture(scope="session")
-def clear_upk_tables(thl_web_rw: PostgresConfig):
-    tables = [
-        "marketplace_propertyitemrange",
-        "marketplace_propertymarketplaceassociation",
-        "marketplace_propertycategoryassociation",
-        "marketplace_category",
-        "marketplace_item",
-        "marketplace_property",
-        "marketplace_propertycountry",
-        "marketplace_question",
-    ]
-    table_str = ", ".join(tables)
-
-    with thl_web_rw.make_connection() as conn:
-        with conn.cursor() as c:
-            c.execute(f"TRUNCATE {table_str} RESTART IDENTITY CASCADE;")
-        conn.commit()
 
-
-@pytest.fixture(scope="session")
-def upk_data(
-    clear_upk_tables,
-    category_data,
-    property_data,
-    item_data,
-    propertycategoryassociation_data,
-    propertycountry_data,
-    propertymarketplaceassociation_data,
-    propertyitemrange_data,
-    question_data,
-) -> None:
-    # Wait a second to make sure the HarmonizerCache refresh loop pulls these in
-    time.sleep(2)
-
-
-def test_fixtures(upk_data):
-    pass
+@pytest.fixture(scope="function")
+def uqa_manager_clear_cache(
+    uqa_manager_clear_cache_factory: Callable[..., None], user: User
+):
+    uqa_manager_clear_cache_factory(user=user)
diff --git a/test_utils/managers/upk/marketplace_category.csv.gz b/test_utils/managers/upk/marketplace_category.csv.gz
deleted file mode 100644
index 0f8ec1c..0000000
--- a/test_utils/managers/upk/marketplace_category.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_item.csv.gz b/test_utils/managers/upk/marketplace_item.csv.gz
deleted file mode 100644
index c12c5d8..0000000
--- a/test_utils/managers/upk/marketplace_item.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_property.csv.gz b/test_utils/managers/upk/marketplace_property.csv.gz
deleted file mode 100644
index a781d1d..0000000
--- a/test_utils/managers/upk/marketplace_property.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_propertycategoryassociation.csv.gz b/test_utils/managers/upk/marketplace_propertycategoryassociation.csv.gz
deleted file mode 100644
index 5b4ea19..0000000
--- a/test_utils/managers/upk/marketplace_propertycategoryassociation.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_propertycountry.csv.gz b/test_utils/managers/upk/marketplace_propertycountry.csv.gz
deleted file mode 100644
index 5d2a637..0000000
--- a/test_utils/managers/upk/marketplace_propertycountry.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_propertyitemrange.csv.gz b/test_utils/managers/upk/marketplace_propertyitemrange.csv.gz
deleted file mode 100644
index 84f4f0e..0000000
--- a/test_utils/managers/upk/marketplace_propertyitemrange.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_propertymarketplaceassociation.csv.gz b/test_utils/managers/upk/marketplace_propertymarketplaceassociation.csv.gz
deleted file mode 100644
index 6b9fd1c..0000000
--- a/test_utils/managers/upk/marketplace_propertymarketplaceassociation.csv.gz
+++ /dev/null
diff --git a/test_utils/managers/upk/marketplace_question.csv.gz b/test_utils/managers/upk/marketplace_question.csv.gz
deleted file mode 100644
index bcfc3ad..0000000
--- a/test_utils/managers/upk/marketplace_question.csv.gz
+++ /dev/null