add a get_user_profiling_data method to get a user's full profiling data, including upk and mrpqs, annotated with enough data to be usefull to display in ui

4 files changed, 186 insertions, 9 deletions

diff --git a/generalresearch/managers/thl/profiling/question.py b/generalresearch/managers/thl/profiling/question.py
index 078894a..7fdd1c2 100644
--- a/generalresearch/managers/thl/profiling/question.py
+++ b/generalresearch/managers/thl/profiling/question.py
@@ -17,7 +17,6 @@ from generalresearch.models.thl.profiling.upk_question import (
 
 
 class QuestionManager(PostgresManager):
-
     def get_multi_upk(self, question_ids: Collection[str]) -> list[UpkQuestion]:
         query = """
         SELECT data, property_code, explanation_template, explanation_fragment_template
@@ -78,7 +77,7 @@ class QuestionManager(PostgresManager):
         return res
 
     @cached(
-        cache=TTLCache(maxsize=256, ttl=3600 + random.randint(-900, 900)),
+        cache=TTLCache(maxsize=1024, ttl=3600 + random.randint(-900, 900)),
         lock=threading.Lock(),
         info=True,
     )
@@ -110,6 +109,39 @@ class QuestionManager(PostgresManager):
         x["data"].pop("categories", None)
         return UpkQuestion.model_validate(x["data"])
 
+    def filter_by_properties_and_locale(
+        self, property_codes: Collection[str], country_iso: str, language_iso: str
+    ) -> list[UpkQuestion]:
+        query = """
+        SELECT data, property_code, explanation_template, explanation_fragment_template
+        FROM marketplace_question
+        WHERE property_code = ANY(%(property_codes)s)
+            AND country_iso = %(country_iso)s
+            AND language_iso = %(language_iso)s
+        """
+        res: list[dict[str, Any]] = self.pg_config.execute_sql_query(
+            query=query,
+            params={
+                "property_codes": list(property_codes),
+                "country_iso": country_iso,
+                "language_iso": language_iso,
+            },
+        )
+        for x in res:
+            x["data"]["ext_question_id"] = x["property_code"]
+            x["data"]["explanation_template"] = x["explanation_template"]
+            x["data"]["explanation_fragment_template"] = x[
+                "explanation_fragment_template"
+            ]
+            x["data"].pop("categories", None)
+        res2 = []
+        for x in res:
+            try:
+                res2.append(UpkQuestion.model_validate(x["data"]))
+            except ValidationError as e:
+                LOG.warning(e)
+        return res2
+
     def filter_by_property(
         self, lookup: Collection[tuple[str, str, str]]
     ) -> list[UpkQuestion]:
diff --git a/generalresearch/managers/thl/profiling/user_profiling_data.py b/generalresearch/managers/thl/profiling/user_profiling_data.py
new file mode 100644
index 0000000..1d29328
--- /dev/null
+++ b/generalresearch/managers/thl/profiling/user_profiling_data.py
@@ -0,0 +1,102 @@
+from typing import Any
+
+from generalresearch.managers.thl.profiling.question import QuestionManager
+from generalresearch.managers.thl.profiling.uqa import UQAManager
+from generalresearch.managers.thl.profiling.user_upk import UserUpkManager
+from generalresearch.managers.thl.userhealth import UserIpHistoryManager
+from generalresearch.models.thl.profiling.upk_property import PropertyType
+from generalresearch.models.thl.profiling.user_info import (
+    MarketProfileKnowledge,
+    UserProfileKnowledge,
+    UserProfilingData,
+)
+from generalresearch.models.thl.profiling.user_question_answer import (
+    MarketplaceResearchProfileQuestion,
+)
+from generalresearch.models.thl.user import User
+from generalresearch.models.thl.user_ref import UserRef
+
+
+def get_user_mrpq(
+    user: User,
+    uqa_manager: UQAManager,
+    question_manager: QuestionManager,
+    country_iso: str | None = None,
+) -> list[MarketplaceResearchProfileQuestion]:
+    res = uqa_manager.get(user=user)
+    if country_iso:
+        res = [x for x in res if x.country_iso == country_iso]
+    mrpqs = []
+    for x in res:
+        mrpqs.extend(x.get_mrpqs(question_manager=question_manager))
+    return mrpqs
+
+
+def get_user_upk_formatted(
+    user_id: int, user_upk_manager: UserUpkManager, country_iso: str | None = None
+):
+    res = user_upk_manager.get_user_upk(user_id=user_id)
+    if country_iso:
+        res = [x for x in res if x.country_iso == country_iso]
+    new_res = []
+    for x in res:
+        d: dict[str, Any] = {
+            "property_id": x.property_id,
+            "property_label": x.property_label,
+            "created": x.created,
+            "translation": "",
+        }
+        if x.prop_type in {PropertyType.UPK_NUMERICAL, PropertyType.UPK_TEXT}:
+            d["answer"] = [{"value": str(x.value)}]
+        else:
+            d["answer"] = [{"id": x.item_id, "label": x.item_label, "translation": ""}]
+        new_res.append(d)
+    d = {}
+    for x in new_res:
+        if x["property_id"] in d:
+            d[x["property_id"]]["answer"].extend(x["answer"])
+        else:
+            d[x["property_id"]] = x
+    return list(d.values())
+
+
+def get_user_profiling_data(
+    user_upk_manager: UserUpkManager,
+    uqa_manager: UQAManager,
+    question_manager: QuestionManager,
+    user: User,
+    country_iso: str | None = None,
+    user_ip_history_manager: UserIpHistoryManager | None = None,
+) -> UserProfilingData:
+    """
+    Get user's mrpq and upk data, for API response
+    """
+    assert user.user_id is not None
+    user = user if isinstance(user, UserRef) else user.to_user_ref()
+
+    if country_iso is None and user_ip_history_manager is not None:
+        country_iso = user_ip_history_manager.get_user_latest_country(user)
+
+    mrpq = get_user_mrpq(
+        user=user,
+        uqa_manager=uqa_manager,
+        question_manager=question_manager,
+        country_iso=country_iso,
+    )
+    mrpq = [
+        MarketProfileKnowledge.from_MarketplaceResearchProfileQuestion(x) for x in mrpq
+    ]
+
+    upk = get_user_upk_formatted(
+        user_id=user.user_id,
+        user_upk_manager=user_upk_manager,
+        country_iso=country_iso,
+    )
+    upk = [UserProfileKnowledge.model_validate(x) for x in upk]
+
+    d = {
+        "user_profile_knowledge": upk,
+        "marketplace_profile_knowledge": mrpq,
+        "user": user,
+    }
+    return UserProfilingData.model_validate(d)
diff --git a/generalresearch/models/thl/profiling/user_info.py b/generalresearch/models/thl/profiling/user_info.py
index 46733d4..34c957b 100644
--- a/generalresearch/models/thl/profiling/user_info.py
+++ b/generalresearch/models/thl/profiling/user_info.py
@@ -1,14 +1,13 @@
 from __future__ import annotations
 
 from pydantic import BaseModel, ConfigDict, Field
-from pydantic.json_schema import SkipJsonSchema
 
 from generalresearch.models.custom_types import AwareDatetimeISO
 from generalresearch.models.definitions import Source
 from generalresearch.models.thl.profiling.user_question_answer import (
     MarketplaceResearchProfileQuestion,
 )
-from generalresearch.models.thl.user import User
+from generalresearch.models.thl.user_ref import UserRef
 
 
 class UserProfileKnowledgeAnswer(BaseModel):
@@ -43,14 +42,20 @@ class MarketProfileKnowledge(BaseModel):
     )
 
     question_id: str = Field(examples=["gender", "1843", "gender_plus"])
+    question_text: str | None = Field(default=None)
 
     answer: list[str] = Field(
         default_factory=list, examples=[["male"], ["7657644"], ["1"]]
     )
+    answer_text: list[str] | None = Field(default=None)
 
     created: AwareDatetimeISO = Field(
         description="When the User submitted this Profiling data"
     )
+    is_derived: bool = Field(
+        default=False,
+        description='What this answer derived from a "harmonized" / "gold-standard" question?',
+    )
 
     @classmethod
     def from_MarketplaceResearchProfileQuestion(
@@ -61,16 +66,24 @@ class MarketProfileKnowledge(BaseModel):
             question_id=q.question_code,
             answer=list(q.answer),
             created=q.timestamp,
+            question_text=q.question_text,
+            answer_text=list(q.answer_text) if q.answer_text is not None else None,
+            is_derived=q.is_derived,
         )
 
 
-class UserInfo(BaseModel):
+class UserProfilingData(BaseModel):
     model_config = ConfigDict()
 
-    user: SkipJsonSchema[User | None] = Field(exclude=True, default=None)
+    user: UserRef = Field()
 
     user_profile_knowledge: list[UserProfileKnowledge] = Field(default_factory=list)
 
     marketplace_profile_knowledge: list[MarketProfileKnowledge] = Field(
         default_factory=list
     )
+
+
+class UserInfo(UserProfilingData):
+    # fsb endpoint uses this name. todo: rename it
+    pass
diff --git a/generalresearch/models/thl/profiling/user_question_answer.py b/generalresearch/models/thl/profiling/user_question_answer.py
index 342b65e..bef844b 100644
--- a/generalresearch/models/thl/profiling/user_question_answer.py
+++ b/generalresearch/models/thl/profiling/user_question_answer.py
@@ -14,6 +14,7 @@ from pydantic import (
     model_validator,
 )
 
+from generalresearch.managers.thl.profiling.question import QuestionManager
 from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr
 from generalresearch.models.definitions import MAX_INT32, Source
 from generalresearch.models.thl.locales import CountryISO, LanguageISO
@@ -57,6 +58,10 @@ class UserQuestionAnswer(BaseModel):
         )
         return calc_answers
 
+    @property
+    def is_gold_standard(self) -> bool:
+        return self.property_code.startswith("gr:")
+
     def model_dump_mysql(self, session_id: str | None = None) -> dict[str, Any]:
         d = self.model_dump(mode="json", exclude={"calc_answers", "timestamp"})
         d["answer"] = json.dumps(self.answer)
@@ -67,17 +72,36 @@ class UserQuestionAnswer(BaseModel):
         d["session_id"] = session_id
         return d
 
-    def get_mrpqs(self) -> Iterator[MarketplaceResearchProfileQuestion]:
+    def get_mrpqs(
+        self, question_manager: QuestionManager
+    ) -> Iterator[MarketplaceResearchProfileQuestion]:
         assert self.calc_answers
+        property_codes = set(self.calc_answers.keys())
+        qs = question_manager.filter_by_properties_and_locale(
+            property_codes, country_iso=self.country_iso, language_iso=self.language_iso
+        )
+        text_lookup = {q.ext_question_id: q.text for q in qs}
+        ans_lookup = {
+            q.ext_question_id: {c.id: c.text for c in q.choices}
+            for q in qs
+            if q.choices is not None
+        }
+
         for k, v in self.calc_answers.items():
             source, question_code = k.split(":", 1)
+            ans = ans_lookup.get(k, {})
+            ans_text = tuple(sorted([ans.get(vv, "") for vv in v]))
+            ans_text = ans_text or None
             yield MarketplaceResearchProfileQuestion(
                 question_code=question_code,
-                source=source,
+                question_text=text_lookup.get(k),
+                source=Source(source),
                 country_iso=self.country_iso,
                 language_iso=self.language_iso,
                 answer=tuple(sorted(set(v))),
+                answer_text=ans_text,
                 timestamp=self.timestamp,
+                is_derived=self.is_gold_standard
             )
 
     @field_validator("answer")
@@ -124,17 +148,23 @@ DUMMY_UQA = UserQuestionAnswer(
 
 
 class MarketplaceResearchProfileQuestion(BaseModel):
-    """Answer submitted to a question by a user, that has been transformed
+    """Answer submitted to a question by a user that has been transformed
     into a question answer that is specific to a marketplace."""
 
     question_code: str = Field(
         description="# the question id/code on the marketplace", min_length=1
     )
+    question_text: str | None = Field(default=None)
     source: Source = Field()  # the one or two-letter marketplace code
     answer: tuple[str, ...] = Field(min_length=1)
+    answer_text: tuple[str, ...] | None = Field(min_length=1, default=None)
     timestamp: AwareDatetimeISO = Field()
     country_iso: CountryISO = Field()
     language_iso: LanguageISO = Field()
+    is_derived: bool = Field(
+        default=False,
+        description='What this answer derived from a "harmonized" / "gold-standard" question?',
+    )
 
     @model_validator(mode="after")
     def validate_keys(self):