From 352520a2bd4ac066e90587e44d8b818043407917 Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 21 Aug 2026 16:19:25 -0600 Subject: Use a BusinessPayoutEventCreate model so we dont have all of these optional int warnings. cleaner --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) (limited to 'pyproject.toml') diff --git a/pyproject.toml b/pyproject.toml index 183e271..dbdf3b9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,3 +54,6 @@ include = ["generalresearch", "generalresearch.*", "test_utils", "test_utils.*"] [tool.pytest.ini_options] testpaths = ["tests"] addopts = "-v --tb=short" + +[tool.ruff] +target-version = "py314" \ No newline at end of file -- cgit v1.2.3 From aeeb7fef2594ccd34fbe96a77f6c5b392299fed7 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Thu, 27 Aug 2026 17:46:11 -0700 Subject: Ruff afternoon --- generalresearch/managers/thl/product.py | 1 - generalresearch/models/network/nmap/parser.py | 8 +- generalresearch/models/precision/question.py | 4 +- generalresearch/models/precision/survey.py | 17 +- generalresearch/models/prodege/survey.py | 3 +- generalresearch/models/spectrum/question.py | 9 +- generalresearch/models/spectrum/survey.py | 13 +- generalresearch/models/spectrum/task_collection.py | 2 +- generalresearch/models/thl/contest/contest.py | 10 +- generalresearch/models/thl/contest/leaderboard.py | 13 +- generalresearch/models/thl/contest/milestone.py | 20 +- generalresearch/models/thl/contest/raffle.py | 46 ++-- generalresearch/models/thl/finance.py | 12 +- generalresearch/models/thl/product.py | 36 +-- .../models/thl/profiling/other_option.py | 4 +- .../models/thl/profiling/upk_question.py | 13 +- .../models/thl/profiling/user_question_answer.py | 31 +-- pyproject.toml | 5 +- .../collections/test_df_collection_item_thl_web.py | 273 ++++++++++----------- .../mergers/foundations/test_enriched_session.py | 52 ++-- .../foundations/test_enriched_task_adjust.py | 38 ++- .../mergers/foundations/test_enriched_wall.py | 73 +++--- tests/incite/mergers/test_merge_collection.py | 53 +++- tests/incite/mergers/test_merge_collection_item.py | 25 +- tests/incite/mergers/test_pop_ledger.py | 109 ++++---- tests/incite/mergers/test_ym_survey_merge.py | 55 +++-- tests/incite/schemas/test_admin_responses.py | 36 +-- tests/incite/schemas/test_thl_web.py | 8 +- tests/incite/test_collection_base.py | 62 ++--- tests/incite/test_collection_base_item.py | 74 +++--- tests/incite/test_interval_idx.py | 6 +- tests/managers/gr/test_business.py | 60 +++-- tests/managers/gr/test_team.py | 58 +++-- tests/managers/network/test_label.py | 4 +- .../managers/thl/test_contest/test_leaderboard.py | 5 +- tests/managers/thl/test_contest/test_milestone.py | 5 +- tests/managers/thl/test_contest/test_raffle.py | 12 +- tests/managers/thl/test_ledger/test_lm_accounts.py | 2 +- tests/managers/thl/test_ledger/test_lm_tx.py | 4 - .../thl/test_ledger/test_thl_lm_accounts.py | 11 +- .../thl/test_ledger/test_thl_lm_bp_payout.py | 54 ++-- tests/managers/thl/test_ledger/test_thl_lm_tx.py | 26 +- tests/managers/thl/test_payout.py | 2 - tests/managers/thl/test_survey.py | 6 +- tests/managers/thl/test_user_manager/test_base.py | 2 +- tests/models/network/test_nmap.py | 2 +- tests/models/spectrum/test_survey.py | 44 ++++ tests/models/thl/test_product.py | 4 +- 48 files changed, 815 insertions(+), 597 deletions(-) (limited to 'pyproject.toml') diff --git a/generalresearch/managers/thl/product.py b/generalresearch/managers/thl/product.py index d924e17..54fa7c8 100644 --- a/generalresearch/managers/thl/product.py +++ b/generalresearch/managers/thl/product.py @@ -33,7 +33,6 @@ if TYPE_CHECKING: ProfilingConfig, SessionConfig, SourcesConfig, - SupplyConfigs, UserCreateConfig, UserHealthConfig, UserWalletConfig, diff --git a/generalresearch/models/network/nmap/parser.py b/generalresearch/models/network/nmap/parser.py index ecaf2d1..866b4bd 100644 --- a/generalresearch/models/network/nmap/parser.py +++ b/generalresearch/models/network/nmap/parser.py @@ -48,7 +48,7 @@ class NmapXmlParser: try: root = ET.fromstring(nmap_data) - except Exception as e: + except ET.ParseError as e: emsg = f"Wrong XML structure: cannot parse data: {e}" raise NmapParserException(emsg) @@ -103,7 +103,7 @@ class NmapXmlParser: @classmethod def _parse_scaninfo(cls, scaninfo_el: ET.Element) -> NmapScanInfo: - data = dict() + data = {} data["type"] = NmapScanType(scaninfo_el.attrib["type"]) data["protocol"] = IPProtocol(scaninfo_el.attrib["protocol"]) data["num_services"] = scaninfo_el.attrib["numservices"] @@ -132,7 +132,7 @@ class NmapXmlParser: @classmethod def _parse_nmaprun(cls, nmaprun_el: ET.Element) -> dict: - nmap_data = dict() + nmap_data = {} nmaprun = dict(nmaprun_el.attrib) nmap_data["command_line"] = nmaprun["args"] nmap_data["started_at"] = datetime.fromtimestamp( @@ -148,7 +148,7 @@ class NmapXmlParser: Receives a XML tag representing a scanned host with its services. """ - data = dict() + data = {} # status_el = host_el.find("status") diff --git a/generalresearch/models/precision/question.py b/generalresearch/models/precision/question.py index a2189d5..cc90aa9 100644 --- a/generalresearch/models/precision/question.py +++ b/generalresearch/models/precision/question.py @@ -6,7 +6,7 @@ import logging from enum import StrEnum from typing import TYPE_CHECKING, Any, Literal -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, ValidationError, field_validator, model_validator from generalresearch.models import Source, string_utils from generalresearch.models.precision import PrecisionQuestionID @@ -112,7 +112,7 @@ class PrecisionQuestion(MarketplaceQuestion): """ try: return cls._from_api(d) - except Exception as e: + except ValidationError as e: logger.warning(f"Unable to parse question: {d}. {e}") return None diff --git a/generalresearch/models/precision/survey.py b/generalresearch/models/precision/survey.py index f515552..b27b8c4 100644 --- a/generalresearch/models/precision/survey.py +++ b/generalresearch/models/precision/survey.py @@ -99,11 +99,11 @@ class PrecisionQuota(BaseModel): self, criteria_evaluation: dict[str, bool | None] ) -> tuple[bool | None, list[str]]: # Passes back "matches" (T/F/none) and a list of unknown criterion hashes - unknowns = list() + unknowns = [] for c in self.condition_hashes: eval_value = criteria_evaluation.get(c) if eval_value is False: - return False, list() + return False, [] if eval_value is None: unknowns.append(c) if unknowns: @@ -245,11 +245,10 @@ class PrecisionSurvey(MarketplaceTask): # Fancy repr that abbreviates exclude_pids and excluded_surveys repr_args = list(self.__repr_args__()) for n, (k, v) in enumerate(repr_args): - if k in {"excluded_surveys"}: - if v and len(v) > 6: - v = sorted(v) - v = v[:3] + ["…"] + v[-3:] - repr_args[n] = (k, v) + if k in {"excluded_surveys"} and v and len(v) > 6: + v = sorted(v) + v = v[:3] + ["…"] + v[-3:] + repr_args[n] = (k, v) join_str = ", " repr_str = join_str.join( repr(v) if a is None else f"{a}={v!r}" for a, v in repr_args @@ -369,6 +368,4 @@ class PrecisionSurvey(MarketplaceTask): return False if self.group_id in att_group_ids: return False - if self.excluded_surveys & att_survey_ids: - return False - return True + return not self.excluded_surveys & att_survey_ids diff --git a/generalresearch/models/prodege/survey.py b/generalresearch/models/prodege/survey.py index 3f4c88f..7ab6df6 100644 --- a/generalresearch/models/prodege/survey.py +++ b/generalresearch/models/prodege/survey.py @@ -13,6 +13,7 @@ from pydantic import ( BaseModel, ConfigDict, Field, + ValidationError, computed_field, field_validator, model_validator, @@ -513,7 +514,7 @@ class ProdegeSurvey(MarketplaceTask): def from_api(cls, d: dict[str, Any]) -> ProdegeSurvey | None: try: return cls._from_api(d) - except Exception as e: + except ValidationError as e: logger.warning(f"Unable to parse survey: {d}. {e}") return None diff --git a/generalresearch/models/spectrum/question.py b/generalresearch/models/spectrum/question.py index c8eea4a..7add692 100644 --- a/generalresearch/models/spectrum/question.py +++ b/generalresearch/models/spectrum/question.py @@ -13,6 +13,7 @@ from pydantic import ( BaseModel, Field, PositiveInt, + ValidationError, field_validator, model_validator, ) @@ -132,7 +133,7 @@ class SpectrumQuestionType(StrEnum): @classmethod def from_api(cls, a: int): api_type_map = cls.get_api_map() - return api_type_map[a] if a in api_type_map else None + return api_type_map.get(a, None) class SpectrumQuestionClass(IntEnum): @@ -260,7 +261,7 @@ class SpectrumQuestion(MarketplaceQuestion): return None try: return cls._from_api(d, country_iso, language_iso) - except Exception as e: + except ValidationError as e: logger.warning(f"Unable to parse question: {d}. {e}") return None @@ -280,7 +281,9 @@ class SpectrumQuestion(MarketplaceQuestion): ] created = ( - datetime.utcfromtimestamp(d["crtd_on"] / 1000).replace(tzinfo=UTC) + datetime.fromtimestamp(timestamp=d["crtd_on"] / 1000, tz=UTC).replace( + tzinfo=UTC + ) if d.get("crtd_on") else None ) diff --git a/generalresearch/models/spectrum/survey.py b/generalresearch/models/spectrum/survey.py index f9a5e27..424d206 100644 --- a/generalresearch/models/spectrum/survey.py +++ b/generalresearch/models/spectrum/survey.py @@ -75,8 +75,7 @@ class SpectrumCondition(MarketplaceCondition): rs["from"] = round(rs["from"] / 12) rs["to"] = round(rs["to"] / 12) d["values"] = [ - f"{rs["from"] or "inf"}-{rs["to"] or "inf"}" - for rs in d["range_sets"] + f"{rs["from"] or "inf"}-{rs["to"] or "inf"}" for rs in d["range_sets"] ] d["value_type"] = ConditionValueType.RANGE return cls.model_validate(d) @@ -103,7 +102,7 @@ class SpectrumQuota(BaseModel): # There is no explicit status. The quota is closed if the count is 0 def __hash__(self) -> int: - return hash(tuple((tuple(self.condition_hashes), self.remaining_count))) + return hash((tuple(self.condition_hashes), self.remaining_count)) @property def is_open(self) -> bool: @@ -113,7 +112,7 @@ class SpectrumQuota(BaseModel): return self.remaining_count >= min_open_spots @classmethod - def from_api(cls, d: dict) -> Self: + def from_api(cls, d: dict[str, Any]) -> Self: d["remaining_count"] = d["quantities"]["currently_open"] return cls.model_validate(d) @@ -323,7 +322,7 @@ class SpectrumSurvey(MarketplaceTask): def from_api(cls, d: dict[str, Any]) -> SpectrumSurvey | None: try: return cls._from_api(d) - except Exception as e: + except (AssertionError, ValueError) as e: logger.warning(f"Unable to parse survey: {d}. {e}") return None @@ -336,7 +335,7 @@ class SpectrumSurvey(MarketplaceTask): else TaskCalculationType.COMPLETES ) - d["conditions"] = dict() + d["conditions"] = {} # If we haven't hit the "detail" endpoint, we won't get this d.setdefault("qualifications", []) @@ -454,7 +453,7 @@ class SpectrumSurvey(MarketplaceTask): quota_eval = { quota: quota.matches_soft(criteria_evaluation) for quota in self.quotas } - evals = set(g[0] for g in quota_eval.values()) + evals = {g[0] for g in quota_eval.values()} if any(m[0] is True and not q.is_open for q, m in quota_eval.items()): # matched a full quota return False, set() diff --git a/generalresearch/models/spectrum/task_collection.py b/generalresearch/models/spectrum/task_collection.py index 6715378..8ca5a93 100644 --- a/generalresearch/models/spectrum/task_collection.py +++ b/generalresearch/models/spectrum/task_collection.py @@ -91,7 +91,7 @@ class SpectrumTaskCollection(TaskCollection): "survey_id", ] rows = [] - d = dict() + d = {} for k in fields: d[k] = getattr(s, k) if hasattr(s, k) else None d["used_question_ids"] = list(s.used_question_ids) diff --git a/generalresearch/models/thl/contest/contest.py b/generalresearch/models/thl/contest/contest.py index 2a8853d..bd0fc04 100644 --- a/generalresearch/models/thl/contest/contest.py +++ b/generalresearch/models/thl/contest/contest.py @@ -136,10 +136,12 @@ class Contest(ContestBase): # return False def should_end(self) -> tuple[bool, ContestEndReason | None]: - if self.status == ContestStatus.ACTIVE: - if self.end_condition.ends_at: - if datetime.now(tz=UTC) >= self.end_condition.ends_at: - return True, ContestEndReason.ENDS_AT + if ( + self.status == ContestStatus.ACTIVE + and self.end_condition.ends_at + and datetime.now(tz=UTC) >= self.end_condition.ends_at + ): + return True, ContestEndReason.ENDS_AT return False, None diff --git a/generalresearch/models/thl/contest/leaderboard.py b/generalresearch/models/thl/contest/leaderboard.py index 696cdea..e923383 100644 --- a/generalresearch/models/thl/contest/leaderboard.py +++ b/generalresearch/models/thl/contest/leaderboard.py @@ -151,7 +151,8 @@ class LeaderboardContest(LeaderboardContestCreate, Contest): len(self.country_isos) == 1 ), "Can only set 1 country_iso in a leaderboard contest" assert ( - list(self.country_isos)[0] == self.leaderboard_key_parts["country_iso"] + next(iter(self.country_isos)) + == self.leaderboard_key_parts["country_iso"] ), "leaderboard_key country_iso must match the country_isos" else: self.country_isos = {self.leaderboard_key_parts["country_iso"]} @@ -192,10 +193,12 @@ class LeaderboardContest(LeaderboardContestCreate, Contest): return lbm def should_end(self) -> tuple[bool, ContestEndReason | None]: - if self.status == ContestStatus.ACTIVE: - if self.end_condition.ends_at: - if datetime.now(tz=UTC) >= self.end_condition.ends_at: - return True, ContestEndReason.ENDS_AT + if ( + self.status == ContestStatus.ACTIVE + and self.end_condition.ends_at + and datetime.now(tz=UTC) >= self.end_condition.ends_at + ): + return True, ContestEndReason.ENDS_AT return False, None diff --git a/generalresearch/models/thl/contest/milestone.py b/generalresearch/models/thl/contest/milestone.py index 8d96fcb..5fc27fa 100644 --- a/generalresearch/models/thl/contest/milestone.py +++ b/generalresearch/models/thl/contest/milestone.py @@ -132,10 +132,12 @@ class MilestoneContest(MilestoneContestCreate, Contest): if res: return res, msg - if self.status == ContestStatus.ACTIVE: - if self.end_condition.max_winners: - if self.win_count >= self.end_condition.max_winners: - return True, ContestEndReason.MAX_WINNERS + if ( + self.status == ContestStatus.ACTIVE + and self.end_condition.max_winners + and self.win_count >= self.end_condition.max_winners + ): + return True, ContestEndReason.MAX_WINNERS return False, None @@ -189,16 +191,10 @@ class MilestoneUserView(MilestoneContest, ContestUserView): ) def should_award(self): - if self.status == ContestStatus.ACTIVE: - if self.should_have_awarded(): - return True - return False + return bool(self.status == ContestStatus.ACTIVE and self.should_have_awarded()) def should_have_awarded(self): - if self.target_amount: - if self.user_amount >= self.target_amount: - return True - return False + return bool(self.target_amount and self.user_amount >= self.target_amount) def is_user_eligible(self, country_iso: str) -> tuple[bool, str]: passes, msg = super().is_user_eligible(country_iso=country_iso) diff --git a/generalresearch/models/thl/contest/raffle.py b/generalresearch/models/thl/contest/raffle.py index 08243f4..16a0a47 100644 --- a/generalresearch/models/thl/contest/raffle.py +++ b/generalresearch/models/thl/contest/raffle.py @@ -127,7 +127,7 @@ class RaffleContest(RaffleContestCreate, Contest): # If there is more than 1 prize, the winning entry is subtracted # from the user's entry count user_amount = defaultdict(int) - user_id_user = dict() + user_id_user = {} for entry in self.entries: user_amount[entry.user.user_id] += entry.amount user_id_user[entry.user.user_id] = entry.user @@ -149,10 +149,12 @@ class RaffleContest(RaffleContestCreate, Contest): res, msg = super().should_end() if res: return res, msg - if self.status == ContestStatus.ACTIVE: - if self.end_condition.target_entry_amount: - if self.current_amount >= self.end_condition.target_entry_amount: - return True, ContestEndReason.TARGET_ENTRY_AMOUNT + if ( + self.status == ContestStatus.ACTIVE + and self.end_condition.target_entry_amount + and self.current_amount >= self.end_condition.target_entry_amount + ): + return True, ContestEndReason.TARGET_ENTRY_AMOUNT return False, None @staticmethod @@ -278,17 +280,19 @@ class RaffleUserView(RaffleContest, ContestUserView): return probs def is_entry_eligible(self, entry: ContestEntry) -> tuple[bool, str]: - if self.entry_rule.max_entry_amount_per_user: - if ( - self.user_amount + entry.amount - ) > self.entry_rule.max_entry_amount_per_user: - return False, "Entry would exceed max amount per user." - - if self.entry_rule.max_daily_entries_per_user: - if ( - self.user_amount_today + entry.amount - ) > self.entry_rule.max_daily_entries_per_user: - return False, "Entry would exceed max amount per user per day." + if ( + self.entry_rule.max_entry_amount_per_user + and (self.user_amount + entry.amount) + > self.entry_rule.max_entry_amount_per_user + ): + return False, "Entry would exceed max amount per user." + + if ( + self.entry_rule.max_daily_entries_per_user + and (self.user_amount_today + entry.amount) + > self.entry_rule.max_daily_entries_per_user + ): + return False, "Entry would exceed max amount per user per day." return True, "" def is_user_eligible(self, country_iso: str) -> tuple[bool, str]: @@ -296,16 +300,18 @@ class RaffleUserView(RaffleContest, ContestUserView): if not passes: return False, msg - if self.entry_rule.max_entry_amount_per_user: + if self.entry_rule.max_entry_amount_per_user: # noqa: SIM102 # Greater or equal b/c we're asking if the user is eligible to # enter MORE, now! If it equals, nothing is wrong, just that they # are not eligible anymore. if self.user_amount >= self.entry_rule.max_entry_amount_per_user: return False, "Reached max amount per user." - if self.entry_rule.max_daily_entries_per_user: - if self.user_amount_today >= self.entry_rule.max_daily_entries_per_user: - return False, "Reached max amount today." + if ( + self.entry_rule.max_daily_entries_per_user + and self.user_amount_today >= self.entry_rule.max_daily_entries_per_user + ): + return False, "Reached max amount today." # This would indicate something is wrong, as something else should have done this e, _ = self.should_end() diff --git a/generalresearch/models/thl/finance.py b/generalresearch/models/thl/finance.py index 0856825..8c94390 100644 --- a/generalresearch/models/thl/finance.py +++ b/generalresearch/models/thl/finance.py @@ -124,10 +124,10 @@ class POPFinancial(BaseModel): Direction, ) - assert all([a.account_type == AccountType.BP_WALLET for a in accounts]) - assert all([a.normal_balance == Direction.CREDIT for a in accounts]) + assert all(a.account_type == AccountType.BP_WALLET for a in accounts) + assert all(a.normal_balance == Direction.CREDIT for a in accounts) if not is_debug(): - assert all([a.currency == "USD" for a in accounts]) + assert all(a.currency == "USD" for a in accounts) if input_data.empty: return [] @@ -850,11 +850,11 @@ class BusinessBalances(BaseModel): # Validate the input accounts assert len(accounts) > 0, "Must provide accounts" - assert all([a.account_type == AccountType.BP_WALLET for a in accounts]) - assert all([a.normal_balance == Direction.CREDIT for a in accounts]) + assert all(a.account_type == AccountType.BP_WALLET for a in accounts) + assert all(a.normal_balance == Direction.CREDIT for a in accounts) if not is_debug(): - assert all([a.currency == "USD" for a in accounts]) + assert all(a.currency == "USD" for a in accounts) # Validate the input dataframe assert input_data.index.name == "account_id" diff --git a/generalresearch/models/thl/product.py b/generalresearch/models/thl/product.py index 76a8e83..a7ecd55 100644 --- a/generalresearch/models/thl/product.py +++ b/generalresearch/models/thl/product.py @@ -430,8 +430,8 @@ class UserWalletConfig(BaseModel): @field_serializer("supported_payout_types", when_used="json") def serialize_supported_payout_types_in_order( self, supported_payout_types: set[PayoutType] - ) -> set[PayoutType]: - return set(sorted(supported_payout_types)) + ) -> list[PayoutType]: + return sorted(supported_payout_types) @field_validator("min_cashout", mode="after") @classmethod @@ -552,14 +552,14 @@ class PayoutTransformation(BaseModel): min_payout = Decimal(0) pct = Decimal(pct) - payout = Decimal(payout) + _payout = Decimal(payout) min_payout = Decimal(min_payout) max_payout = Decimal(max_payout) if max_payout else None - payout: Decimal = payout * pct - payout: Decimal = max([payout, min_payout]) - payout: Decimal = min([payout, max_payout]) if max_payout else payout - return payout + _payout: Decimal = _payout * pct + _payout: Decimal = max([_payout, min_payout]) + _payout: Decimal = min([_payout, max_payout]) if max_payout else payout + return _payout def payout_transformation_amt( self, payout: Decimal, user_wallet_balance: Decimal | None = None @@ -569,22 +569,22 @@ class PayoutTransformation(BaseModel): # (display, adjustment) so ignore the 7-cent rounding. if user_wallet_balance is None: return self.payout_transformation_percent(payout=payout, pct=Decimal(".95")) - payout = Decimal(payout) + _payout = Decimal(payout) - payout: Decimal = payout * Decimal("0.95") - new_balance = payout + user_wallet_balance + _payout: Decimal = _payout * Decimal("0.95") + new_balance = _payout + user_wallet_balance # If the new_balance is <0, we aren't paying anything, so use the # full amount if new_balance < 0: - return payout + return _payout amt = (5 * math.floor((int(new_balance * 100) - 2) / 5)) + 2 rounded_new_balance = Decimal(amt / 100).quantize(Decimal("0.00")) - payout = rounded_new_balance - user_wallet_balance - if payout < Decimal(0): + _payout = rounded_new_balance - user_wallet_balance + if _payout < Decimal(0): return Decimal(0) - return payout + return _payout class SourceConfig(BaseModel): @@ -731,8 +731,8 @@ class SupplyConfig(BaseModel): Use global config. """ d = self.global_scoped_policies_dict.copy() - d.update(self.team_scoped_policies_dict.get(team_id, dict())) - d.update(self.product_scoped_policies_dict.get(product_id, dict())) + d.update(self.team_scoped_policies_dict.get(team_id, {})) + d.update(self.product_scoped_policies_dict.get(product_id, {})) return d def get_config_for_product(self, product: Product) -> MergedSupplyConfig: @@ -751,7 +751,7 @@ class SupplyConfig(BaseModel): supply_policy=policy_dict[source], source_config=sources_dict[source], ) - for source in policy_dict.keys() + for source in policy_dict ] ) @@ -1000,10 +1000,12 @@ class Product(BaseModel, validate_assignment=True): @property def business_uuid(self) -> UUIDStr: + assert self.business_id return self.business_id @property def team_uuid(self) -> UUIDStr: + assert self.team_id return self.team_id @property diff --git a/generalresearch/models/thl/profiling/other_option.py b/generalresearch/models/thl/profiling/other_option.py index 6d789e5..2f3cac9 100644 --- a/generalresearch/models/thl/profiling/other_option.py +++ b/generalresearch/models/thl/profiling/other_option.py @@ -51,6 +51,4 @@ def option_is_catch_all(c: UpkQuestionChoice) -> bool: return True if c.text.lower() in texts_exact: return True - if any(t in c.text.lower() for t in texts_in): - return True - return False + return bool(any(t in c.text.lower() for t in texts_in)) diff --git a/generalresearch/models/thl/profiling/upk_question.py b/generalresearch/models/thl/profiling/upk_question.py index 77bba6f..3bb0733 100644 --- a/generalresearch/models/thl/profiling/upk_question.py +++ b/generalresearch/models/thl/profiling/upk_question.py @@ -475,10 +475,9 @@ class UpkQuestion(BaseModel): # Almost nothing has >1k options, besides location stuff (cities, # etc.) which should get harmonized. When presenting them, we'll # filter down options to at most 50. - if self.choices and (len(self.choices) <= 1 or len(self.choices) > 1000): - return False - - return True + return not ( + self.choices and (len(self.choices) <= 1 or len(self.choices) > 1000) + ) @property def md5sum(self): @@ -534,7 +533,7 @@ class UpkQuestion(BaseModel): ), "Multiple of the same answer submitted" if self.type == UpkQuestionType.MULTIPLE_CHOICE: assert len(answer) >= 1, "MC question with no selected answers" - choice_codes = set(x.id for x in self.choices) + choice_codes = {x.id for x in self.choices} if self.selector == UpkQuestionSelectorMC.SINGLE_ANSWER: assert ( len(answer) == 1 @@ -563,9 +562,7 @@ class UpkQuestion(BaseModel): assert len(answer) == 1, "Only one answer allowed" answer = answer[0] assert len(answer) > 0, "Must provide answer" - max_length = ( - self.configuration.max_length if self.configuration else 0 or 100000 - ) + max_length = self.configuration.max_length if self.configuration else 100000 assert len(answer) <= max_length, "Answer longer than allowed" if self.validation and self.validation.patterns: for pattern in self.validation.patterns: diff --git a/generalresearch/models/thl/profiling/user_question_answer.py b/generalresearch/models/thl/profiling/user_question_answer.py index 2db07b7..378345e 100644 --- a/generalresearch/models/thl/profiling/user_question_answer.py +++ b/generalresearch/models/thl/profiling/user_question_answer.py @@ -3,7 +3,7 @@ from __future__ import annotations import json from collections.abc import Iterator from datetime import UTC, datetime, timedelta -from typing import Any, Literal, Self +from typing import Any, Literal from pydantic import ( BaseModel, @@ -14,7 +14,6 @@ from pydantic import ( model_validator, ) -from generalresearch.grpc import timestamp_to_datetime from generalresearch.models import MAX_INT32, Source from generalresearch.models.custom_types import AwareDatetimeISO, UUIDStr from generalresearch.models.thl.locales import CountryISO, LanguageISO @@ -39,19 +38,23 @@ class UserQuestionAnswer(BaseModel): calc_answers: dict[str, tuple[str, ...]] | None = Field(default=None) @field_validator("calc_answers") - def sorted_calc_answers(cls, calc_answers) -> dict[str, tuple[str, ...]] | None: + def sorted_calc_answers( + cls, calc_answers: dict[str, tuple[str, ...]] | None + ) -> dict[str, tuple[str, ...]] | None: if calc_answers is None: return None return {k: tuple(sorted(v)) for k, v in calc_answers.items()} @field_validator("calc_answers") - def validate_keys(cls, calc_answers) -> dict[str, tuple[str, ...]] | None: + def validate_keys( + cls, calc_answers: dict[str, tuple[str, ...]] | None + ) -> dict[str, tuple[str, ...]] | None: if calc_answers is None: return None assert all( - ":" in k for k in calc_answers.keys() + ":" in k for k in calc_answers ), "calc_answers expects the keys to be in format source:question_code" return calc_answers @@ -66,6 +69,7 @@ class UserQuestionAnswer(BaseModel): return d def get_mrpqs(self) -> Iterator[MarketplaceResearchProfileQuestion]: + assert self.calc_answers for k, v in self.calc_answers.items(): source, question_code = k.split(":", 1) yield MarketplaceResearchProfileQuestion( @@ -105,21 +109,6 @@ class UserQuestionAnswer(BaseModel): def is_stale(self) -> bool: return self.timestamp < datetime.now(tz=UTC) - timedelta(days=30) - @classmethod - def from_grpc(cls, msg, default_timestamp: datetime) -> Self: - """ - Handles correctly issues with grpc timestamps - :param msg: "thl.protos.generalresearch_pb2.ProfilingQuestionAnswer" - """ - assert default_timestamp.tzinfo is not None, "must use tz-aware timestamps" - timestamp = timestamp_to_datetime(msg.timestamp) - timestamp = default_timestamp if timestamp < datetime(2000, 1, 1) else timestamp - return cls( - question_id=msg.question_id, - answer=tuple(msg.answer), - timestamp=timestamp, - ) - # We can't set a redis list to [] vs None. We'll push this dummy answer into # the cache to signify the user has no answered questions. It'll get removed @@ -131,7 +120,7 @@ DUMMY_UQA = UserQuestionAnswer( country_iso="xx", language_iso="xxx", property_code="dummy", - calc_answers=dict(), + calc_answers={}, ) diff --git a/pyproject.toml b/pyproject.toml index dbdf3b9..03a1a1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,4 +56,7 @@ testpaths = ["tests"] addopts = "-v --tb=short" [tool.ruff] -target-version = "py314" \ No newline at end of file +target-version = "py314" +exclude = [ + "generalresearch/thl_django", +] \ No newline at end of file diff --git a/tests/incite/collections/test_df_collection_item_thl_web.py b/tests/incite/collections/test_df_collection_item_thl_web.py index 8038d3b..edf90f7 100644 --- a/tests/incite/collections/test_df_collection_item_thl_web.py +++ b/tests/incite/collections/test_df_collection_item_thl_web.py @@ -5,12 +5,12 @@ from datetime import UTC, datetime, timedelta from itertools import product as iter_product from os.path import join as pjoin from pathlib import Path, PurePath -from typing import TYPE_CHECKING from uuid import uuid4 import dask.dataframe as dd import pandas as pd import pytest +from dask.distributed import Client as DaskClient from distributed import Client, Scheduler, Worker # noinspection PyUnresolvedReferences @@ -21,20 +21,19 @@ from faker import Faker from pandera.pandas import DataFrameSchema from pydantic import FilePath -from generalresearch.incite.base import CollectionItemBase +from generalresearch.incite.base import CollectionItemBase, GRLDatasets from generalresearch.incite.collections import ( + DFCollection, DFCollectionItem, DFCollectionType, ) from generalresearch.incite.schemas import ARCHIVE_AFTER +from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.models.thl.product import Product from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig from generalresearch.sql_helper import PostgresDsn -if TYPE_CHECKING: - from generalresearch.incite.base import GRLDatasets - fake = Faker() df_collections = [ @@ -72,7 +71,12 @@ class TestDFCollectionItemBase: ) class TestDFCollectionItemProperties: - def test_filename(self, df_collection_data_type, df_collection, offset: str): + def test_filename( + self, + df_collection_data_type: DFCollectionType, + df_collection: DFCollection, + offset: str, + ): for i in df_collection.items: assert isinstance(i.filename, str) @@ -89,37 +93,59 @@ class TestDFCollectionItemProperties: ) class TestDFCollectionItemPropertiesBase: - def test_name(self, df_collection_data_type, offset: str, df_collection): + def test_name( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.name, str) - def test_finish(self, df_collection_data_type, offset: str, df_collection): + def test_finish( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.finish, datetime) - def test_interval(self, df_collection_data_type, offset: str, df_collection): + def test_interval( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.interval, pd.Interval) def test_partial_filename( - self, df_collection_data_type, offset: str, df_collection + self, + df_collection: DFCollection, ): for i in df_collection.items: assert isinstance(i.partial_filename, str) - def test_empty_filename(self, df_collection_data_type, offset: str, df_collection): + def test_empty_filename( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.empty_filename, str) - def test_path(self, df_collection_data_type, offset: str, df_collection): + def test_path( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.path, FilePath) - def test_partial_path(self, df_collection_data_type, offset: str, df_collection): + def test_partial_path( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.partial_path, FilePath) - def test_empty_path(self, df_collection_data_type, offset: str, df_collection): + def test_empty_path( + self, + df_collection: DFCollection, + ): for i in df_collection.items: assert isinstance(i.empty_path, FilePath) @@ -138,11 +164,8 @@ class TestDFCollectionItemMethod: def test_has_mysql( self, - df_collection, + df_collection: DFCollection, thl_web_rr: PostgresConfig, - offset: str, - duration: timedelta, - df_collection_data_type, delete_df_collection: Callable[..., None], ): delete_df_collection(coll=df_collection) @@ -168,12 +191,6 @@ class TestDFCollectionItemMethod: @pytest.mark.skip def test_update_partial_archive( self, - df_collection, - offset: str, - duration: timedelta, - thl_web_rw: PostgresConfig, - df_collection_data_type, - delete_df_collection: Callable[..., None], ): # for i in collection.items: # assert i.update_partial_archive() @@ -183,28 +200,12 @@ class TestDFCollectionItemMethod: @pytest.mark.skip def test_create_partial_archive( self, - df_collection, - offset: str, - duration: str, - create_main_accounts: Callable[..., None], - thl_web_rw: PostgresConfig, - thl_lm, - df_collection_data_type, - user_factory: Callable[..., User], - product: product: Product, - client_no_amm, - incite_item_factory, - delete_df_collection: Callable[..., None], - mnt_filepath: GRLDatasets, ): assert 1 + 1 == 2 def test_dict( self, - df_collection_data_type, - offset: str, - duration: timedelta, - df_collection, + df_collection: DFCollection, delete_df_collection: Callable[..., None], ): delete_df_collection(coll=df_collection) @@ -225,15 +226,15 @@ class TestDFCollectionItemMethod: def test_from_mysql( self, - df_collection_data_type, - df_collection, + df_collection_data_type: DFCollectionType, + df_collection: DFCollection, offset: str, duration: timedelta, create_main_accounts: Callable[..., None], thl_web_rw: PostgresConfig, user_factory: Callable[..., User], - product: product: Product, - incite_item_factory, + product: Product, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], ): @@ -253,12 +254,14 @@ class TestDFCollectionItemMethod: if df_collection.data_type == DFCollectionType.LEDGER: assert df is None else: + assert isinstance(df, pd.DataFrame) assert df.empty assert set(df.columns) == set(df_collection._schema.columns.keys()) incite_item_factory(user=u1, item=item) df = item.from_mysql() + assert isinstance(df, pd.DataFrame) assert not df.empty assert set(df.columns) == set(df_collection._schema.columns.keys()) if df_collection.data_type == DFCollectionType.LEDGER: @@ -270,13 +273,13 @@ class TestDFCollectionItemMethod: def test_from_mysql_standard( self, - df_collection_data_type, - df_collection, + df_collection_data_type: DFCollectionType, + df_collection: DFCollection, offset: str, duration: timedelta, user_factory: Callable[..., User], - product: product: Product, - incite_item_factory, + product: Product, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], ): @@ -293,7 +296,7 @@ class TestDFCollectionItemMethod: # We're using parametrize, so this If statement is just to # confirm other Item Types will always raise an assertion with pytest.raises(expected_exception=AssertionError) as cm: - res = item.from_mysql_standard() + _ = item.from_mysql_standard() assert ( "Can't call from_mysql_standard for Ledger DFCollectionItem" in str(cm.value) @@ -304,32 +307,34 @@ class TestDFCollectionItemMethod: # Unlike .from_mysql_ledger(), .from_mysql_standard() will return # back and empty df with the correct columns in place df = item.from_mysql_standard() + assert isinstance(df, pd.DataFrame) assert df.empty assert set(df.columns) == set(df_collection._schema.columns.keys()) incite_item_factory(user=u1, item=item) df = item.from_mysql_standard() + assert isinstance(df, pd.DataFrame) assert not df.empty assert set(df.columns) == set(df_collection._schema.columns.keys()) assert df.shape[0] > 0 def test_from_mysql_ledger( self, - df_collection, + df_collection: DFCollection, user: User, create_main_accounts: Callable[..., None], offset: str, duration: timedelta, thl_web_rw: PostgresConfig, - thl_lm, - df_collection_data_type, + thl_ledger_manager: ThlLedgerManager, + df_collection_data_type: DFCollectionType, user_factory: Callable[..., User], - product: product: Product, - client_no_amm, - incite_item_factory, + product: Product, + client_no_amm: DaskClient, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath, + mnt_filepath: GRLDatasets, ): if df_collection.data_type != DFCollectionType.LEDGER: @@ -370,17 +375,17 @@ class TestDFCollectionItemMethod: def test_to_archive( self, - df_collection, + df_collection: DFCollection, user: User, offset: str, duration: timedelta, - df_collection_data_type, + df_collection_data_type: DFCollectionType, user_factory: Callable[..., User], - product: product: Product, - client_no_amm, - incite_item_factory, + product: Product, + client_no_amm: DaskClient, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath, + mnt_filepath: GRLDatasets, ): if df_collection.data_type in unsupported_mock_types: @@ -407,17 +412,17 @@ class TestDFCollectionItemMethod: def test__to_archive( self, - df_collection_data_type, - df_collection, + df_collection_data_type: DFCollectionType, + df_collection: DFCollection, user_factory: Callable[..., User], - product: product: Product, + product: Product, offset: str, duration: timedelta, - client_no_amm, + client_no_amm: DaskClient, user: User, - incite_item_factory, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath, + mnt_filepath: GRLDatasets, ): """We already have a test for the "non-private" version of this, which primarily just uses the respective Client to determine if @@ -480,19 +485,19 @@ class TestDFCollectionItemMethod: @pytest.mark.skip def test_to_archive_numbered_partial( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_initial_load( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_clear_corrupt_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @@ -505,34 +510,40 @@ class TestDFCollectionItemMethodBase: @pytest.mark.skip def test_path_exists( - self, df_collection_data_type, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_next_numbered_path( - self, df_collection_data_type, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_search_highest_numbered_path( - self, df_collection_data_type, offset: str, duration: timedelta + self, + df_collection_data_type: DFCollectionType, + offset: str, + duration: timedelta, ): pass @pytest.mark.skip def test_tmp_filename( - self, df_collection_data_type, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip - def test_tmp_path(self, df_collection_data_type, offset: str, duration: timedelta): + def test_tmp_path( + self, + ): pass def test_is_empty( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): """ test_has_empty was merged into this because item.has_empty is @@ -549,7 +560,8 @@ class TestDFCollectionItemMethodBase: assert item.has_empty() def test_has_partial_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): for item in df_collection.items: assert not item.has_partial_archive() @@ -557,7 +569,8 @@ class TestDFCollectionItemMethodBase: assert item.has_partial_archive() def test_has_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): for item in df_collection.items: # (1) Originally, nothing exists... so let's just make a file and @@ -594,7 +607,8 @@ class TestDFCollectionItemMethodBase: assert item.has_archive(include_empty=True) def test_delete_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): for item in df_collection.items: item: DFCollectionItem @@ -617,7 +631,8 @@ class TestDFCollectionItemMethodBase: assert not item.partial_path.exists() def test_should_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): schema: DataFrameSchema = df_collection._schema aa = schema.metadata[ARCHIVE_AFTER] @@ -635,12 +650,13 @@ class TestDFCollectionItemMethodBase: @pytest.mark.skip def test_set_empty( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass def test_valid_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): # Originally, nothing has been saved or anything.. so confirm it # always comes back as None @@ -664,18 +680,19 @@ class TestDFCollectionItemMethodBase: @pytest.mark.skip def test_validate_df( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_from_archive( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass def test__to_dict( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): for item in df_collection.items: @@ -694,19 +711,19 @@ class TestDFCollectionItemMethodBase: @pytest.mark.skip def test_delete_partial( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_cleanup_partials( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @pytest.mark.skip def test_delete_dangling_partials( - self, df_collection_data_type, df_collection, offset: str, duration: timedelta + self, ): pass @@ -726,7 +743,9 @@ async def test_client(client, s, worker): ) @gen_cluster(client=True, nthreads=[("127.0.0.1", 1)]) @pytest.mark.anyio -async def test_client_parametrize(c, s, w, df_collection_data_type, offset: str): +async def test_client_parametrize( + c, s, w, df_collection_data_type: DFCollectionType, offset: str +): """c,s,a are all required - the secondary Worker (b) is not required""" assert isinstance(c, Client), f"c is not Client, it's {type(c)}" @@ -750,17 +769,12 @@ class TestDFCollectionItemFunctionalTest: def test_to_archive_and_ddf( self, - df_collection_data_type, - offset: str, - duration: timedelta, - client_no_amm, - df_collection, - user: User, + client_no_amm: DaskClient, + df_collection: DFCollection, user_factory: Callable[..., User], - product: product: Product, - incite_item_factory, + product: Product, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath: GRLDatasets, ): if df_collection.data_type in unsupported_mock_types: @@ -799,17 +813,11 @@ class TestDFCollectionItemFunctionalTest: def test_filesize_estimate( self, - df_collection, - user: User, - offset: str, - duration: timedelta, - client_no_amm, + df_collection: DFCollection, user_factory: Callable[..., User], - product: product: Product, - df_collection_data_type, - incite_item_factory, + product: Product, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath: GRLDatasets, ): """A functional test to write some Parquet files for the DFCollection and then confirm that the files get written @@ -846,16 +854,12 @@ class TestDFCollectionItemFunctionalTest: def test_to_archive_client( self, - client_no_amm, - df_collection, + client_no_amm: DaskClient, + df_collection: DFCollection, user_factory: Callable[..., User], - product: product: Product, - offset: str, - duration: timedelta, - df_collection_data_type, - incite_item_factory, + product: Product, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], - mnt_filepath: GRLDatasets, ): delete_df_collection(coll=df_collection) @@ -885,7 +889,8 @@ class TestDFCollectionItemFunctionalTest: @pytest.mark.skip def test_get_items( - self, df_collection, product: product: Product, offset: str, duration: timedelta + self, + df_collection: DFCollection, ): with pytest.warns(expected_warning=ResourceWarning) as cm: df_collection.get_items_last365() @@ -898,16 +903,11 @@ class TestDFCollectionItemFunctionalTest: def test_saving_protections( self, - client_no_amm, - df_collection_data_type, - df_collection, - incite_item_factory, + df_collection: DFCollection, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], user_factory: Callable[..., User], - product: product: Product, - offset: str, - duration: timedelta, - mnt_filepath: GRLDatasets, + product: Product, ): """Don't allow creating an archive for data that will likely be overwritten or updated @@ -939,15 +939,8 @@ class TestDFCollectionItemFunctionalTest: def test_empty_item( self, - client_no_amm, - df_collection_data_type, - df_collection, - incite_item_factory, + df_collection: DFCollection, delete_df_collection: Callable[..., None], - user: User, - offset: str, - duration: timedelta, - mnt_filepath: GRLDatasets, ): delete_df_collection(coll=df_collection) @@ -967,16 +960,12 @@ class TestDFCollectionItemFunctionalTest: def test_file_touching( self, - client_no_amm, - df_collection_data_type, - df_collection, - incite_item_factory, + client_no_amm: DaskClient, + df_collection: DFCollection, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], user_factory: Callable[..., User], - product: product: Product, - offset: str, - duration: timedelta, - mnt_filepath, + product: Product, ): delete_df_collection(coll=df_collection) diff --git a/tests/incite/mergers/foundations/test_enriched_session.py b/tests/incite/mergers/foundations/test_enriched_session.py index 8254d81..2a161e4 100644 --- a/tests/incite/mergers/foundations/test_enriched_session.py +++ b/tests/incite/mergers/foundations/test_enriched_session.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from collections.abc import Callable from datetime import UTC, datetime, timedelta from decimal import Decimal from itertools import product @@ -5,10 +8,24 @@ from itertools import product import dask.dataframe as dd import pandas as pd import pytest +from dask.distributed import Client as DaskClient +from generalresearch.incite.collections.thl_web import ( + SessionDFCollection, + WallDFCollection, +) +from generalresearch.incite.mergers.foundations.enriched_session import ( + EnrichedSessionMerge, +) from generalresearch.incite.schemas.admin_responses import ( AdminPOPSessionSchema, ) +from generalresearch.models.admin.request import ( + ReportRequest, +) +from generalresearch.models.thl.product import Product +from generalresearch.models.thl.session import Session +from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig @@ -25,21 +42,20 @@ class TestEnrichedSession: def test_base( self, - client_no_amm, + client_no_amm: DaskClient, product: Product, user_factory: Callable[..., User], - wall_collection, - session_collection, - enriched_session_merge, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, + enriched_session_merge: EnrichedSessionMerge, thl_web_rr: PostgresConfig, delete_df_collection: Callable[..., None], - incite_item_factory, + incite_item_factory: Callable[..., None], ): - from generalresearch.models.thl.user import User delete_df_collection(coll=session_collection) - u1: User = user_factory(product=product: Product, created=session_collection.start) + u1: User = user_factory(product=product, created=session_collection.start) for item in session_collection.items: incite_item_factory(item=item, user=u1) @@ -52,7 +68,7 @@ class TestEnrichedSession: client=client_no_amm, wall_coll=wall_collection, session_coll=session_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) # -- @@ -85,16 +101,16 @@ class TestEnrichedSessionAdmin: def test_to_admin_response( self, - event_report_request, - enriched_session_merge, - client_no_amm, - wall_collection, - session_collection, + event_report_request: ReportRequest, + enriched_session_merge: EnrichedSessionMerge, + client_no_amm: DaskClient, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, thl_web_rr: PostgresConfig, - session_report_request, + session_report_request: ReportRequest, user_factory: Callable[..., User], - start, - session_factory, + start: datetime, + session_factory: Callable[..., Session], product_factory: Callable[..., Product], delete_df_collection: Callable[..., None], ): @@ -107,7 +123,7 @@ class TestEnrichedSessionAdmin: for p in [p1, p2]: u = user_factory(product=p) for i in range(50): - s = session_factory( + _ = session_factory( user=u, wall_count=1, wall_req_cpi=Decimal("1.00"), @@ -120,7 +136,7 @@ class TestEnrichedSessionAdmin: client=client_no_amm, session_coll=session_collection, wall_coll=wall_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) df = enriched_session_merge.to_admin_response( diff --git a/tests/incite/mergers/foundations/test_enriched_task_adjust.py b/tests/incite/mergers/foundations/test_enriched_task_adjust.py index a33a55a..0606b6f 100644 --- a/tests/incite/mergers/foundations/test_enriched_task_adjust.py +++ b/tests/incite/mergers/foundations/test_enriched_task_adjust.py @@ -1,9 +1,28 @@ +from __future__ import annotations + +from collections.abc import Callable from datetime import timedelta from itertools import product as iter_product import dask.dataframe as dd import pandas as pd import pytest +from dask.distributed import Client as DaskClient + +from generalresearch.incite.collections.thl_web import ( + SessionDFCollection, + TaskAdjustmentDFCollection, + WallDFCollection, +) +from generalresearch.incite.mergers.foundations.enriched_task_adjust import ( + EnrichedTaskAdjustMerge, +) +from generalresearch.incite.mergers.foundations.enriched_wall import ( + EnrichedWallMerge, +) +from generalresearch.models.thl.product import Product +from generalresearch.models.thl.user import User +from generalresearch.pg_helper import PostgresConfig @pytest.mark.parametrize( @@ -20,19 +39,18 @@ class TestEnrichedTaskAdjust: @pytest.mark.skip def test_base( self, - client_no_amm, + client_no_amm: DaskClient, user_factory: Callable[..., User], product: Product, - task_adj_collection, - wall_collection, - session_collection, - enriched_wall_merge, - enriched_task_adjust_merge, - incite_item_factory, + task_adj_collection: TaskAdjustmentDFCollection, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, + enriched_wall_merge: EnrichedWallMerge, + enriched_task_adjust_merge: EnrichedTaskAdjustMerge, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], thl_web_rr: PostgresConfig, ): - from generalresearch.models.thl.user import User # -- Build & Setup delete_df_collection(coll=session_collection) @@ -48,14 +66,14 @@ class TestEnrichedTaskAdjust: client=client_no_amm, session_coll=session_collection, wall_coll=wall_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) enriched_task_adjust_merge.build( client=client_no_amm, task_adjust_coll=task_adj_collection, enriched_wall=enriched_wall_merge, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) # -- diff --git a/tests/incite/mergers/foundations/test_enriched_wall.py b/tests/incite/mergers/foundations/test_enriched_wall.py index a0ca4dd..0cb8f60 100644 --- a/tests/incite/mergers/foundations/test_enriched_wall.py +++ b/tests/incite/mergers/foundations/test_enriched_wall.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from datetime import UTC, datetime, timedelta from decimal import Decimal from itertools import product as iter_product @@ -5,11 +6,23 @@ from itertools import product as iter_product import dask.dataframe as dd import pandas as pd import pytest +from dask.distributed import Client as DaskClient + +from generalresearch.incite.collections.thl_web import ( + SessionDFCollection, + WallDFCollection, +) # noinspection PyUnresolvedReferences from generalresearch.incite.mergers.foundations.enriched_wall import ( + EnrichedWallMerge, EnrichedWallMergeItem, ) +from generalresearch.models.admin.request import ReportRequest +from generalresearch.models.thl.product import Product +from generalresearch.models.thl.session import Session +from generalresearch.models.thl.user import User +from generalresearch.pg_helper import PostgresConfig @pytest.mark.parametrize( @@ -20,22 +33,21 @@ class TestEnrichedWall: def test_base( self, - client_no_amm, + client_no_amm: DaskClient, product: Product, user_factory: Callable[..., User], - wall_collection, + wall_collection: WallDFCollection, thl_web_rr: PostgresConfig, - session_collection, - enriched_wall_merge, + session_collection: SessionDFCollection, + enriched_wall_merge: EnrichedWallMerge, delete_df_collection: Callable[..., None], - incite_item_factory, + incite_item_factory: Callable[..., None], ): - from generalresearch.models.thl.user import User # -- Build & Setup delete_df_collection(coll=session_collection) delete_df_collection(coll=wall_collection) - u1: User = user_factory(product=product: Product, created=session_collection.start) + u1: User = user_factory(product=product, created=session_collection.start) for item in session_collection.items: incite_item_factory(item=item, user=u1) @@ -48,7 +60,7 @@ class TestEnrichedWall: client=client_no_amm, wall_coll=wall_collection, session_coll=session_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) # -- @@ -63,19 +75,19 @@ class TestEnrichedWall: def test_base_item( self, - client_no_amm, + client_no_amm: DaskClient, product: Product, user_factory: Callable[..., User], - wall_collection, - session_collection, - enriched_wall_merge, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, + enriched_wall_merge: EnrichedWallMerge, delete_df_collection: Callable[..., None], thl_web_rr: PostgresConfig, - incite_item_factory, + incite_item_factory: Callable[..., None], ): # -- Build & Setup delete_df_collection(coll=session_collection) - u = user_factory(product=product: Product, created=session_collection.start) + u = user_factory(product=product, created=session_collection.start) for item in session_collection.items: incite_item_factory(item=item, user=u) @@ -87,7 +99,7 @@ class TestEnrichedWall: client=client_no_amm, wall_coll=wall_collection, session_coll=session_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) # -- @@ -99,14 +111,14 @@ class TestEnrichedWall: try: modified_time1 = path.stat().st_mtime - except Exception: + except OSError: modified_time1 = 0 item.build( client=client_no_amm, wall_coll=wall_collection, session_coll=session_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) modified_time2 = path.stat().st_mtime @@ -150,7 +162,12 @@ class TestEnrichedWallToAdmin: def duration(self) -> timedelta | None: return timedelta(days=5) - def test_empty(self, enriched_wall_merge, client_no_amm, start): + def test_empty( + self, + enriched_wall_merge: EnrichedWallMerge, + client_no_amm: DaskClient, + start: datetime, + ): from generalresearch.models.admin.request import ReportRequest rr = ReportRequest.model_validate({"interval": "5min", "start": start}) @@ -167,18 +184,18 @@ class TestEnrichedWallToAdmin: def test_to_admin_response( self, - event_report_request, - enriched_wall_merge, - client_no_amm, - wall_collection, - session_collection, + event_report_request: ReportRequest, + enriched_wall_merge: EnrichedWallMerge, + client_no_amm: DaskClient, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, thl_web_rr: PostgresConfig, - user, - session_factory, + user: User, + session_factory: Callable[..., Session], delete_df_collection: Callable[..., None], product_factory: Callable[..., Product], user_factory: Callable[..., User], - start, + start: datetime, ): delete_df_collection(coll=wall_collection) delete_df_collection(coll=session_collection) @@ -189,7 +206,7 @@ class TestEnrichedWallToAdmin: for p in [p1, p2]: u = user_factory(product=p) for i in range(50): - s = session_factory( + _ = session_factory( user=u, wall_count=2, wall_req_cpi=Decimal("1.00"), @@ -203,7 +220,7 @@ class TestEnrichedWallToAdmin: client=client_no_amm, wall_coll=wall_collection, session_coll=session_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) df = enriched_wall_merge.to_admin_response( diff --git a/tests/incite/mergers/test_merge_collection.py b/tests/incite/mergers/test_merge_collection.py index 15fa4db..cf8315f 100644 --- a/tests/incite/mergers/test_merge_collection.py +++ b/tests/incite/mergers/test_merge_collection.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import UTC, datetime, timedelta from itertools import product @@ -5,12 +7,13 @@ import pandas as pd import pytest from pandera.pandas import DataFrameSchema +from generalresearch.incite.base import GRLDatasets from generalresearch.incite.mergers import ( MergeCollection, MergeType, ) -merge_types = list(e for e in MergeType if e != MergeType.TEST) +merge_types = [e for e in MergeType if e != MergeType.TEST] @pytest.mark.parametrize( @@ -26,7 +29,11 @@ merge_types = list(e for e in MergeType if e != MergeType.TEST) ) class TestMergeCollection: - def test_init(self, mnt_filepath, merge_type, offset, duration, start): + def test_init( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + ): with pytest.raises(expected_exception=ValueError) as cm: MergeCollection(archive_path=mnt_filepath.data_src) assert "Must explicitly provide a merge_type" in str(cm.value) @@ -37,7 +44,14 @@ class TestMergeCollection: ) assert instance.merge_type == merge_type - def test_items(self, mnt_filepath, merge_type, offset, duration, start): + def test_items( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + offset: str, + duration: timedelta, + start: datetime, + ): instance = MergeCollection( merge_type=merge_type, offset=offset, @@ -48,7 +62,14 @@ class TestMergeCollection: assert len(instance.interval_range) == len(instance.items) - def test_progress(self, mnt_filepath, merge_type, offset, duration, start): + def test_progress( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + offset: str, + duration: timedelta, + start: datetime, + ): instance = MergeCollection( merge_type=merge_type, offset=offset, @@ -62,7 +83,11 @@ class TestMergeCollection: assert instance.progress.shape[1] == 7 assert instance.progress["group_by"].isnull().all() - def test_schema(self, mnt_filepath, merge_type, offset, duration, start): + def test_schema( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + ): instance = MergeCollection( merge_type=merge_type, archive_path=mnt_filepath.archive_path(enum_type=merge_type), @@ -70,7 +95,14 @@ class TestMergeCollection: assert isinstance(instance._schema, DataFrameSchema) - def test_load(self, mnt_filepath, merge_type, offset, duration, start): + def test_load( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + offset: str, + duration: timedelta, + start: datetime, + ): instance = MergeCollection( merge_type=merge_type, start=start, @@ -82,7 +114,14 @@ class TestMergeCollection: # Confirm that there are no archives available yet assert instance.progress.has_archive.eq(False).all() - def test_get_items(self, mnt_filepath, merge_type, offset, duration, start): + def test_get_items( + self, + mnt_filepath: GRLDatasets, + merge_type: MergeType, + offset: str, + duration: timedelta, + start: datetime, + ): instance = MergeCollection( start=start, finished=start + duration, diff --git a/tests/incite/mergers/test_merge_collection_item.py b/tests/incite/mergers/test_merge_collection_item.py index 3d0b644..5ca2f6b 100644 --- a/tests/incite/mergers/test_merge_collection_item.py +++ b/tests/incite/mergers/test_merge_collection_item.py @@ -1,10 +1,16 @@ +from __future__ import annotations + from datetime import timedelta from itertools import product from pathlib import PurePath import pytest -from generalresearch.incite.mergers import MergeCollectionItem, MergeType +from generalresearch.incite.mergers import ( + MergeCollection, + MergeCollectionItem, + MergeType, +) @pytest.mark.parametrize( @@ -19,7 +25,10 @@ from generalresearch.incite.mergers import MergeCollectionItem, MergeType ) class TestMergeCollectionItem: - def test_file_naming(self, merge_collection, offset, duration, start): + def test_file_naming( + self, + merge_collection: MergeCollection, + ): assert len(merge_collection.items) == 25 items: list[MergeCollectionItem] = merge_collection.items @@ -34,7 +43,10 @@ class TestMergeCollectionItem: assert i._collection.offset in i.filename assert i.start.strftime("%Y-%m-%d-%H-%M-%S") in i.filename - def test_archives(self, merge_collection, offset, duration, start): + def test_archives( + self, + merge_collection: MergeCollection, + ): assert len(merge_collection.items) == 25 for i in merge_collection.items: @@ -44,10 +56,13 @@ class TestMergeCollectionItem: assert not i.has_partial_archive() assert i.has_archive() == i.path_exists(generic_path=i.path) - res = set([i.should_archive() for i in merge_collection.items]) + res = {i.should_archive() for i in merge_collection.items} assert len(res) == 1 - def test_item_to_archive(self, merge_collection, offset, duration, start): + def test_item_to_archive( + self, + merge_collection: MergeCollection, + ): for item in merge_collection.items: item: MergeCollectionItem assert not item.has_archive() diff --git a/tests/incite/mergers/test_pop_ledger.py b/tests/incite/mergers/test_pop_ledger.py index d054eb6..529a641 100644 --- a/tests/incite/mergers/test_pop_ledger.py +++ b/tests/incite/mergers/test_pop_ledger.py @@ -1,12 +1,25 @@ +from __future__ import annotations + +from collections.abc import Callable from datetime import UTC, datetime, timedelta from itertools import product as iter_product import pandas as pd import pytest +from dask.distributed import Client as DaskClient +from generalresearch.incite.base import GRLDatasets +from generalresearch.incite.collections.thl_web import ( + LedgerDFCollection, + SessionDFCollection, +) +from generalresearch.incite.mergers.pop_ledger import PopLedgerMerge from generalresearch.incite.schemas.mergers.pop_ledger import ( numerical_col_names, ) +from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager +from generalresearch.models.thl.product import Product +from generalresearch.models.thl.user import User @pytest.mark.parametrize( @@ -30,20 +43,20 @@ class TestMergePOPLedger: def test_base( self, - client_no_amm, - ledger_collection, - pop_ledger_merge, + client_no_amm: DaskClient, + ledger_collection: LedgerDFCollection, + pop_ledger_merge: PopLedgerMerge, product: Product, user_factory: Callable[..., User], create_main_accounts: Callable[..., None], - thl_lm, + thl_ledger_manager: ThlLedgerManager, delete_df_collection: Callable[..., None], - incite_item_factory, + incite_item_factory: Callable[..., None], delete_ledger_db: Callable[..., None], ): from generalresearch.models.thl.ledger import LedgerAccount - u = user_factory(product=product: Product, created=ledger_collection.start) + u = user_factory(product=product, created=ledger_collection.start) # -- Build & Setup delete_ledger_db() @@ -73,19 +86,21 @@ class TestMergePOPLedger: # -- - user_wallet_account: LedgerAccount = thl_lm.get_account_or_create_user_wallet( - user=u + user_wallet_account: LedgerAccount = ( + thl_ledger_manager.get_account_or_create_user_wallet(user=u) + ) + cash_account: LedgerAccount = thl_ledger_manager.get_account_cash() + rev_account: LedgerAccount = ( + thl_ledger_manager.get_account_task_complete_revenue() ) - cash_account: LedgerAccount = thl_lm.get_account_cash() - rev_account: LedgerAccount = thl_lm.get_account_task_complete_revenue() item_finishes = [i.finish for i in ledger_collection.items] item_finishes.sort(reverse=True) last_item_finish = item_finishes[0] # Pure SQL based lookups - cash_balance: int = thl_lm.get_account_balance(account=cash_account) - rev_balance: int = thl_lm.get_account_balance(account=rev_account) + cash_balance: int = thl_ledger_manager.get_account_balance(account=cash_account) + rev_balance: int = thl_ledger_manager.get_account_balance(account=rev_account) assert cash_balance > rev_balance # (1) Test Cash Account @@ -123,39 +138,42 @@ class TestMergePOPLedger: def test_pydantic_init( self, - client_no_amm, - ledger_collection, - pop_ledger_merge, - mnt_filepath, + client_no_amm: DaskClient, + ledger_collection: LedgerDFCollection, + pop_ledger_merge: PopLedgerMerge, + mnt_filepath: GRLDatasets, product: Product, user_factory: Callable[..., User], create_main_accounts: Callable[..., None], - offset, - duration, - start, - thl_lm, - incite_item_factory, + offset: str, + duration: timedelta, + start: datetime, + thl_ledger_manager: ThlLedgerManager, + incite_item_factory: Callable[..., None], delete_df_collection: Callable[..., None], delete_ledger_db: Callable[..., None], - session_collection, + session_collection: SessionDFCollection, ): from generalresearch.models.thl.finance import ProductBalances from generalresearch.models.thl.ledger import LedgerAccount from generalresearch.models.thl.product import Product - u = user_factory(product=product: Product, created=session_collection.start) + u = user_factory(product=product, created=session_collection.start) assert ledger_collection.finished is not None - assert isinstance(u.product: Product, Product) + assert isinstance(u.product, Product) delete_ledger_db() - create_main_accounts(), + create_main_accounts() + delete_df_collection(coll=ledger_collection) - bp_account: LedgerAccount = thl_lm.get_account_or_create_bp_wallet( + bp_account: LedgerAccount = thl_ledger_manager.get_account_or_create_bp_wallet( product=u.product ) - cash_account: LedgerAccount = thl_lm.get_account_cash() - rev_account: LedgerAccount = thl_lm.get_account_task_complete_revenue() + cash_account: LedgerAccount = thl_ledger_manager.get_account_cash() + rev_account: LedgerAccount = ( + thl_ledger_manager.get_account_task_complete_revenue() + ) for item in ledger_collection.items: incite_item_factory(item=item, user=u) @@ -185,8 +203,10 @@ class TestMergePOPLedger: assert instance.payout == instance.net == instance.bp_payment_credit assert instance.available_balance < instance.net assert instance.available_balance + instance.retainer == instance.net - assert instance.balance == thl_lm.get_account_balance(bp_account) - assert df["bp_payment.CREDIT"].sum() == thl_lm.get_account_balance(bp_account) + assert instance.balance == thl_ledger_manager.get_account_balance(bp_account) + assert df["bp_payment.CREDIT"].sum() == thl_ledger_manager.get_account_balance( + bp_account + ) # (2) Filter by the Cash Account ddf = pop_ledger_merge.ddf( @@ -199,7 +219,7 @@ class TestMergePOPLedger: ) df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True) - cash_balance: int = thl_lm.get_account_balance(account=cash_account) + cash_balance: int = thl_ledger_manager.get_account_balance(account=cash_account) assert df["bp_payment.CREDIT"].sum() == 0 assert cash_balance > 0 assert df["mp_payment.CREDIT"].sum() == 0 @@ -216,7 +236,7 @@ class TestMergePOPLedger: ) df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True) - rev_balance: int = thl_lm.get_account_balance(account=rev_account) + rev_balance: int = thl_ledger_manager.get_account_balance(account=rev_account) assert rev_balance == 0 assert df["bp_payment.CREDIT"].sum() == 0 assert df["mp_payment.DEBIT"].sum() == 0 @@ -224,27 +244,28 @@ class TestMergePOPLedger: def test_resample( self, - client_no_amm, - ledger_collection, - pop_ledger_merge, - mnt_filepath, + client_no_amm: DaskClient, + ledger_collection: LedgerDFCollection, + pop_ledger_merge: PopLedgerMerge, + mnt_filepath: GRLDatasets, user_factory: Callable[..., User], product: Product, create_main_accounts: Callable[..., None], - offset, - duration, - start, - thl_lm, + offset: str, + duration: timedelta, + start: datetime, + thl_ledger_manager: ThlLedgerManager, delete_df_collection: Callable[..., None], - incite_item_factory, + incite_item_factory: Callable[..., None], ): - from generalresearch.models.thl.user import User assert ledger_collection.finished is not None delete_df_collection(coll=ledger_collection) u1: User = user_factory(product=product) - bp_account = thl_lm.get_account_or_create_bp_wallet(product=u1.product) + bp_account = thl_ledger_manager.get_account_or_create_bp_wallet( + product=u1.product + ) for item in ledger_collection.items: incite_item_factory(user=u1, item=item) @@ -274,7 +295,7 @@ class TestMergePOPLedger: assert isinstance(df.index, pd.Index) assert isinstance(df.index, pd.DatetimeIndex) - bp_account_balance = thl_lm.get_account_balance(account=bp_account) + bp_account_balance = thl_ledger_manager.get_account_balance(account=bp_account) # Initial sum initial_sum = df.sum().sum() diff --git a/tests/incite/mergers/test_ym_survey_merge.py b/tests/incite/mergers/test_ym_survey_merge.py index a0b8b87..8a4897b 100644 --- a/tests/incite/mergers/test_ym_survey_merge.py +++ b/tests/incite/mergers/test_ym_survey_merge.py @@ -1,8 +1,24 @@ +from __future__ import annotations + +from collections.abc import Callable from datetime import UTC, datetime, timedelta from itertools import product import pandas as pd import pytest +from dask.distributed import Client as DaskClient + +from generalresearch.incite.collections.thl_web import ( + SessionDFCollection, + WallDFCollection, +) +from generalresearch.incite.mergers.foundations.enriched_session import ( + EnrichedSessionMerge, +) +from generalresearch.incite.mergers.ym_survey_wall import YMSurveyWallMerge +from generalresearch.models.thl.product import Product +from generalresearch.models.thl.user import User +from generalresearch.pg_helper import PostgresConfig # noinspection PyUnresolvedReferences @@ -27,21 +43,20 @@ class TestYMSurveyMerge: def test_base( self, - client_no_amm, + client_no_amm: DaskClient, user_factory: Callable[..., User], product: Product, - ym_survey_wall_merge, - wall_collection, - session_collection, - enriched_session_merge, + ym_survey_wall_merge: YMSurveyWallMerge, + wall_collection: WallDFCollection, + session_collection: SessionDFCollection, + enriched_session_merge: EnrichedSessionMerge, delete_df_collection: Callable[..., None], - incite_item_factory, + incite_item_factory: Callable[..., None], thl_web_rr: PostgresConfig, ): - from generalresearch.models.thl.user import User delete_df_collection(coll=session_collection) - user: User = user_factory(product=product: Product, created=session_collection.start) + user: User = user_factory(product=product, created=session_collection.start) # -- Build & Setup assert ym_survey_wall_merge.start is None @@ -61,15 +76,15 @@ class TestYMSurveyMerge: client=client_no_amm, session_coll=session_collection, wall_coll=wall_collection, - pg_config=thl_web_rr: PostgresConfig, + pg_config=thl_web_rr, ) assert enriched_session_merge.progress.has_archive.eq(True).all() ddf = enriched_session_merge.ddf() - df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True) + df1: pd.DataFrame | None = client_no_amm.compute(collections=ddf, sync=True) - assert isinstance(df, pd.DataFrame) - assert not df.empty + assert isinstance(df1, pd.DataFrame) + assert not df1.empty # -- @@ -83,18 +98,18 @@ class TestYMSurveyMerge: # -- ddf = ym_survey_wall_merge.ddf() - df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True) + df2: pd.DataFrame | None = client_no_amm.compute(collections=ddf, sync=True) - assert isinstance(df, pd.DataFrame) - assert not df.empty + assert isinstance(df2, pd.DataFrame) + assert not df2.empty # -- - assert df.product_id.nunique() == 1 - assert df.team_id.nunique() == 1 - assert df.source.nunique() > 1 + assert df2.product_id.nunique() == 1 + assert df2.team_id.nunique() == 1 + assert df2.source.nunique() > 1 - started_min_ts = df.started.min() - started_max_ts = df.started.max() + started_min_ts = df2.started.min() + started_max_ts = df2.started.max() assert type(started_min_ts) is pd.Timestamp assert type(started_max_ts) is pd.Timestamp diff --git a/tests/incite/schemas/test_admin_responses.py b/tests/incite/schemas/test_admin_responses.py index e98eecd..d2658ea 100644 --- a/tests/incite/schemas/test_admin_responses.py +++ b/tests/incite/schemas/test_admin_responses.py @@ -1,8 +1,11 @@ +from __future__ import annotations + from datetime import UTC, datetime, timedelta from random import sample import numpy as np import pandas as pd +import pandera as pa import pytest from generalresearch.incite.schemas import empty_dataframe_from_schema @@ -16,12 +19,14 @@ from generalresearch.locales import Localelator class TestAdminPOPSchema: schema_df = empty_dataframe_from_schema(AdminPOPSchema) countries = list(Localelator().get_all_countries())[:5] - dates = [datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10)] + dates = [ + datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10) # noqa + ] @classmethod def assign_valid_vals(cls, df: pd.DataFrame) -> pd.DataFrame: for c in df.columns: - check_attrs: dict = AdminPOPSchema.columns[c].checks[0].statistics + check_attrs = AdminPOPSchema.columns[c].checks[0].statistics df[c] = np.random.randint( check_attrs["min_value"], check_attrs["max_value"], df.shape[0] ) @@ -29,7 +34,7 @@ class TestAdminPOPSchema: return df def test_empty(self): - with pytest.raises(Exception): + with pytest.raises(pa.errors.SchemaError): AdminPOPSchema.validate(pd.DataFrame()) def test_new_empty_df(self): @@ -42,7 +47,7 @@ class TestAdminPOPSchema: def test_valid(self): # (1) Works with raw naive datetime dates = [ - datetime(year=2024, month=1, day=i, tzinfo=None).isoformat() + datetime(year=2024, month=1, day=i, tzinfo=None).isoformat() # noqa for i in range(1, 10) ] df = pd.DataFrame( @@ -57,7 +62,10 @@ class TestAdminPOPSchema: assert isinstance(df, pd.DataFrame) # (2) Works with isoformat naive datetime - dates = [datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10)] + dates = [ + datetime(year=2024, month=1, day=i, tzinfo=None) # noqa + for i in range(1, 10) + ] df = pd.DataFrame( index=pd.MultiIndex.from_product( iterables=[dates, self.countries], names=["index0", "index1"] @@ -84,12 +92,12 @@ class TestAdminPOPSchema: # Initially, they're all set with a timezone timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)] - assert all([ts.tz == UTC for ts in timestmaps]) + assert all(ts.tz == UTC for ts in timestmaps) # After validation, the timezone is removed df = AdminPOPSchema.validate(df) timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)] - assert all([ts.tz is None for ts in timestmaps]) + assert all(ts.tz is None for ts in timestmaps) def test_index_tz_no_future_beyond_one_year(self): now = datetime.now(tz=UTC) @@ -123,12 +131,12 @@ class TestAdminPOPSchema: df = self.assign_valid_vals(df) vals = [i for i in df.index.get_level_values(1)] - assert all([isinstance(v, float) for v in vals]) + assert all(isinstance(v, float) for v in vals) df = AdminPOPSchema.validate(df, lazy=True) vals = [i for i in df.index.get_level_values(1)] - assert all([isinstance(v, str) for v in vals]) + assert all(isinstance(v, str) for v in vals) # --- int to str --- @@ -142,12 +150,12 @@ class TestAdminPOPSchema: df = self.assign_valid_vals(df) vals = [i for i in df.index.get_level_values(1)] - assert all([isinstance(v, int) for v in vals]) + assert all(isinstance(v, int) for v in vals) df = AdminPOPSchema.validate(df, lazy=True) vals = [i for i in df.index.get_level_values(1)] - assert all([isinstance(v, str) for v in vals]) + assert all(isinstance(v, str) for v in vals) # a = 1 assert isinstance(df, pd.DataFrame) @@ -170,7 +178,7 @@ class TestAdminPOPSchema: assert isinstance(df, pd.DataFrame) timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)] - assert all([ts.tz is None for ts in timestmaps]) + assert all(ts.tz is None for ts in timestmaps) # (2) Timezones are removed dates = [ @@ -187,12 +195,12 @@ class TestAdminPOPSchema: # Has tz before validation, and none after timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)] - assert all([ts.tz is UTC for ts in timestmaps]) + assert all(ts.tz is UTC for ts in timestmaps) df = AdminPOPSchema.validate(df, lazy=True) timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)] - assert all([ts.tz is None for ts in timestmaps]) + assert all(ts.tz is None for ts in timestmaps) def test_clipping(self): df = pd.DataFrame( diff --git a/tests/incite/schemas/test_thl_web.py b/tests/incite/schemas/test_thl_web.py index 7f4434b..9b34ce0 100644 --- a/tests/incite/schemas/test_thl_web.py +++ b/tests/incite/schemas/test_thl_web.py @@ -16,7 +16,7 @@ class TestWallSchema: df = pd.DataFrame(columns=THLWallSchema.columns.keys()) - with pytest.raises(SchemaError) as cm: + with pytest.raises(SchemaError): THLWallSchema.validate(df) def test_no_rows(self): @@ -24,7 +24,7 @@ class TestWallSchema: df = pd.DataFrame(index=["uuid"], columns=THLWallSchema.columns.keys()) - with pytest.raises(SchemaError) as cm: + with pytest.raises(SchemaError): THLWallSchema.validate(df) def test_new_empty_df(self): @@ -50,7 +50,7 @@ class TestSessionSchema: df = pd.DataFrame(columns=THLSessionSchema.columns.keys()) df.set_index("uuid", inplace=True) - with pytest.raises(SchemaError) as cm: + with pytest.raises(SchemaError): THLSessionSchema.validate(df) def test_no_rows(self): @@ -58,7 +58,7 @@ class TestSessionSchema: df = pd.DataFrame(index=["id"], columns=THLSessionSchema.columns.keys()) - with pytest.raises(SchemaError) as cm: + with pytest.raises(SchemaError): THLSessionSchema.validate(df) def test_new_empty_df(self): diff --git a/tests/incite/test_collection_base.py b/tests/incite/test_collection_base.py index 7e1577a..d6ce2b1 100644 --- a/tests/incite/test_collection_base.py +++ b/tests/incite/test_collection_base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import UTC, datetime, timedelta, timezone from os.path import exists as pexists from os.path import join as pjoin @@ -9,7 +11,7 @@ import pandas as pd import pytest from _pytest._code.code import ExceptionInfo -from generalresearch.incite.base import CollectionBase +from generalresearch.incite.base import CollectionBase, GRLDatasets AGO_15min = (datetime.now(tz=UTC) - timedelta(minutes=15)).replace(microsecond=0) AGO_1HR = (datetime.now(tz=UTC) - timedelta(hours=1)).replace(microsecond=0) @@ -17,11 +19,11 @@ AGO_2HR = (datetime.now(tz=UTC) - timedelta(hours=2)).replace(microsecond=0) class TestCollectionBase: - def test_init(self, mnt_filepath): + def test_init(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) assert instance.df.empty is True - def test_init_df(self, mnt_filepath): + def test_init_df(self, mnt_filepath: GRLDatasets): # Only an empty pd.DataFrame can ever be provided instance = CollectionBase( df=pd.DataFrame({}), archive_path=mnt_filepath.data_src @@ -43,7 +45,7 @@ class TestCollectionBase: ) assert "Do not provide a pd.DataFrame" in str(cm.value) - def test_init_start(self, mnt_filepath): + def test_init_start(self, mnt_filepath: GRLDatasets): with pytest.raises(expected_exception=ValueError) as cm: cm: ExceptionInfo CollectionBase( @@ -74,7 +76,7 @@ class TestCollectionBase: cm.value ) - def test_init_archive_path(self, mnt_filepath): + def test_init_archive_path(self, mnt_filepath: GRLDatasets): """DirectoryPath is apparently smart enough to confirm that the directory path exists. """ @@ -99,7 +101,7 @@ class TestCollectionBase: CollectionBase(archive_path=new_path) assert "Path does not point to a directory" in str(cm.value) - def test_init_offset(self, mnt_filepath): + def test_init_offset(self, mnt_filepath: GRLDatasets): with pytest.raises(expected_exception=ValueError) as cm: cm: ExceptionInfo CollectionBase(offset="1:X", archive_path=mnt_filepath.data_src) @@ -118,14 +120,14 @@ class TestCollectionBase: class TestCollectionBaseProperties: - def test_items(self, mnt_filepath): + def test_items(self, mnt_filepath: GRLDatasets): with pytest.raises(expected_exception=NotImplementedError) as cm: cm: ExceptionInfo instance = CollectionBase(archive_path=mnt_filepath.data_src) - x = instance.items + _ = instance.items assert "Must override" in str(cm.value) - def test_interval_range(self, mnt_filepath): + def test_interval_range(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) # Private method requires the end parameter with pytest.raises(expected_exception=AssertionError) as cm: @@ -147,7 +149,7 @@ class TestCollectionBaseProperties: assert res.is_monotonic_increasing assert res.is_unique - def test_interval_range2(self, mnt_filepath): + def test_interval_range2(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) assert isinstance(instance.interval_range, list) @@ -166,16 +168,16 @@ class TestCollectionBaseProperties: ) assert len(instance.interval_range) == 2 - def test_progress(self, mnt_filepath): + def test_progress(self, mnt_filepath: GRLDatasets): with pytest.raises(expected_exception=NotImplementedError) as cm: cm: ExceptionInfo instance = CollectionBase( start=AGO_15min, offset="3min", archive_path=mnt_filepath.data_src ) - x = instance.progress + _ = instance.progress assert "Must override" in str(cm.value) - def test_progress2(self, mnt_filepath): + def test_progress2(self, mnt_filepath: GRLDatasets): instance = CollectionBase( start=AGO_2HR, offset="15min", @@ -184,10 +186,10 @@ class TestCollectionBaseProperties: assert instance.df.empty with pytest.raises(expected_exception=NotImplementedError) as cm: - df = instance.progress + _ = instance.progress assert "Must override" in str(cm.value) - def test_items2(self, mnt_filepath): + def test_items2(self, mnt_filepath: GRLDatasets): """There can't be a test for this because the Items need a path whic isn't possible in the generic form """ @@ -197,7 +199,7 @@ class TestCollectionBaseProperties: with pytest.raises(expected_exception=NotImplementedError) as cm: cm: ExceptionInfo - items = instance.items + _ = instance.items assert "Must override" in str(cm.value) # item = items[-3] @@ -208,19 +210,19 @@ class TestCollectionBaseProperties: # assert str(df.product_id.dtype) == "object" # assert str(ddf.product_id.dtype) == "string" - def test_items3(self, mnt_filepath): + def test_items3(self, mnt_filepath: GRLDatasets): instance = CollectionBase( start=AGO_2HR, offset="15min", archive_path=mnt_filepath.data_src, ) with pytest.raises(expected_exception=NotImplementedError) as cm: - item = instance.items[0] + _ = instance.items[0] assert "Must override" in str(cm.value) class TestCollectionBaseMethodsCleanup: - def test_fetch_force_rr_latest(self, mnt_filepath): + def test_fetch_force_rr_latest(self, mnt_filepath: GRLDatasets): coll = CollectionBase(archive_path=mnt_filepath.data_src) with pytest.raises(expected_exception=Exception) as cm: @@ -228,7 +230,7 @@ class TestCollectionBaseMethodsCleanup: coll.fetch_force_rr_latest(sources=[]) assert "Must override" in str(cm.value) - def test_fetch_all_paths(self, mnt_filepath): + def test_fetch_all_paths(self, mnt_filepath: GRLDatasets): coll = CollectionBase(archive_path=mnt_filepath.data_src) with pytest.raises(expected_exception=NotImplementedError) as cm: @@ -242,16 +244,16 @@ class TestCollectionBaseMethodsCleanup: class TestCollectionBaseMethodsCleanup: @pytest.mark.skip - def test_cleanup_partials(self, mnt_filepath): + def test_cleanup_partials(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) assert instance.cleanup_partials() is None # it doesn't return anything - def test_clear_tmp_archives(self, mnt_filepath): + def test_clear_tmp_archives(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) assert instance.clear_tmp_archives() is None # it doesn't return anything @pytest.mark.skip - def test_clear_corrupt_archives(self, mnt_filepath): + def test_clear_corrupt_archives(self, mnt_filepath: GRLDatasets): """TODO: expand this so it actually has corrupt archives that we check to see if they're removed """ @@ -259,14 +261,14 @@ class TestCollectionBaseMethodsCleanup: assert instance.clear_corrupt_archives() is None # it doesn't return anything @pytest.mark.skip - def test_rebuild_symlinks(self, mnt_filepath): + def test_rebuild_symlinks(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) assert instance.rebuild_symlinks() is None class TestCollectionBaseMethodsSourceTiming: - def test_get_item(self, mnt_filepath): + def test_get_item(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) i = pd.Interval(left=1, right=2, closed="left") @@ -274,7 +276,7 @@ class TestCollectionBaseMethodsSourceTiming: instance.get_item(interval=i) assert "Must override" in str(cm.value) - def test_get_item_start(self, mnt_filepath): + def test_get_item_start(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) dt = datetime.now(tz=UTC) @@ -284,7 +286,7 @@ class TestCollectionBaseMethodsSourceTiming: instance.get_item_start(start=start) assert "Must override" in str(cm.value) - def test_get_items(self, mnt_filepath): + def test_get_items(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) dt = datetime.now(tz=UTC) @@ -293,21 +295,21 @@ class TestCollectionBaseMethodsSourceTiming: instance.get_items(since=dt) assert "Must override" in str(cm.value) - def test_get_items_from_year(self, mnt_filepath): + def test_get_items_from_year(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) with pytest.raises(expected_exception=NotImplementedError) as cm: instance.get_items_from_year(year=2020) assert "Must override" in str(cm.value) - def test_get_items_last90(self, mnt_filepath): + def test_get_items_last90(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) with pytest.raises(expected_exception=NotImplementedError) as cm: instance.get_items_last90() assert "Must override" in str(cm.value) - def test_get_items_last365(self, mnt_filepath): + def test_get_items_last365(self, mnt_filepath: GRLDatasets): instance = CollectionBase(archive_path=mnt_filepath.data_src) with pytest.raises(expected_exception=NotImplementedError) as cm: diff --git a/tests/incite/test_collection_base_item.py b/tests/incite/test_collection_base_item.py index 7a0a581..e09f54a 100644 --- a/tests/incite/test_collection_base_item.py +++ b/tests/incite/test_collection_base_item.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import UTC, datetime from os.path import join as pjoin from pathlib import Path @@ -8,7 +10,7 @@ import pandas as pd import pytest from pydantic import ValidationError -from generalresearch.incite.base import CollectionItemBase +from generalresearch.incite.base import CollectionItemBase, GRLDatasets class TestCollectionItemBase: @@ -40,20 +42,20 @@ class TestCollectionItemBaseProperties: def test_finish(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.finish + with pytest.raises(expected_exception=AttributeError): + _ = instance.finish def test_interval(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.interval + with pytest.raises(expected_exception=AttributeError): + _ = instance.interval def test_filename(self): instance = CollectionItemBase() with pytest.raises(expected_exception=NotImplementedError) as cm: - res = instance.filename + _ = instance.filename assert "Do not use CollectionItemBase directly" in str(cm.value) @@ -61,7 +63,7 @@ class TestCollectionItemBaseProperties: instance = CollectionItemBase() with pytest.raises(expected_exception=NotImplementedError) as cm: - res = instance.filename + _ = instance.filename assert "Do not use CollectionItemBase directly" in str(cm.value) @@ -69,27 +71,27 @@ class TestCollectionItemBaseProperties: instance = CollectionItemBase() with pytest.raises(expected_exception=NotImplementedError) as cm: - res = instance.filename + _ = instance.filename assert "Do not use CollectionItemBase directly" in str(cm.value) def test_path(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.path + with pytest.raises(expected_exception=AttributeError): + _ = instance.path def test_partial_path(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.partial_path + with pytest.raises(expected_exception=AttributeError): + _ = instance.partial_path def test_empty_path(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.empty_path + with pytest.raises(expected_exception=AttributeError): + _ = instance.empty_path class TestCollectionItemBaseMethods: @@ -106,41 +108,41 @@ class TestCollectionItemBaseMethods: instance = CollectionItemBase() with pytest.raises(expected_exception=NotImplementedError) as cm: - res = instance.tmp_filename() + _ = instance.tmp_filename() assert "Do not use CollectionItemBase directly" in str(cm.value) def test_tmp_path(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.tmp_path() + with pytest.raises(expected_exception=AttributeError): + instance.tmp_path() def test_is_empty(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.is_empty() + with pytest.raises(expected_exception=AttributeError): + instance.is_empty() def test_has_empty(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.has_empty() + with pytest.raises(expected_exception=AttributeError): + instance.has_empty() def test_has_partial_archive(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.has_partial_archive() + with pytest.raises(expected_exception=AttributeError): + instance.has_partial_archive() @pytest.mark.parametrize("include_empty", [True, False]) - def test_has_archive(self, include_empty): + def test_has_archive(self, include_empty: bool): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.has_archive(include_empty=include_empty) + with pytest.raises(expected_exception=AttributeError): + instance.has_archive(include_empty=include_empty) - def test_delete_archive_file(self, mnt_filepath): + def test_delete_archive_file(self, mnt_filepath: GRLDatasets): path1 = Path(pjoin(mnt_filepath.data_src, f"{uuid4().hex}.zip")) # Confirm it doesn't exist, and that delete_archive() doesn't throw @@ -155,7 +157,7 @@ class TestCollectionItemBaseMethods: CollectionItemBase.delete_archive(generic_path=path1) assert not path1.exists() - def test_delete_archive_dir(self, mnt_filepath): + def test_delete_archive_dir(self, mnt_filepath: GRLDatasets): path1 = Path(pjoin(mnt_filepath.data_src, f"{uuid4().hex}")) # Confirm it doesn't exist, and that delete_archive() doesn't throw @@ -174,20 +176,20 @@ class TestCollectionItemBaseMethods: def test_should_archive(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.should_archive() + with pytest.raises(expected_exception=AttributeError): + _ = instance.should_archive() def test_set_empty(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.set_empty() + with pytest.raises(expected_exception=AttributeError): + _ = instance.set_empty() def test_valid_archive(self): instance = CollectionItemBase() - with pytest.raises(expected_exception=AttributeError) as cm: - res = instance.valid_archive(generic_path=None, sample=None) + with pytest.raises(expected_exception=AttributeError): + _ = instance.valid_archive(generic_path=None, sample=None) class TestCollectionItemBaseMethodsORM: @@ -197,11 +199,11 @@ class TestCollectionItemBaseMethodsORM: pass @pytest.mark.parametrize("is_partial", [True, False]) - def test_to_archive(self, is_partial): + def test_to_archive(self, is_partial: bool): instance = CollectionItemBase() with pytest.raises(expected_exception=NotImplementedError) as cm: - res = instance.to_archive( + _ = instance.to_archive( ddf=dd.from_pandas(data=pd.DataFrame()), is_partial=is_partial ) assert "Must override" in str(cm.value) diff --git a/tests/incite/test_interval_idx.py b/tests/incite/test_interval_idx.py index 3034c21..03d29ea 100644 --- a/tests/incite/test_interval_idx.py +++ b/tests/incite/test_interval_idx.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import UTC, datetime import pandas as pd @@ -6,8 +6,8 @@ import pandas as pd class TestIntervalIndex: def test_init(self): - start = datetime(year=2000, month=1, day=1) - end = datetime(year=2000, month=1, day=10) + start = datetime(year=2000, month=1, day=1, tzinfo=UTC) + end = datetime(year=2000, month=1, day=10, tzinfo=UTC) iv_r: pd.IntervalIndex = pd.interval_range( start=start, end=end, freq="1d", closed="left" diff --git a/tests/managers/gr/test_business.py b/tests/managers/gr/test_business.py index 3490403..ed141b1 100644 --- a/tests/managers/gr/test_business.py +++ b/tests/managers/gr/test_business.py @@ -2,17 +2,36 @@ from uuid import uuid4 import pytest +from generalresearch.managers.gr.business import ( + BusinessAddressManager, + BusinessBankAccountManager, + BusinessManager, +) +from generalresearch.managers.gr.team import MembershipManager, TeamManager +from generalresearch.models.gr.authentication import GRUser +from generalresearch.models.gr.business import ( + Business, + BusinessAddress, + BusinessBankAccount, + TransferMethod, +) +from generalresearch.pg_helper import PostgresConfig + class TestBusinessBankAccountManager: - def test_init(self, business_bank_account_manager, gr_db): + def test_init( + self, + business_bank_account_manager: BusinessBankAccountManager, + gr_db: PostgresConfig, + ): assert business_bank_account_manager.pg_config == gr_db - def test_create(self, business: Business, business_bank_account_manager): - from generalresearch.models.gr.business import ( - BusinessBankAccount, - TransferMethod, - ) + def test_create( + self, + business: Business, + business_bank_account_manager: BusinessBankAccountManager, + ): instance = business_bank_account_manager.create( business_id=business.id, @@ -33,8 +52,9 @@ class TestBusinessBankAccountManager: class TestBusinessAddressManager: - def test_create(self, business: Business, business_address_manager): - from generalresearch.models.gr.business import BusinessAddress + def test_create( + self, business: Business, business_address_manager: BusinessAddressManager + ): res = business_address_manager.create(uuid=uuid4().hex, business_id=business.id) assert isinstance(res, BusinessAddress) @@ -43,14 +63,13 @@ class TestBusinessAddressManager: class TestBusinessManager: - def test_create(self, business_manager): - from generalresearch.models.gr.business import Business + def test_create(self, business_manager: BusinessManager): instance = business_manager.create_dummy() assert isinstance(instance, Business) assert isinstance(instance.id, int) - def test_get_or_create(self, business_manager): + def test_get_or_create(self, business_manager: BusinessManager): uuid_key = uuid4().hex assert business_manager.get_by_uuid(business_uuid=uuid_key) is None @@ -61,9 +80,10 @@ class TestBusinessManager: ) res = business_manager.get_by_uuid(business_uuid=uuid_key) + assert isinstance(res, Business) assert res.id == instance.id - def test_get_all(self, business_manager): + def test_get_all(self, business_manager: BusinessManager): res1 = business_manager.get_all() assert isinstance(res1, list) @@ -76,7 +96,11 @@ class TestBusinessManager: pass def test_get_by_user_id( - self, business_manager, gr_user, team_manager, membership_manager + self, + business_manager: BusinessManager, + gr_user: GRUser, + team_manager: TeamManager, + membership_manager: MembershipManager, ): res = business_manager.get_by_user_id(user_id=gr_user.id) assert len(res) == 0 @@ -93,7 +117,7 @@ class TestBusinessManager: # Create a Membership for the gr_user to the Team... but it doesn't # matter because the Team doesn't have any Business yet - m1 = membership_manager.create(team=t1, gr_user=gr_user) + _ = membership_manager.create(team=t1, gr_user=gr_user) res = business_manager.get_by_user_id(user_id=gr_user.id) assert len(res) == 0 @@ -113,15 +137,17 @@ class TestBusinessManager: def test_get_uuids_by_user_id(self): pass - def test_get_by_uuid(self, business: Business, business_manager): + def test_get_by_uuid(self, business: Business, business_manager: BusinessManager): instance = business_manager.get_by_uuid(business_uuid=business.uuid) + assert isinstance(instance, Business) assert business.id == instance.id - def test_get_by_id(self, business: Business, business_manager): + def test_get_by_id(self, business: Business, business_manager: BusinessManager): instance = business_manager.get_by_id(business_id=business.id) + assert isinstance(instance, Business) assert business.uuid == instance.uuid - def test_cache_key(self, business): + def test_cache_key(self, business: Business): assert "business:" in business.cache_key # def test_create_raise_on_duplicate(self): diff --git a/tests/managers/gr/test_team.py b/tests/managers/gr/test_team.py index 5e5c565..ae3e1bb 100644 --- a/tests/managers/gr/test_team.py +++ b/tests/managers/gr/test_team.py @@ -1,18 +1,29 @@ +from __future__ import annotations + +from collections.abc import Callable from uuid import uuid4 +from generalresearch.managers.gr.authentication import GRUserManager +from generalresearch.managers.gr.team import MembershipManager, TeamManager +from generalresearch.models.gr.authentication import GRUser +from generalresearch.models.gr.team import Membership, Team +from generalresearch.models.thl.product import Product +from generalresearch.pg_helper import PostgresConfig +from generalresearch.redis_helper import RedisConfig + class TestMembershipManager: - def test_init(self, membership_manager, gr_db): + def test_init(self, membership_manager: MembershipManager, gr_db: PostgresConfig): assert membership_manager.pg_config == gr_db class TestTeamManager: - def test_init(self, team_manager, gr_db): + def test_init(self, team_manager: TeamManager, gr_db: PostgresConfig): assert team_manager.pg_config == gr_db - def test_get_or_create(self, team_manager): + def test_get_or_create(self, team_manager: TeamManager): from generalresearch.models.gr.team import Team new_uuid = uuid4().hex @@ -24,7 +35,7 @@ class TestTeamManager: assert team.uuid == new_uuid assert team.name == "< Unknown >" - def test_get_all(self, team_manager): + def test_get_all(self, team_manager: TeamManager): res1 = team_manager.get_all() assert isinstance(res1, list) @@ -32,16 +43,20 @@ class TestTeamManager: res2 = team_manager.get_all() assert len(res1) == len(res2) - 1 - def test_create(self, team_manager): - from generalresearch.models.gr.team import Team + def test_create(self, team_manager: TeamManager): team: Team = team_manager.create_dummy() assert isinstance(team, Team) assert isinstance(team.id, int) - def test_add_user(self, team, team_manager, gr_um, gr_db, gr_redis_config): - from generalresearch.models.gr.authentication import GRUser - from generalresearch.models.gr.team import Membership + def test_add_user( + self, + team: Team, + team_manager: TeamManager, + gr_um: GRUserManager, + gr_db: PostgresConfig, + gr_redis_config: RedisConfig, + ): user: GRUser = gr_um.create_dummy() @@ -54,25 +69,23 @@ class TestTeamManager: assert len(team.gr_users) assert team.gr_users == [user] - def test_get_by_uuid(self, team_manager): - from generalresearch.models.gr.team import Team + def test_get_by_uuid(self, team_manager: TeamManager): team: Team = team_manager.create_dummy() instance = team_manager.get_by_uuid(team_uuid=team.uuid) assert team.id == instance.id - def test_get_by_id(self, team_manager): - from generalresearch.models.gr.team import Team + def test_get_by_id(self, team_manager: TeamManager): team: Team = team_manager.create_dummy() instance = team_manager.get_by_id(team_id=team.id) assert team.uuid == instance.uuid - def test_get_by_user(self, team, team_manager, gr_um): - from generalresearch.models.gr.authentication import GRUser - from generalresearch.models.gr.team import Team + def test_get_by_user( + self, team: Team, team_manager: TeamManager, gr_um: GRUserManager + ): user: GRUser = gr_um.create_dummy() team_manager.add_user(team=team, gr_user=user) @@ -86,15 +99,12 @@ class TestTeamManager: def test_get_by_user_duplicates( self, - gr_user_token, - gr_user, - membership, + gr_user: GRUser, product_factory: Callable[..., Product], - membership_factory, - team, - thl_web_rr: PostgresConfig, - gr_redis_config, - gr_db, + membership_factory: Callable[..., Membership], + team: Team, + gr_redis_config: RedisConfig, + gr_db: PostgresConfig, ): product_factory(team=team) membership_factory(team=team, gr_user=gr_user) diff --git a/tests/managers/network/test_label.py b/tests/managers/network/test_label.py index bfc7518..71efa95 100644 --- a/tests/managers/network/test_label.py +++ b/tests/managers/network/test_label.py @@ -27,7 +27,7 @@ def ip_label(utc_now) -> IPLabel: provider="GeoNodE", created_at=utc_now, ip=ip, - metadata=IPLabelMetadata(services=["RDP"]) + metadata=IPLabelMetadata(services=["RDP"]), ) @@ -181,7 +181,7 @@ def test_label_cidr_and_ipinfo( ip = fake.ipv6() ip_information_factory(ip=ip, geoname=ip_geoname) # We normalize for storage into ipinfo table - ip_norm, prefix = normalize_ip(ip) + ip_norm, _ = normalize_ip(ip) # Test with a larger network ip_48 = ipaddress.IPv6Network((ip, 48), strict=False) diff --git a/tests/managers/thl/test_contest/test_leaderboard.py b/tests/managers/thl/test_contest/test_leaderboard.py index 07d8d74..3a63075 100644 --- a/tests/managers/thl/test_contest/test_leaderboard.py +++ b/tests/managers/thl/test_contest/test_leaderboard.py @@ -5,7 +5,6 @@ from zoneinfo import ZoneInfo from generalresearch.currency import USDCent from generalresearch.managers.thl.contest_manager import ContestManager -from generalresearch.managers.thl.ledger_manager.ledger import LedgerManager from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.managers.thl.user_manager.user_manager import UserManager from generalresearch.models.thl.contest.definitions import ( @@ -116,7 +115,9 @@ class TestLeaderboardContestCRUD: assert decision assert reason == ContestEndReason.ENDS_AT - contest_manager.end_contest_if_over(contest=contest, ledger_manager=thl_lm) + contest_manager.end_contest_if_over( + contest=contest, ledger_manager=thl_ledger_manager + ) c: LeaderboardContest = contest_manager.get(contest_uuid=contest.uuid) assert c.status == ContestStatus.COMPLETED diff --git a/tests/managers/thl/test_contest/test_milestone.py b/tests/managers/thl/test_contest/test_milestone.py index a2d575b..e3889bc 100644 --- a/tests/managers/thl/test_contest/test_milestone.py +++ b/tests/managers/thl/test_contest/test_milestone.py @@ -292,7 +292,10 @@ class TestMilestoneContestUserViews: assert len(cs) == 1 contest_manager.enter_milestone_contest( - contest_uuid=c.uuid, user=user, country_iso="us", ledger_manager=thl_lm + contest_uuid=c.uuid, + user=user, + country_iso="us", + ledger_manager=thl_ledger_manager, ) # User isn't eligible anymore diff --git a/tests/managers/thl/test_contest/test_raffle.py b/tests/managers/thl/test_contest/test_raffle.py index b435576..06d4676 100644 --- a/tests/managers/thl/test_contest/test_raffle.py +++ b/tests/managers/thl/test_contest/test_raffle.py @@ -14,6 +14,7 @@ from generalresearch.managers.thl.ledger_manager.exceptions import ( ) from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager from generalresearch.models.thl.contest import ( + Contest, ContestEndCondition, ContestEntryRule, ContestPrize, @@ -40,8 +41,6 @@ class TestRaffleContest: def test_should_end( self, contest: RaffleContest, - thl_ledger_manager: ThlLedgerManager, - contest_manager: ContestManager, ): # contest is active and has no entries should, msg = contest.should_end() @@ -67,7 +66,6 @@ class TestRaffleContestCRUD: self, contest_create: RaffleContestCreate, product_user_wallet_yes: Product, - thl_ledger_manager: ThlLedgerManager, contest_manager: ContestManager, ): c = contest_manager.create( @@ -329,7 +327,7 @@ class TestRaffleContestCRUD: contest_uuid=c.uuid, entry=entry, country_iso="us", - ledger_manager=thl_lm, + ledger_manager=thl_ledger_manager, ) assert "Entry would exceed max amount per user." in str(e.value) @@ -342,7 +340,7 @@ class TestRaffleContestCRUD: contest_uuid=c.uuid, entry=entry, country_iso="us", - ledger_manager=thl_lm, + ledger_manager=thl_ledger_manager, ) assert "Entry would exceed max amount per user per day." in str(e.value) @@ -354,7 +352,7 @@ class TestRaffleContestCRUD: contest_uuid=c.uuid, entry=entry, country_iso="us", - ledger_manager=thl_lm, + ledger_manager=thl_ledger_manager, ) # Then can't anymore @@ -366,7 +364,7 @@ class TestRaffleContestCRUD: contest_uuid=c.uuid, entry=entry, country_iso="us", - ledger_manager=thl_lm, + ledger_manager=thl_ledger_manager, ) assert "Entry would exceed max amount per user per day." in str(e.value) diff --git a/tests/managers/thl/test_ledger/test_lm_accounts.py b/tests/managers/thl/test_ledger/test_lm_accounts.py index 540bea8..7b65b2d 100644 --- a/tests/managers/thl/test_ledger/test_lm_accounts.py +++ b/tests/managers/thl/test_ledger/test_lm_accounts.py @@ -63,7 +63,7 @@ class TestLedgerAccountManagerNoResults: acct_id: UUIDStr, lm: LedgerManager, ): - qn = ":".join([currency, kind, acct_id]) + qn = f"{currency}:{kind}:{acct_id}" # (1) .get_many_ assert lm.get_account_many_(qualified_names=[qn], raise_on_error=False) == [] diff --git a/tests/managers/thl/test_ledger/test_lm_tx.py b/tests/managers/thl/test_ledger/test_lm_tx.py index 13495a7..ce609d6 100644 --- a/tests/managers/thl/test_ledger/test_lm_tx.py +++ b/tests/managers/thl/test_ledger/test_lm_tx.py @@ -24,8 +24,6 @@ class TestLedgerManagerCreateTx: """Confirm that the Permission values that are set on the Ledger Manger allow the Creation action to occur. """ - acct_uuid = uuid4().hex - # (1) With no Permissions defined test_lm = LedgerManager( pg_config=ledger_manager.pg_config, @@ -44,8 +42,6 @@ class TestLedgerManagerCreateTx: def test_create_assertions( self, - ledger_account_debit: LedgerAccount, - ledger_account_credit: LedgerAccount, ledger_manager: LedgerManager, ): with pytest.raises(expected_exception=ValueError) as excinfo: diff --git a/tests/managers/thl/test_ledger/test_thl_lm_accounts.py b/tests/managers/thl/test_ledger/test_thl_lm_accounts.py index dce9116..60eb71c 100644 --- a/tests/managers/thl/test_ledger/test_thl_lm_accounts.py +++ b/tests/managers/thl/test_ledger/test_thl_lm_accounts.py @@ -229,6 +229,7 @@ class TestThlLedgerManagerAccounts: # (1) known account and confirm it comes back res = ledger_manager.get_account(qualified_name=account1.qualified_name) + assert isinstance(res, LedgerAccount) assert account1.model_dump_json() == res.model_dump_json() # (2) known accounts and confirm they both come back @@ -291,6 +292,7 @@ class TestThlLedgerManagerAccounts: assert len(res) == 2 # Confirm an empty array comes back for all unknown qualified names + assert isinstance(ledger_manager.currency, LedgerCurrency) res = ledger_manager.get_accounts_if_exists( qualified_names=[ f"{ledger_manager.currency.value}:bp_wall:{uuid4().hex}" @@ -328,7 +330,7 @@ class TestThlLedgerManagerAccounts: product_uuids=product_uuids ) assert len(res) == len(product_uuids) - assert all([isinstance(i, LedgerAccount) for i in res]) + assert all(isinstance(i, LedgerAccount) for i in res) class TestLedgerAccountManager: @@ -351,10 +353,10 @@ class TestLedgerAccountManager: # First we want to validate that using the get_account method raises # an error for a random LedgerAccount which we know does not exist. with pytest.raises(LedgerAccountDoesntExistError): - lam.get_account(qualified_name=account.qualified_name) + ledger_account_manager.get_account(qualified_name=account.qualified_name) # Now that we know it doesn't exist, get_or_create for it - instance = lam.get_account_or_create(account=account) + instance = ledger_account_manager.get_account_or_create(account=account) # It should always return assert isinstance(instance, LedgerAccount) @@ -364,10 +366,11 @@ class TestLedgerAccountManager: self, user: User, thl_ledger_manager: ThlLedgerManager, - ledger_manager: LedgerManager, ledger_account_manager: LedgerAccountManager, ): + assert isinstance(user.product, Product) + with pytest.raises(LedgerAccountDoesntExistError): ledger_account_manager.get_account( qualified_name=f"test:bp_wallet:{user.product.id}" diff --git a/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py b/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py index e4a25a3..b518453 100644 --- a/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py +++ b/tests/managers/thl/test_ledger/test_thl_lm_bp_payout.py @@ -133,16 +133,17 @@ class TestThlLedgerManagerBPPayout: ) payoutevent_uuid = uuid4().hex - with caplog.at_level(logging.INFO): - with pytest.raises(LedgerTransactionConditionFailedError): - thl_ledger_manager.create_tx_bp_payout( - user.product, - amount=USDCent(10_000), - created=now + timedelta(minutes=2), - skip_one_per_day_check=True, - skip_wallet_balance_check=False, - payoutevent_uuid=payoutevent_uuid, - ) + with caplog.at_level(logging.INFO), pytest.raises( + LedgerTransactionConditionFailedError + ): + thl_ledger_manager.create_tx_bp_payout( + user.product, + amount=USDCent(10_000), + created=now + timedelta(minutes=2), + skip_one_per_day_check=True, + skip_wallet_balance_check=False, + payoutevent_uuid=payoutevent_uuid, + ) assert "failed condition check balance:" in caplog.text thl_ledger_manager.create_tx_bp_payout( @@ -197,17 +198,18 @@ class TestThlLedgerManagerBPPayout: assert balance == int(rand_amount) * -1 # Test some basic assertions - with caplog.at_level(logging.INFO): - with pytest.raises(expected_exception=Exception): - thl_ledger_manager.create_tx_bp_payout( - product=product, - amount=rand_amount, - payoutevent_uuid=uuid4().hex, - created=datetime.now(tz=UTC), - skip_wallet_balance_check=False, - skip_one_per_day_check=False, - skip_flag_check=False, - ) + with caplog.at_level(logging.INFO), pytest.raises( + expected_exception=ValueError + ): + thl_ledger_manager.create_tx_bp_payout( + product=product, + amount=rand_amount, + payoutevent_uuid=uuid4().hex, + created=datetime.now(tz=UTC), + skip_wallet_balance_check=False, + skip_one_per_day_check=False, + skip_flag_check=False, + ) assert "failed condition check >1 tx per day" in caplog.text def test_create_tx_redis_failure( @@ -291,7 +293,7 @@ class TestThlLedgerManagerBPPayout: # Will fail due to multiple per day payoutevent_uuid2 = uuid4().hex with pytest.raises(expected_exception=Exception) as e: - tx = thl_ledger_manager.create_tx_bp_payout( + thl_ledger_manager.create_tx_bp_payout( product=product, amount=rand_amount, payoutevent_uuid=payoutevent_uuid2, @@ -348,7 +350,7 @@ class TestThlLedgerManagerBPPayout: # Create TX will fail on lock exit, after the tx was created! with pytest.raises(expected_exception=Exception) as e: - tx = thl_ledger_manager.create_tx_bp_payout( + thl_ledger_manager.create_tx_bp_payout( product=product, amount=rand_amount, payoutevent_uuid=payoutevent_uuid, @@ -384,7 +386,9 @@ class TestPayoutEventManagerBPPayout: product, rand_amount, now, direction=Direction.CREDIT ) assert thl_ledger_manager.get_account_balance(bp_wallet_account) == rand_amount - brokerage_product_payout_event_manager.set_account_lookup_table(thl_lm=thl_lm) + brokerage_product_payout_event_manager.set_account_lookup_table( + thl_lm=thl_ledger_manager + ) pe = brokerage_product_payout_event_manager.create_bp_payout_event( thl_ledger_manager=thl_ledger_manager, @@ -557,7 +561,7 @@ class TestPayoutEventManagerBPPayout: # Will fail on lock exit, after the tx was created! # But it'll see that the tx was created and so everything will be fine Lock.release = broken_release - pe = brokerage_product_payout_event_manager.create_bp_payout_event( + brokerage_product_payout_event_manager.create_bp_payout_event( thl_ledger_manager=thl_ledger_manager, product=product, created=now, diff --git a/tests/managers/thl/test_ledger/test_thl_lm_tx.py b/tests/managers/thl/test_ledger/test_thl_lm_tx.py index 89adb0b..1860d6d 100644 --- a/tests/managers/thl/test_ledger/test_thl_lm_tx.py +++ b/tests/managers/thl/test_ledger/test_thl_lm_tx.py @@ -23,7 +23,6 @@ from generalresearch.models.thl.definitions import ( WALL_ALLOWED_STATUS_STATUS_CODE, ) from generalresearch.models.thl.ledger import ( - AccountType, Direction, LedgerAccount, TransactionType, @@ -287,17 +286,18 @@ class TestThlLedgerTxManager: assert balance == int(rand_amount) * -1 # Test some basic assertions - with caplog.at_level(logging.INFO): - with pytest.raises(expected_exception=Exception): - thl_ledger_manager.create_tx_bp_payout( - product=product, - amount=rand_amount, - payoutevent_uuid=uuid4().hex, - created=datetime.now(tz=UTC), - skip_wallet_balance_check=False, - skip_one_per_day_check=False, - skip_flag_check=False, - ) + with caplog.at_level(logging.INFO), pytest.raises( + expected_exception=ValueError + ): + thl_ledger_manager.create_tx_bp_payout( + product=product, + amount=rand_amount, + payoutevent_uuid=uuid4().hex, + created=datetime.now(tz=UTC), + skip_wallet_balance_check=False, + skip_one_per_day_check=False, + skip_flag_check=False, + ) assert "failed condition check >1 tx per day" in caplog.text def test_create_tx_bp_payout_( @@ -1794,7 +1794,7 @@ class TestThlLedgerManagerAdj: ) thl_ledger_manager.create_tx_bp_payment(session, created=wall1.started) - revenue = ththl_ledger_managerl_lm.get_account_task_complete_revenue() + revenue = thl_ledger_manager.get_account_task_complete_revenue() bp_wallet_account = thl_ledger_manager.get_account_or_create_bp_wallet( user.product ) diff --git a/tests/managers/thl/test_payout.py b/tests/managers/thl/test_payout.py index e6c597b..0f3f103 100644 --- a/tests/managers/thl/test_payout.py +++ b/tests/managers/thl/test_payout.py @@ -727,8 +727,6 @@ class TestBusinessPayoutEventManager: # {"uuid": bp_pe.uuid, "status": PayoutStatus.FAILED}, # ) - assert 1 == 0 - def test_ach_payment( self, mnt_filepath: GRLDatasets, diff --git a/tests/managers/thl/test_survey.py b/tests/managers/thl/test_survey.py index 2c2bf9d..c3ab162 100644 --- a/tests/managers/thl/test_survey.py +++ b/tests/managers/thl/test_survey.py @@ -11,9 +11,6 @@ from generalresearch.managers.thl.buyer import BuyerManager from generalresearch.managers.thl.profiling.question import ( QuestionManager, ) -from generalresearch.managers.thl.profiling.schema import ( - UpkSchemaManager, -) from generalresearch.managers.thl.profiling.uqa import UQAManager from generalresearch.managers.thl.survey import SurveyManager, SurveyStatManager from generalresearch.models import Source @@ -183,7 +180,8 @@ class TestSurvey: ] uqad = {} for uqa in uqas: - for k, _ in uqa.calc_answers.items(): + assert uqa.calc_answers + for k in uqa.calc_answers: if k in qualifying_questions: uqad[k] = uqa uqad[uqa.property_code] = uqa diff --git a/tests/managers/thl/test_user_manager/test_base.py b/tests/managers/thl/test_user_manager/test_base.py index 5822207..8cd83ad 100644 --- a/tests/managers/thl/test_user_manager/test_base.py +++ b/tests/managers/thl/test_user_manager/test_base.py @@ -21,7 +21,7 @@ from generalresearch.managers.thl.user_manager.user_manager import ( UserManager, ) from generalresearch.managers.thl.userhealth import AuditLogManager -from generalresearch.models.thl.product import Product, UserCreateConfig, product +from generalresearch.models.thl.product import Product, UserCreateConfig from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig diff --git a/tests/models/network/test_nmap.py b/tests/models/network/test_nmap.py index 5e9f4d0..db39997 100644 --- a/tests/models/network/test_nmap.py +++ b/tests/models/network/test_nmap.py @@ -8,7 +8,7 @@ from generalresearch.managers.network.tool_run import ToolRunManager from generalresearch.models.network.definitions import IPProtocol from generalresearch.models.network.nmap.execute import execute_nmap from generalresearch.models.network.nmap.result import NmapResult, PortState -from generalresearch.models.network.tool_run import NmapRun, Status, ToolClass, ToolName +from generalresearch.models.network.tool_run import NmapRun, ToolClass, ToolName fake = faker.Faker() diff --git a/tests/models/spectrum/test_survey.py b/tests/models/spectrum/test_survey.py index 7ddd407..f97860f 100644 --- a/tests/models/spectrum/test_survey.py +++ b/tests/models/spectrum/test_survey.py @@ -405,3 +405,47 @@ class TestSpectrumSurvey: assert (None, {"c", "d"}) == s.determine_eligibility_soft( {"a": True, "b": True, "c": None, "d": None} ) + + +def test_spectrum_something(spectrum_api_surveys_json: list[str]): + # make sure hashes for 111111 are in db + c1 = SpectrumCondition( + question_id="1001", + value_type=ConditionValueType.LIST, + values=["a", "b", "c"], + negate=False, + logical_operator=LogicalOperator.OR, + ) + c2 = SpectrumCondition( + question_id="1001", + value_type=ConditionValueType.LIST, + values=["a"], + negate=False, + logical_operator=LogicalOperator.OR, + ) + c3 = SpectrumCondition( + question_id="1002", + value_type=ConditionValueType.RANGE, + values=["18-24", "30-32"], + negate=False, + logical_operator=LogicalOperator.OR, + ) + c4 = SpectrumCondition( + question_id="212", + value_type=ConditionValueType.LIST, + values=["23", "24"], + negate=False, + logical_operator=LogicalOperator.OR, + ) + c5 = SpectrumCondition( + question_id="1031", + value_type=ConditionValueType.LIST, + values=["113", "114", "121"], + negate=False, + logical_operator=LogicalOperator.OR, + ) + _conditions = [c1, c2, c3, c4, c5] + + survey = SpectrumSurvey.model_validate_json(spectrum_api_surveys_json[0]) + assert c1.criterion_hash in survey.qualifications + assert c3.criterion_hash in survey.qualifications diff --git a/tests/models/thl/test_product.py b/tests/models/thl/test_product.py index adf276d..880799a 100644 --- a/tests/models/thl/test_product.py +++ b/tests/models/thl/test_product.py @@ -1013,7 +1013,7 @@ class TestProductCache: assert res is None with pytest.raises(expected_exception=AssertionError): product.set_cache( - thl_lm=thl_lm, + thl_lm=thl_ledger_manager, ds=mnt_filepath, client=client_no_amm, bp_pem=brokerage_product_payout_event_manager, @@ -1035,7 +1035,7 @@ class TestProductCache: # Now try again with everything in place product.set_cache( - thl_lm=thl_lm, + thl_lm=thl_ledger_manager, ds=mnt_filepath, client=client_no_amm, bp_pem=brokerage_product_payout_event_manager, -- cgit v1.2.3 From dda0067fdfe563e8270562f2e0c9da7c74c37a0b Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Sat, 29 Aug 2026 22:00:43 -0700 Subject: pylint ⭕️ cyclic-import checks --- generalresearch/config.py | 5 +- generalresearch/grliq/managers/__init__.py | 34 - generalresearch/grliq/models/forensic_summary.py | 6 +- generalresearch/incite/__init__.py | 4 + generalresearch/incite/base.py | 10 +- generalresearch/incite/collections/__init__.py | 692 -------------------- generalresearch/incite/collections/base.py | 693 +++++++++++++++++++++ .../incite/collections/thl_marketplaces.py | 2 +- generalresearch/incite/defaults.py | 10 +- generalresearch/incite/mergers/__init__.py | 301 --------- generalresearch/incite/mergers/base.py | 301 +++++++++ .../incite/mergers/foundations/enriched_session.py | 2 +- .../mergers/foundations/enriched_task_adjust.py | 2 +- .../incite/mergers/foundations/enriched_wall.py | 4 +- .../incite/mergers/foundations/user_id_product.py | 2 +- generalresearch/incite/mergers/pop_ledger.py | 2 +- generalresearch/incite/mergers/ym_survey_wall.py | 2 +- generalresearch/incite/mergers/ym_wall_summary.py | 2 +- generalresearch/incite/schemas/thl_web.py | 4 +- generalresearch/models/network/label.py | 1 + generalresearch/models/thl/definitions.py | 2 +- generalresearch/models/thl/ipinfo.py | 22 +- generalresearch/models/thl/ledger.py | 63 +- generalresearch/models/thl/ledger_example.py | 64 -- generalresearch/models/thl/maxmind/__init__.py | 0 generalresearch/models/thl/maxmind/definitions.py | 22 - generalresearch/models/thl/session.py | 25 +- generalresearch/models/thl/user_iphistory.py | 9 +- pyproject.toml | 6 +- test_utils/grliq/conftest.py | 50 +- test_utils/incite/mergers/conftest.py | 2 +- 31 files changed, 1161 insertions(+), 1183 deletions(-) create mode 100644 generalresearch/incite/collections/base.py create mode 100644 generalresearch/incite/mergers/base.py delete mode 100644 generalresearch/models/thl/ledger_example.py delete mode 100644 generalresearch/models/thl/maxmind/__init__.py delete mode 100644 generalresearch/models/thl/maxmind/definitions.py (limited to 'pyproject.toml') diff --git a/generalresearch/config.py b/generalresearch/config.py index c414069..73af565 100644 --- a/generalresearch/config.py +++ b/generalresearch/config.py @@ -115,9 +115,8 @@ class GRLBaseSettings(BaseSettings): amt_bonus_cashout_method_id: str | None = Field(default=None) amt_assignment_cashout_method_id: str | None = Field(default=None) - # --- Maxmind Configuration --- - maxmind_account_id: str | None = Field(default=None) - maxmind_license_key: str | None = Field(default=None) + # --- GRIP Configuration --- + grip_token: str | None = Field(default=None) EXAMPLE_PRODUCT_ID = "1108d053e4fa47c5b0dbdcd03a7981e7" diff --git a/generalresearch/grliq/managers/__init__.py b/generalresearch/grliq/managers/__init__.py index 849b6c2..e69de29 100644 --- a/generalresearch/grliq/managers/__init__.py +++ b/generalresearch/grliq/managers/__init__.py @@ -1,34 +0,0 @@ -from generalresearch.grliq.models.forensic_data import GrlIqData -from generalresearch.grliq.models.forensic_result import ( - GrlIqCheckerResults, - GrlIqForensicCategoryResult, -) - -DUMMY_GRLIQ_DATA = [ - { - "data": GrlIqData.model_validate_json( - """{"mid": "3722ed29314940fabd37b42d808dcf5a", "uuid": "b11441da5a854dfbb8401d4c32e56db5", "phase": "offerwall-enter", "events": null, "vendor": "Google Inc.", "app_name": "Netscape", "calendar": "gregory", "language": "en-US", "platform": "Linux x86_64", "timezone": "America/Mexico_City", "client_ip": "131.196.250.250", "timestamp": "2025-02-27T16:05:34-06:00", "webrtc_ip": "131.196.250.250", "created_at": "2025-02-27T22:05:35.370589Z", "language_2": "en-US", "language_3": null, "platform_2": "Linux x86_64", "platform_3": null, "prefetched": true, "product_id": "d0606a0b5d034a8d81b1e3579d1f76fd", "webgl_flag": true, "webgl_hash": "da27e1b9b660057a3f5e185d3f5deabe", "canvas_hash": "14ed764326ec454d976c322261d99f16", "color_gamut": "3", "country_iso": "mx", "inner_width": 612, "outer_width": 1813, "product_sub": "20030107", "audio_codecs": "1,1,1,1,1,3,1,3,1,3,3,1,1,3,3,3,3,1,3,3,3,2,1,1", "cookie_check": "", "graphics_api": "WebKit WebGL", "inner_height": 1174, "mouse_events": null, "ontouchstart": false, "outer_height": 1261, "plugins_hash": "4c05fa2f766a444d4f253ead792c8b0e|2", "screen_width": 2560, "video_codecs": "1,3,3,3,3,3,3,3,3,3,1,1,1,1,1,1,3,1,1,1,3,3,1", "webgl_hash_2": "fc73fd5db75e2c36222fe34251be3971", "webrtc_error": false, "window_opera": false, "battery_level": 0.9, "canvas_hash_2": "bd11ebbf5c26fd20e0217820b4159752", "dynamic_range": false, "error_message": "Cannot read", "forced_colors": false, "math_result_1": "1.9275814160560204e-50", "math_result_2": "1.6182817135715877", "screen_height": 1440, "webgl_check_1": true, "webgl_context": "webgl2", "window_chrome": true, "connection_rtt": 150, "history_length": 16, "user_agent_str": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "web_sql_exists": false, "calender_locale": "en-US", "connection_type": "", "inverted_colors": true, "navigator_brave": false, "product_user_id": "d1d55df1-959e-4740-b77c-fa1f4fc457ae", "request_headers": {"host": "test", "accept": "*/*", "connection": "keep-alive", "user-agent": "python-httpx/0.27.0", "content-length": "3646", "accept-encoding": "gzip, deflate", "x-forwarded-for": "131.196.250.250"}, "timezone_offset": 360, "webrtc_local_ip": "50486637-6b64-4812-b10a-0a75337c31bd.local", "battery_charging": true, "client_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "131.196.250.250", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "mx", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "max_touch_points": 0, "numbering_system": "latn", "path_fingerprint": 3252, "prefers_contrast": "0", "rendering_engine": "WebKit", "timezone_success": "pass", "user_agent_hints": {"model": null, "brands": [{"brand": "Google Chrome", "version": "131"}, {"brand": "Chromium", "version": "131"}, {"brand": "Not_A Brand", "version": "24"}], "mobile": false, "bitness": "64", "platform": "Linux", "brands_full": [{"brand": "Google Chrome", "version": "131.0.6778.204"}, {"brand": "Chromium", "version": "131.0.6778.204"}, {"brand": "Not_A Brand", "version": "24.0.0.0"}], "architecture": "x86", "platform_version": "6.2.0"}, "user_agent_str_2": null, "webgl_extensions": "EXT_clip_control|EXT_color_buffer_float|EXT_color_buffer_half_float|EXT_conservative_depth|EXT_depth_clamp|EXT_disjoint_timer_query_webgl2|EXT_float_blend|EXT_polygon_offset_clamp|EXT_render_snorm|EXT_texture_compression_bptc|EXT_texture_compression_rgtc|EXT_texture_filter_anisotropic|EXT_texture_mirror_clamp_to_edge|EXT_texture_norm16|KHR_parallel_shader_compile|NV_shader_noperspective_interpolation|OES_draw_buffers_indexed|OES_sample_variables|OES_shader_multisample_interpolation|OES_texture_float_linear|OVR_multiview2|WEBGL_blend_func_extended|WEBGL_clip_cull_distance|WEBGL_compressed_texture_astc|WEBGL_compressed_texture_etc|WEBGL_compressed_texture_etc1|WEBGL_compressed_texture_s3tc|WEBGL_compressed_texture_s3tc_srgb|WEBGL_debug_renderer_info|WEBGL_debug_shaders|WEBGL_lose_context|WEBGL_multi_draw|WEBGL_polygon_mode|WEBGL_provoking_vertex|WEBGL_stencil_texturing", "webrtc_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "131.196.250.250", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "mx", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "chrome_extensions": "", "execution_time_ms": 371.0999999642372, "graphics_renderer": "WebGL 2.0 (OpenGL ES 3.0 Chromium)", "keyboard_detected": true, "mime_types_length": 2, "request_fs_exists": true, "audio_context_flag": "pass", "audio_context_hash": "9307303774dec3248c18a939392090da", "canvas_fingerprint": 258, "canvas_pixel_check": false, "device_pixel_ratio": 1.0, "indexedDbData_blob": true, "navigator_keys_len": 79, "no_edge_pdf_plugin": false, "screen_avail_width": 2560, "webdriver_detected": false, "window_orientation": 0, "connection_downlink": 10.0, "navigator_webdriver": false, "non_native_function": false, "screen_avail_height": 1400, "supported_fonts_str": "72|768|262144|1073741824|0|0|540672|73728|7340032|1342177280|117446656|256|16|0|543|4290797636|1677723648|4168998400|0|1048576|262144|268500994|1342177280|262144|125829376|37888000|0|435363842|0|2147483648|109543424|1880099872|268435471", "text_2d_fingerprint": "bfcce91c9e71d11af7b14dbee4c75f83", "webrtc_is_supported": "pass", "canvas_support_level": "full", "do_not_track_enabled": "1", "hardware_concurrency": 12, "keyboard_layout_size": 48, "prefers_color_scheme": false, "webgl_max_anisotropy": 16, "battery_charging_time": 0.0, "browser_by_properties": "c", "eval_to_string_length": 33, "performance_loop_time": 0.09999996423721313, "session_storage_check": "pass", "unmasked_vendor_webgl": "Google Inc. (Intel)", "hardware_concurrency_2": 12, "hardware_concurrency_3": null, "localStorage_available": true, "memory_jsHeapSizeLimit": 4294705152, "mozilla_web_app_exists": false, "navigator_deviceMemory": 8.0, "navigator_java_enabled": false, "prefers_reduced_motion": false, "storage_estimate_quota": 1178717110272, "webdriver_detected_msg": "", "window_active_x_object": false, "window_external_exists": true, "color_depth_pixel_depth": "24-24", "indexedDbData_available": true, "navigator_cookieEnabled": true, "unmasked_renderer_webgl": "ANGLE (Intel, Mesa Intel(R) Graphics (RPL-P), OpenGL 4.6)", "battery_discharging_time": 0.0, "connection_effectiveType": "4g", "non_native_function_flag": "", "speech_synthesis_voice_1": "Google Bahasa Indonesia", "window_client_information": true, "audio_compressor_reduction": 20.538288116455078, "navigator_mediaDevices_len": 3, "audio_intensity_fingerprint": 124.04347527516074, "speech_synthesis_voice_hash": "8010ee3313813de521e48e63bd5a6f13", "microsoft_credentials_exists": false, "window_installTrigger_exists": false, "speech_synthesis_voices_count": 19, "webgl_shading_language_version": "WebGL GLSL ES 3.00 (OpenGL ES GLSL ES 3.0 Chromium)", "error_message_stack_access_count": 0, "speech_synthesis_avail_voices_count": 19, "error_message_stack_access_count_worker": 0}""" - ), - "result_data": GrlIqCheckerResults.model_validate_json( - """{"uuid": "b11441da5a854dfbb8401d4c32e56db5", "check_codecs": {"score": 0}, "check_timezone": {"score": 0}, "check_timestamp": {"score": 0}, "check_user_type": {"score": 0}, "check_ip_changes": {"score": 0}, "check_ip_country": {"score": 0}, "check_environment": {"score": 0}, "check_ip_timezone": {"score": 0}, "check_isp_changes": {"score": 0}, "check_useragent_js": {"score": 0}, "check_required_fonts": {"score": 0}, "check_user_anonymous": {"score": 0}, "check_webrtc_success": {"score": 0}, "check_seen_timestamps": {"msg": "duplicate timestamp", "score": 100}, "check_country_timezone": {"score": 0}, "check_prohibited_fonts": {"score": 0}, "check_timezone_changes": {"score": 0}, "check_execution_time_ms": {"msg": "duplicate execution_time_ms", "score": 100}, "check_fingerprint_reuse": {"score": 0}, "check_fingerprint_cycling": {"score": 0}, "check_ip_webrtc_ip_detail": {"score": 0}, "check_environment_critical": {"score": 0}, "check_useragent_other_enums": {"score": 0}, "check_useragent_ip_properties": {"score": 0}, "check_useragent_data_properties": {"score": 0}, "check_useragent_device_family_brand": {"score": 0}}""" - ), - "category_result": GrlIqForensicCategoryResult.model_validate_json( - """{"uuid": "b11441da5a854dfbb8401d4c32e56db5", "is_bot": 0, "is_tampered": 100, "is_velocity": 0, "is_anonymous": 0, "suspicious_ip": 0, "is_oscillating": 0, "is_teleporting": 0, "is_inconsistent": 0, "platform_ip_inconsistent": 0}""" - ), - "fraud_score": 100, - "is_attempt_allowed": False, - }, - { - "data": GrlIqData.model_validate_json( - """{"mid": "35f6f5c30bc74ea7ac4aca7b40a02352", "uuid": "d54509f2f310499f8ab74839b10b2a41", "phase": "offerwall-enter", "events": null, "vendor": "Google Inc.", "app_name": "Netscape", "calendar": "gregory", "language": "en-US", "platform": "Linux x86_64", "timezone": "America/Los_Angeles", "client_ip": "104.9.125.144", "timestamp": "2025-02-28T11:34:39-08:00", "webrtc_ip": "172.56.209.195", "created_at": "2025-02-28T19:34:39.681872Z", "language_2": "en-US", "language_3": null, "platform_2": "Linux x86_64", "platform_3": null, "prefetched": true, "product_id": "d0606a0b5d034a8d81b1e3579d1f76fd", "webgl_flag": true, "webgl_hash": "da27e1b9b660057a3f5e185d3f5deabe", "canvas_hash": "e6e4d17da26050ce85ad00d3c6ea999e", "color_gamut": "3", "country_iso": "us", "inner_width": 841, "outer_width": 1680, "product_sub": "20030107", "audio_codecs": "1,1,1,1,1,3,1,3,1,3,3,1,1,3,3,3,3,1,3,3,3,2,1,1", "cookie_check": "", "graphics_api": "WebKit WebGL", "inner_height": 891, "mouse_events": null, "ontouchstart": false, "outer_height": 978, "plugins_hash": "4c05fa2f766a444d4f253ead792c8b0e|2", "screen_width": 1680, "video_codecs": "1,3,3,3,3,3,3,3,3,3,1,1,1,1,1,1,3,1,1,1,3,3,1", "webgl_hash_2": "fc73fd5db75e2c36222fe34251be3971", "webrtc_error": false, "window_opera": false, "battery_level": 0.41, "canvas_hash_2": "e0559d49b1864985cafc0d1c3a6b053c", "dynamic_range": false, "error_message": "Cannot read", "forced_colors": false, "math_result_1": "1.9275814160560204e-50", "math_result_2": "1.6182817135715877", "screen_height": 1050, "webgl_check_1": true, "webgl_context": "webgl2", "window_chrome": true, "connection_rtt": 100, "history_length": 11, "user_agent_str": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "web_sql_exists": false, "calender_locale": "en-US", "connection_type": "", "inverted_colors": true, "navigator_brave": false, "product_user_id": "test-unit", "request_headers": {"dnt": "1", "host": "127.0.0.1:8081", "accept": "application/json, lk/null q=0.1", "origin": "http://127.0.0.1:8080", "referer": "http://127.0.0.1:8080/", "sec-ch-ua": "\\"Google Chrome\\";v=\\"131\\", \\"Chromium\\";v=\\"131\\", \\"Not_A Brand\\";v=\\"24\\"", "connection": "keep-alive", "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "content-type": "application/json", "content-length": "3313", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "accept-encoding": "gzip, deflate, br, zstd", "accept-language": "en-US,en;q=0.9", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\\"Linux\\""}, "timezone_offset": 480, "webrtc_local_ip": "10.253.217.45,[2607:fb91:20c5:c6af:cda0:10b4:830a:a85e]", "battery_charging": false, "client_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "104.9.125.144", "isp": "AT&T Internet", "latitude": 37.3897, "city_name": "Mountain View", "longitude": -122.083, "time_zone": "America/Los_Angeles", "user_type": "residential", "country_iso": "us", "postal_code": "94041", "is_anonymous": false, "accuracy_radius": 5, "static_ip_score": 40.3, "subdivision_1_iso": "CA", "subdivision_2_iso": null, "subdivision_1_name": "California", "subdivision_2_name": null, "registered_country_iso": "us"}, "max_touch_points": 0, "numbering_system": "latn", "path_fingerprint": 3252, "prefers_contrast": "0", "rendering_engine": "WebKit", "timezone_success": "pass", "user_agent_hints": {"model": null, "brands": [{"brand": "Google Chrome", "version": "131"}, {"brand": "Chromium", "version": "131"}, {"brand": "Not_A Brand", "version": "24"}], "mobile": false, "bitness": "64", "platform": "Linux", "brands_full": [{"brand": "Google Chrome", "version": "131.0.6778.204"}, {"brand": "Chromium", "version": "131.0.6778.204"}, {"brand": "Not_A Brand", "version": "24.0.0.0"}], "architecture": "x86", "platform_version": "6.2.0"}, "user_agent_str_2": null, "webgl_extensions": "EXT_clip_control|EXT_color_buffer_float|EXT_color_buffer_half_float|EXT_conservative_depth|EXT_depth_clamp|EXT_disjoint_timer_query_webgl2|EXT_float_blend|EXT_polygon_offset_clamp|EXT_render_snorm|EXT_texture_compression_bptc|EXT_texture_compression_rgtc|EXT_texture_filter_anisotropic|EXT_texture_mirror_clamp_to_edge|EXT_texture_norm16|KHR_parallel_shader_compile|NV_shader_noperspective_interpolation|OES_draw_buffers_indexed|OES_sample_variables|OES_shader_multisample_interpolation|OES_texture_float_linear|OVR_multiview2|WEBGL_blend_func_extended|WEBGL_clip_cull_distance|WEBGL_compressed_texture_astc|WEBGL_compressed_texture_etc|WEBGL_compressed_texture_etc1|WEBGL_compressed_texture_s3tc|WEBGL_compressed_texture_s3tc_srgb|WEBGL_debug_renderer_info|WEBGL_debug_shaders|WEBGL_lose_context|WEBGL_multi_draw|WEBGL_polygon_mode|WEBGL_provoking_vertex|WEBGL_stencil_texturing", "webrtc_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "172.56.209.195", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "us", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "chrome_extensions": "", "execution_time_ms": 924.5, "graphics_renderer": "WebGL 2.0 (OpenGL ES 3.0 Chromium)", "keyboard_detected": true, "mime_types_length": 2, "request_fs_exists": true, "audio_context_flag": "pass", "audio_context_hash": "9307303774dec3248c18a939392090da", "canvas_fingerprint": 258, "canvas_pixel_check": false, "device_pixel_ratio": 1.0, "indexedDbData_blob": true, "navigator_keys_len": 79, "no_edge_pdf_plugin": false, "screen_avail_width": 1680, "webdriver_detected": false, "window_orientation": 0, "connection_downlink": 10.0, "navigator_webdriver": false, "non_native_function": false, "screen_avail_height": 1010, "supported_fonts_str": "72|17152|327680|1073741824|0|0|540736|73728|7340032|1342177280|117446657|256|16|0|262687|4290797636|1677723648|4168998400|0|1048576|262144|268500994|1342177280|262144|125829376|37888000|0|435363842|0|2147483648|109543680|1880099888|301989903", "text_2d_fingerprint": "bfcce91c9e71d11af7b14dbee4c75f83", "webrtc_is_supported": "pass", "canvas_support_level": "full", "do_not_track_enabled": "1", "hardware_concurrency": 12, "keyboard_layout_size": 48, "prefers_color_scheme": false, "webgl_max_anisotropy": 16, "battery_charging_time": 0.0, "browser_by_properties": "c", "eval_to_string_length": 33, "performance_loop_time": 0.09999999962747097, "session_storage_check": "pass", "unmasked_vendor_webgl": "Google Inc. (Intel)", "hardware_concurrency_2": 12, "hardware_concurrency_3": null, "localStorage_available": true, "memory_jsHeapSizeLimit": 4294705152, "mozilla_web_app_exists": false, "navigator_deviceMemory": 8.0, "navigator_java_enabled": false, "prefers_reduced_motion": false, "storage_estimate_quota": 1178717110272, "webdriver_detected_msg": "", "window_active_x_object": false, "window_external_exists": true, "color_depth_pixel_depth": "24-24", "indexedDbData_available": true, "navigator_cookieEnabled": true, "unmasked_renderer_webgl": "ANGLE (Intel, Mesa Intel(R) Graphics (RPL-P), OpenGL 4.6)", "battery_discharging_time": 4844.0, "connection_effectiveType": "4g", "non_native_function_flag": "", "speech_synthesis_voice_1": "Google Bahasa Indonesia", "window_client_information": true, "audio_compressor_reduction": 20.538288116455078, "navigator_mediaDevices_len": 8, "audio_intensity_fingerprint": 124.04347527516074, "speech_synthesis_voice_hash": "8010ee3313813de521e48e63bd5a6f13", "microsoft_credentials_exists": false, "window_installTrigger_exists": false, "speech_synthesis_voices_count": 19, "webgl_shading_language_version": "WebGL GLSL ES 3.00 (OpenGL ES GLSL ES 3.0 Chromium)", "error_message_stack_access_count": 2, "speech_synthesis_avail_voices_count": 19, "error_message_stack_access_count_worker": 2}""" - ), - "result_data": GrlIqCheckerResults.model_validate_json( - """{"uuid": "d54509f2f310499f8ab74839b10b2a41", "check_codecs": {"score": 0}, "check_timezone": {"score": 0}, "check_timestamp": {"score": 0}, "check_user_type": {"score": 0}, "check_ip_changes": {"score": 0}, "check_ip_country": {"score": 0}, "check_environment": {"msg": "error_message_stack_access_count: 2", "score": 100}, "check_ip_timezone": {"score": 0}, "check_isp_changes": {"score": 0}, "check_useragent_js": {"score": 0}, "check_required_fonts": {"score": 0}, "check_user_anonymous": {"score": 0}, "check_webrtc_success": {"score": 0}, "check_seen_timestamps": {"score": 0}, "check_country_timezone": {"score": 0}, "check_prohibited_fonts": {"score": 0}, "check_timezone_changes": {"score": 0}, "check_execution_time_ms": {"score": 0}, "check_fingerprint_reuse": {"score": 0}, "check_fingerprint_cycling": {"score": 0}, "check_ip_webrtc_ip_detail": {"score": 0}, "check_environment_critical": {"score": 0}, "check_useragent_other_enums": {"score": 0}, "check_useragent_ip_properties": {"score": 0}, "check_useragent_data_properties": {"score": 0}, "check_useragent_device_family_brand": {"score": 0}}""" - ), - "category_result": GrlIqForensicCategoryResult.model_validate_json( - """{"uuid": "d54509f2f310499f8ab74839b10b2a41", "is_bot": 0, "is_tampered": 0, "is_velocity": 0, "is_anonymous": 0, "suspicious_ip": 0, "is_oscillating": 0, "is_teleporting": 0, "is_inconsistent": 10, "platform_ip_inconsistent": 0}""" - ), - "fraud_score": 10, - "is_attempt_allowed": True, - }, -] diff --git a/generalresearch/grliq/models/forensic_summary.py b/generalresearch/grliq/models/forensic_summary.py index 6b0e065..aaefdfb 100644 --- a/generalresearch/grliq/models/forensic_summary.py +++ b/generalresearch/grliq/models/forensic_summary.py @@ -10,6 +10,7 @@ from typing import ( ) import numpy as np +from grip_client.enums import AccessType from pydantic import ( BaseModel, ConfigDict, @@ -27,7 +28,6 @@ from generalresearch.grliq.models.forensic_result import ( ) from generalresearch.models.custom_types import AwareDatetimeISO, IPvAnyAddressStr from generalresearch.models.thl.locales import CountryISO -from generalresearch.models.thl.maxmind.definitions import UserType example_rtt_percentiles = ( [133.332] @@ -185,7 +185,9 @@ class IPTimingDataSummary(BaseModel): client_ip: IPvAnyAddressStr = Field(examples=["123.123.123.123"]) country_iso: CountryISO = Field(examples=["us"]) server_location: Literal["fremont_ca"] = Field(default="fremont_ca") - user_type: UserType | None = Field(default=None, examples=[UserType.RESIDENTIAL]) + user_type: AccessType | None = Field( + default=None, examples=[AccessType.RESIDENTIAL] + ) expected_rtt_range: tuple[float, float] = Field( description="The expected rtt range for this IP (based on country_iso/user_type) to server_location", examples=[(45.193, 120.841)], diff --git a/generalresearch/incite/__init__.py b/generalresearch/incite/__init__.py index e69de29..8b60e4b 100644 --- a/generalresearch/incite/__init__.py +++ b/generalresearch/incite/__init__.py @@ -0,0 +1,4 @@ +import logging + +logging.basicConfig() +LOG = logging.getLogger(f"{__name__}.incite") diff --git a/generalresearch/incite/base.py b/generalresearch/incite/base.py index 060df4e..9dfec4f 100644 --- a/generalresearch/incite/base.py +++ b/generalresearch/incite/base.py @@ -1,7 +1,6 @@ from __future__ import annotations import glob -import logging import os import re import shutil @@ -46,7 +45,8 @@ from pydantic.json_schema import SkipJsonSchema from sentry_sdk import capture_exception from generalresearch.config import is_debug -from generalresearch.incite.collections import DFCollectionItem +from generalresearch.incite import LOG +from generalresearch.incite.collections.base import DFCollectionItem from generalresearch.incite.schemas import ( ARCHIVE_AFTER, empty_dataframe_from_schema, @@ -54,16 +54,14 @@ from generalresearch.incite.schemas import ( from generalresearch.models.custom_types import AwareDatetimeISO if TYPE_CHECKING: - from generalresearch.incite.collections import DFCollection + from generalresearch.incite.collections.base import DFCollection from generalresearch.incite.collections.thl_marketplaces import ( DFCollectionType, ) - from generalresearch.incite.mergers import MergeCollection, MergeType + from generalresearch.incite.mergers.base import MergeCollection, MergeType Collection = DFCollection | MergeCollection -logging.basicConfig() -LOG = logging.getLogger(f"{__name__}.incite") # Item = Union["DFCollectionItem", "MergeCollectionItem"] Item = Any diff --git a/generalresearch/incite/collections/__init__.py b/generalresearch/incite/collections/__init__.py index f1e1b77..e69de29 100644 --- a/generalresearch/incite/collections/__init__.py +++ b/generalresearch/incite/collections/__init__.py @@ -1,692 +0,0 @@ -from __future__ import annotations - -import os -import subprocess -import time -from datetime import datetime -from enum import StrEnum -from sys import platform -from typing import Any - -import dask -import dask.dataframe as dd -import pandas as pd -import pyarrow as pa -import pyarrow.parquet as pq -from dask.distributed import Client as DaskClient -from dask.distributed import Future -from distributed import as_completed -from more_itertools import chunked -from pandera.pandas import DataFrameSchema -from psycopg import Cursor -from pydantic import Field, FilePath, ValidationInfo, field_validator -from sentry_sdk import capture_exception - -from generalresearch.incite.base import LOG, CollectionBase, CollectionItemBase -from generalresearch.incite.schemas import ( - ARCHIVE_AFTER, - ORDER_KEY, - PARTITION_ON, - empty_dataframe_from_schema, -) -from generalresearch.incite.schemas.thl_marketplaces import ( - InnovateSurveyHistorySchema, - MorningSurveyTimeseriesSchema, - SagoSurveyHistorySchema, - SpectrumSurveyTimeseriesSchema, -) -from generalresearch.incite.schemas.thl_web import ( - LedgerSchema, - THLIPInfoSchema, - THLSessionSchema, - THLTaskAdjustmentSchema, - THLUserSchema, - THLWallSchema, - TransactionMetadataColumns, - TxMetaSchema, - TxSchema, - UserHealthAuditLogSchema, - UserHealthIPHistorySchema, - UserHealthIPHistoryWSSchema, -) -from generalresearch.pg_helper import PostgresConfig - -DT_STR = "%Y-%m-%d %H:%M:%S" - - -class DFCollectionType(StrEnum): - TEST = "test" - - USER = "thl_user" - SESSION = "thl_session" - WALL = "thl_wall" - TASK_ADJUSTMENT = "thl_taskadjustment" - IP_INFO = "thl_ipinformation" - - AUDIT_LOG = "userhealth_auditlog" - IP_HISTORY = "userhealth_iphistory" - IP_HISTORY_WS = "userhealth_iphistory_ws" - - LEDGER = "ledger" - - INNOVATE_SURVEY_HISTORY = "innovate_surveyhistory" - MORNING_SURVEY_TIMESERIES = "morning_surveytimeseries" - SAGO_SURVEY_HISTORY = "sago_surveyhistory" - SPECTRUM_SURVEY_TIMESERIES = "spectrum_surveytimeseries" - - -DFCollectionTypeSchemas = { - DFCollectionType.USER: THLUserSchema, - DFCollectionType.WALL: THLWallSchema, - DFCollectionType.SESSION: THLSessionSchema, - DFCollectionType.IP_INFO: THLIPInfoSchema, - DFCollectionType.TASK_ADJUSTMENT: THLTaskAdjustmentSchema, - DFCollectionType.IP_HISTORY: UserHealthIPHistorySchema, - DFCollectionType.IP_HISTORY_WS: UserHealthIPHistoryWSSchema, - DFCollectionType.AUDIT_LOG: UserHealthAuditLogSchema, - DFCollectionType.LEDGER: LedgerSchema, - DFCollectionType.INNOVATE_SURVEY_HISTORY: InnovateSurveyHistorySchema, - DFCollectionType.MORNING_SURVEY_TIMESERIES: MorningSurveyTimeseriesSchema, - DFCollectionType.SAGO_SURVEY_HISTORY: SagoSurveyHistorySchema, - DFCollectionType.SPECTRUM_SURVEY_TIMESERIES: SpectrumSurveyTimeseriesSchema, -} - - -class DFCollectionItem(CollectionItemBase): - - # --- Properties --- - @property - def filename(self) -> str: - return ( - f"{self._collection.data_type.name.lower()}-{self._collection.offset}" - f"-{self.start.strftime('%Y-%m-%d-%H-%M-%S')}.parquet" - ) - - # --- Methods --- - - def has_postgres(self) -> bool: - if self._collection.pg_config is None: - return False - - connected = True - try: - self._collection.pg_config.execute_sql_query("""SELECT 1;""") - except AssertionError: - connected = False - - return connected - - def has_db(self) -> bool: - return self.has_mysql() or self.has_postgres() - - def update_partial_archive(self) -> bool: - if not self.valid_archive(self.partial_path, sample=1000): - LOG.error(f"invalid partial archive: {self.partial_path}") - return self.create_partial_archive() - df = pq.ParquetDataset(self.partial_path).read().to_pandas() - - order_key = self._collection._schema.metadata[ORDER_KEY] - archive_after = self._collection._schema.metadata[ARCHIVE_AFTER] - - partial_max = df[order_key].max().to_pydatetime() - - since = partial_max - archive_after - since = max([since, self.start]) # don't allow to query before the item's start - df = df[df[order_key] < since].copy() - - _df = self.from_db(since=since) - - if _df is not None: - df = pd.concat([df, _df]) - self.to_archive(ddf=dd.from_pandas(df, npartitions=1), is_partial=True) - else: - # The update to the partial returned no rows, but the partial - # still exists, so we'll continue with whatever was calling this. - # We don't need to re-write the partial or really do anything. - pass - return True - - def create_partial_archive(self) -> bool: - _df = self.from_db() - if _df is None: - # Returned no rows, but the period is not closed, so we - # don't want to mark as empty. Do nothing. - return False - return self.to_archive(ddf=dd.from_pandas(_df, npartitions=1), is_partial=True) - - # --- ORM / Data handlers--- - def to_dict(self) -> dict[str, Any]: - return self._to_dict() - - def from_db(self, since: datetime | None = None) -> pd.DataFrame | None: - if self._collection.data_type == DFCollectionType.LEDGER: - assert since is None, "Shouldn't pass since for Ledger item" - assert self._collection.pg_config is not None - return self.from_postgres_ledger() - else: - return self.from_db_standard(since=since) - - def from_db_standard(self, since: datetime | None = None) -> pd.DataFrame | None: - assert ( - self._collection.data_type != DFCollectionType.LEDGER - ), "Can't call from_postgres_standard for Ledger DFCollectionItem" - - start, finish = self.start, self.finish - LOG.debug( - f"{self._collection.data_type.value}.from_postgres(" - f"start={start.strftime(DT_STR)}, " - f"finish={finish.strftime(DT_STR)})" - ) - coll = self._collection - schema = coll._schema - pg_config = coll.pg_config - assert pg_config, "Must provide PostgresConfig" - - start = since or start - order_key = schema.metadata[ORDER_KEY] - cols = list(schema.columns.keys()) + [schema.index.name] - cols_str = ", ".join(cols) - - try: - res = pg_config.execute_sql_query( - query=f""" - SELECT {cols_str} - FROM {coll.data_type.value} - WHERE {order_key} >= %s AND {order_key} < %s; - """, - params=[start, finish], - ) - except AssertionError as e: - capture_exception(error=e) - LOG.error(f"_from_postgres Exception: {e}") - return None - - if not res: - LOG.warning("_from_postgres query returned nothing") - # Return an empty df.DataFrame with the correct columns - return empty_dataframe_from_schema(coll._schema) - - df = pd.DataFrame.from_records(res).set_index(coll._schema.index.name) - df = self.validate_df(df=df) - - if df is None: - LOG.warning("_from_postgres query results failed validation") - # Schema validation can fail... - return None - - return df - - def from_postgres_ledger(self) -> pd.DataFrame | None: - assert ( - self._collection.data_type == DFCollectionType.LEDGER - ), "Can only call from_postgres_ledger on Ledger DFCollectionItem" - - start, finish = self.start, self.finish - LOG.info( - f"{self._collection.data_type.value}.from_postgres_ledger(" - f"start={start.strftime(DT_STR)}, " - f"finish={finish.strftime(DT_STR)})" - ) - - coll = self._collection - assert coll.pg_config, "Must provide PostgresConfig" - pg_config: PostgresConfig = coll.pg_config - - limit = 20000 - offset = 0 - res = [] - while True: - LOG.info( - f"{self._collection.data_type.value}.from_postgres_ledger({limit=}, {offset=})" - ) - chunk = pg_config.execute_sql_query( - query=f""" - SELECT lt.id AS tx_id, lt.created, lt.ext_description, lt.tag, - le.id AS entry_id, le.direction, le.amount, le.account_id, - la.display_name, la.qualified_name, la.account_type, - la.normal_balance, la.reference_type, la.reference_uuid, - la.currency - FROM ledger_transaction AS lt - LEFT JOIN ledger_entry AS le - ON lt.id = le.transaction_id - LEFT JOIN ledger_account AS la - ON la.uuid = le.account_id - WHERE lt.created >= %s AND lt.created < %s - AND le.id IS NOT NULL - ORDER BY lt.created - LIMIT {limit} OFFSET {offset}; - """, - params=[start, finish], - ) - res.extend(chunk) - if not chunk: - break - offset += limit - - if len(res) == 0: - return None - - # Note (AND le.id IS NOT NULL): It is possible we have transactions with - # no ledger entries. This is because the transaction creation failed - # for some reason. The ledger is not unbalanced, it is just an orphan - # transaction. Just skip those here. - - tx_df = TxSchema.validate( - check_obj=pd.DataFrame.from_records(res).set_index("entry_id"), - lazy=True, - ) - - tx_ids = list(tx_df["tx_id"].unique()) - metadata_res = [] - # "MySQL server has gone away" if this is too big - conn = pg_config.make_connection() - c: Cursor = conn.cursor() - for chunk in chunked(tx_ids, n=5_000): - c.execute( - query=""" - SELECT ltm.transaction_id AS tx_id, - ltm.id AS tx_metadata_id, - ltm.key, ltm.value - FROM ledger_transactionmetadata AS ltm - WHERE ltm.transaction_id = ANY(%s); - """, - params=[chunk], - ) - metadata_res += c.fetchall() - - conn.close() - - tx_meta = ( - pd.DataFrame( - TxMetaSchema.validate( - check_obj=pd.DataFrame.from_records(metadata_res).set_index( - ["tx_id", "tx_metadata_id"] - ), - lazy=True, - ).pivot(columns="key", values="value"), - # This makes sure we expand to have all the possible columns - columns=[e.value for e in TransactionMetadataColumns], - ) - .groupby("tx_id") - .first() - ) - - df = tx_df.merge(tx_meta, how="left", left_on="tx_id", right_index=True) - df = self.validate_df(df=df) - - if df is None: - # Schema validation can fail... - return None - - return df - - def to_archive( - self, - ddf: dd.DataFrame, - is_partial: bool = False, - overwrite: bool = False, - ) -> bool: - """ - :returns: bool (saved_successful) - """ - assert isinstance(ddf, dd.DataFrame), "must pass dask df" - - client: DaskClient | None = self._collection._client - # client = None - - if client: - row_len = client.compute(collections=ddf.shape[0], sync=True) - else: - row_len = len(ddf.index) - is_empty = row_len == 0 - - if is_partial: - return self.to_archive_numbered_partial(ddf=ddf) - else: - return self._to_archive( - ddf=ddf, - is_empty=is_empty, - overwrite=overwrite, - ) - - def _to_archive( - self, - ddf: dd.DataFrame | None, - is_empty: bool, - overwrite: bool = False, - ) -> bool: - """ - For archiving an item. Will write an empty file if ddf is empty. This - is NOT for writing partials. - - :returns: bool (saved_successful) - """ - - if ddf is None: - return False - - should_archive = self.should_archive() - if not should_archive: - LOG.warning(f"Cannot create archive for such new data: {self.path}") - return False - - if overwrite is False: - has_archive = self.has_archive(include_empty=True) - if has_archive: - LOG.warning(f"archive already exists: {self.path}") - return False - - if is_empty: - # Create an .empty only if the Item is "archiveable" (which we checked above) - self.set_empty() - return True - - # Incase the file saving is interrupted, or otherwise fails - # save it to a tmp file first, then rename once we can confirm - # that it successfully loads - tmp_path = self.tmp_path() - try: - schema = self._collection._schema - assert schema - assert schema.metadata - partition = schema.metadata.get(PARTITION_ON) - - ddf.to_parquet( - path=tmp_path, - partition_on=partition, - engine="pyarrow", - overwrite=True, - write_metadata_file=True, - compression="brotli", - ) - - except (pa.ArrowInvalid, pa.ArrowIOError, OSError) as e: - LOG.exception(e) - self.delete_archive(tmp_path) - return False - - # It was saved, but the file seems to be corrupt - if not self.valid_archive(tmp_path): - LOG.error(f"not valid archive: {tmp_path}") - self.delete_archive(tmp_path) - # File did not save correctly so return it as saved=False - return False - - # To debug, just set this key to auto expire in 5 seconds - # RC.set(name=f"_to_archive:{self.path.as_posix()}", value=1, ex=15) - # with RC.lock(f"_to_archive:{self.path.as_posix()}:lock", timeout=15): - - if os.path.isfile(tmp_path): - # If the file was saved okay, seems okay, rename it - os.replace(tmp_path, self.path) - os.remove(tmp_path) - - if os.path.isdir(tmp_path): - if os.path.exists(self.path.as_posix()): - if overwrite: - subprocess.call(["rm", "-r", self.path.as_posix()]) - time.sleep(1) - else: - LOG.error(f"already exists: {self.path.as_posix()}") - return False - - if platform == "darwin": - subprocess.call(["mv", tmp_path.as_posix(), self.path.as_posix()]) - else: - # -T will (should) cause the mv to fail if path wasn't successfully deleted - subprocess.call(["mv", "-T", tmp_path.as_posix(), self.path.as_posix()]) - return True - - def to_archive_numbered_partial(self, ddf: dd.DataFrame | None = None) -> bool: - """ - For partial files/dirs only. Writes the .partial file with a number - at the end (.partial.####) and then creates a symlink - from .partial -> .partial.#### - - :returns: bool (saved_successful) - """ - if ddf is None: - return False - - collection = self._collection - schema = collection._schema - assert schema - client: DaskClient | None = collection._client - - next_numbered_path = self.next_numbered_path(self.partial_path) - partial_path = self.partial_path - # finish = self.finish - - # Make sure these are in the same dir. b/c the symlink has to be - # relative, not an absolute path - assert ( - partial_path.parent == next_numbered_path.parent - ), "Can't have numbered_path in a different directory" - target = ( - next_numbered_path.name - ) # this is the symlink's target. it is a relative path (only the name) - - should_archive = self.should_archive() - assert should_archive is False, "Don't write partial if the item is archiveable" - - if client: - row_len = client.compute(collections=ddf.shape[0], sync=True) - else: - row_len = len(ddf.index) - - if row_len == 0: - LOG.warning("Skipping, don't partial save an empty dd.DataFrame") - return False - - try: - assert schema.metadata - partition = schema.metadata.get(PARTITION_ON) - ddf.to_parquet( - path=next_numbered_path, - partition_on=partition, - engine="pyarrow", - overwrite=True, - write_metadata_file=True, - compression="brotli", - ) - except (pa.ArrowInvalid, pa.ArrowIOError, OSError) as e: - LOG.exception(e) - self.delete_archive(next_numbered_path) - return False - - if platform == "darwin": - subprocess.call(["ln", "-sfn", target, partial_path]) - else: - subprocess.call(["ln", "-sfnT", target, partial_path]) - - return True - - def initial_load(self, overwrite: bool = False) -> bool: - - if overwrite is False: - assert not self.has_archive(include_empty=True), "already archived" - - assert self.should_archive(), "not ready to archive!" - - df: pd.DataFrame | None = self.from_db() - - if df is None: - self.set_empty() - return False - - ddf = dd.from_pandas(df, npartitions=1) - return self.to_archive(ddf=ddf, is_partial=False, overwrite=overwrite) - - def clear_corrupt_archive(self): - if self.has_archive(include_empty=False) and not self.valid_archive(self.path): - LOG.warning(f"invalid archive, deleting: {self.path}") - self.delete_archive(self.path) - - -class DFCollection(CollectionBase): - data_type: DFCollectionType | None = Field(default=None) - - # --- Private --- - pg_config: PostgresConfig | None = Field(default=None) - - def __repr__(self): - res = self.signature() + "\n" - if len(self.items) > 6: - items = self.items[:3] + ["..."] + self.items[-3:] - else: - items = self.items - - for i in items: - res += f" – {repr(i) if isinstance(i, DFCollectionItem) else i}\n" - - return res - - def signature(self): - arr = [ - 1 if i.has_archive(include_empty=True) else 0 - for i in self.items - if i.should_archive() - ] - repr_str = ( - f"items={len(self.items)}; start={self.start} @ {self.offset}; {int(sum(arr) / len(arr) * 100)}% " - f"archived" - ) - res = f"{self.__repr_name__()}({repr_str})" - return res - - @field_validator("data_type") - def check_data_type(cls, data_type: DFCollectionType | None, info: ValidationInfo): - if data_type is None: - raise ValueError("Must explicitly provide a data_type") - - if data_type not in DFCollectionTypeSchemas: - raise ValueError("Must provide a supported data_type") - - return data_type - - # --- Properties --- - @property - def items(self) -> list[DFCollectionItem]: - items = [] - for iv in self.interval_range: - cm = DFCollectionItem(start=iv[0]) - cm._collection = self - items.append(cm) - return items - - @property - def _schema(self) -> DataFrameSchema: - return DFCollectionTypeSchemas[self.data_type] - - # --- Methods --- - - def initial_load( - self, - client: DaskClient | None = None, - sync: bool = True, - since: datetime | None = None, - client_resources: dict[str, Any] | None = None, - timeout: float | None = None, - ) -> list[Future]: - # This can be used to just build all local archive files - # We typically want to go backwards first, so we can most quickly - # populate the last 90 days for example - - client = client or self._client - - LOG.info(f"{self.data_type.value}.initial_load({since=}, {sync=})") - - items = self.items - if since: - items = self.get_items(since=since) - - if client is None: - for item in reversed(items): - if item.has_archive(include_empty=True): - continue - if not item.should_archive(): - continue - item.initial_load() - return [] - - fs = [] - for item in items: - if item.has_archive(include_empty=True): - continue - if not item.should_archive(): - continue - f = dask.delayed(item.initial_load)() - fs.append(f) - - if sync: - fs = client.compute(fs, sync=False, priority=2, resources=client_resources) - _ = as_completed(fs, timeout=timeout) - return fs - - else: - return client.compute(fs, sync=True, priority=2, resources=client_resources) - - def fetch_force_rr_latest(self, sources) -> list[FilePath]: - LOG.info( - f"{self.data_type.value}.fetch_force_rr_latest(sources={len(sources)})" - ) - - # We only want 'partial-able' items (those that can not yet be archived). - rr_items = [ - i for i in self.items if not i.should_archive() and not i.is_empty() - ] - if rr_items: - # If the ARCHIVE_AFTER time is > the collection offset (which it is always currently), - # then there typically wouldn't be more than 1 un-archivable item. - _start = rr_items[0].start - _end = rr_items[-1].finish - rr_duration = (_end - _start).total_seconds() - - # TODO: Do we want to be smarter about any rr selects max durations? - # allowing 2x the length of the offset. If we have more than this not archived, - # we want to run the archive first, not fetch from rr - archive_after = self._schema.metadata[ARCHIVE_AFTER] - allowed_rr_duration = ( - (pd.Timedelta(self.offset) * 2) + archive_after - ).total_seconds() - if rr_duration > allowed_rr_duration: - raise ValueError( - f"rr select duration exceeds {pd.Timedelta(allowed_rr_duration)}" - ) - - for rr_item in rr_items: - if ( - rr_item.has_partial_archive() - and self.data_type != DFCollectionType.LEDGER - ): - saved = rr_item.update_partial_archive() - else: - saved = rr_item.create_partial_archive() - if saved: - sources.append(rr_item.partial_path) - - return sources - - def force_rr_latest( - self, - client: DaskClient, - client_resources: dict[str, Any] | None = None, - sync: bool = True, - ) -> list[Future]: - - # For forcing update of any partials asynchronously if desired - LOG.info(f"{self.data_type.value}.force_rr_latest({client=})") - - rr_items = [ - i for i in self.items if not i.should_archive() and not i.is_empty() - ] - fs = [] - for rr_item in rr_items: - if ( - rr_item.has_partial_archive() - and self.data_type != DFCollectionType.LEDGER - ): - fs.append(dask.delayed(rr_item.update_partial_archive)()) - else: - fs.append(dask.delayed(rr_item.create_partial_archive)()) - return client.compute(fs, sync=sync, priority=2, resources=client_resources) diff --git a/generalresearch/incite/collections/base.py b/generalresearch/incite/collections/base.py new file mode 100644 index 0000000..ecfa57a --- /dev/null +++ b/generalresearch/incite/collections/base.py @@ -0,0 +1,693 @@ +from __future__ import annotations + +import os +import subprocess +import time +from datetime import datetime +from enum import StrEnum +from sys import platform +from typing import Any + +import dask +import dask.dataframe as dd +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq +from dask.distributed import Client as DaskClient +from dask.distributed import Future +from distributed import as_completed +from more_itertools import chunked +from pandera.pandas import DataFrameSchema +from psycopg import Cursor +from pydantic import Field, FilePath, ValidationInfo, field_validator +from sentry_sdk import capture_exception + +from generalresearch.incite import LOG +from generalresearch.incite.base import CollectionBase, CollectionItemBase +from generalresearch.incite.schemas import ( + ARCHIVE_AFTER, + ORDER_KEY, + PARTITION_ON, + empty_dataframe_from_schema, +) +from generalresearch.incite.schemas.thl_marketplaces import ( + InnovateSurveyHistorySchema, + MorningSurveyTimeseriesSchema, + SagoSurveyHistorySchema, + SpectrumSurveyTimeseriesSchema, +) +from generalresearch.incite.schemas.thl_web import ( + LedgerSchema, + THLIPInfoSchema, + THLSessionSchema, + THLTaskAdjustmentSchema, + THLUserSchema, + THLWallSchema, + TransactionMetadataColumns, + TxMetaSchema, + TxSchema, + UserHealthAuditLogSchema, + UserHealthIPHistorySchema, + UserHealthIPHistoryWSSchema, +) +from generalresearch.pg_helper import PostgresConfig + +DT_STR = "%Y-%m-%d %H:%M:%S" + + +class DFCollectionType(StrEnum): + TEST = "test" + + USER = "thl_user" + SESSION = "thl_session" + WALL = "thl_wall" + TASK_ADJUSTMENT = "thl_taskadjustment" + IP_INFO = "thl_ipinformation" + + AUDIT_LOG = "userhealth_auditlog" + IP_HISTORY = "userhealth_iphistory" + IP_HISTORY_WS = "userhealth_iphistory_ws" + + LEDGER = "ledger" + + INNOVATE_SURVEY_HISTORY = "innovate_surveyhistory" + MORNING_SURVEY_TIMESERIES = "morning_surveytimeseries" + SAGO_SURVEY_HISTORY = "sago_surveyhistory" + SPECTRUM_SURVEY_TIMESERIES = "spectrum_surveytimeseries" + + +DFCollectionTypeSchemas = { + DFCollectionType.USER: THLUserSchema, + DFCollectionType.WALL: THLWallSchema, + DFCollectionType.SESSION: THLSessionSchema, + DFCollectionType.IP_INFO: THLIPInfoSchema, + DFCollectionType.TASK_ADJUSTMENT: THLTaskAdjustmentSchema, + DFCollectionType.IP_HISTORY: UserHealthIPHistorySchema, + DFCollectionType.IP_HISTORY_WS: UserHealthIPHistoryWSSchema, + DFCollectionType.AUDIT_LOG: UserHealthAuditLogSchema, + DFCollectionType.LEDGER: LedgerSchema, + DFCollectionType.INNOVATE_SURVEY_HISTORY: InnovateSurveyHistorySchema, + DFCollectionType.MORNING_SURVEY_TIMESERIES: MorningSurveyTimeseriesSchema, + DFCollectionType.SAGO_SURVEY_HISTORY: SagoSurveyHistorySchema, + DFCollectionType.SPECTRUM_SURVEY_TIMESERIES: SpectrumSurveyTimeseriesSchema, +} + + +class DFCollectionItem(CollectionItemBase): + + # --- Properties --- + @property + def filename(self) -> str: + return ( + f"{self._collection.data_type.name.lower()}-{self._collection.offset}" + f"-{self.start.strftime('%Y-%m-%d-%H-%M-%S')}.parquet" + ) + + # --- Methods --- + + def has_postgres(self) -> bool: + if self._collection.pg_config is None: + return False + + connected = True + try: + self._collection.pg_config.execute_sql_query("""SELECT 1;""") + except AssertionError: + connected = False + + return connected + + def has_db(self) -> bool: + return self.has_mysql() or self.has_postgres() + + def update_partial_archive(self) -> bool: + if not self.valid_archive(self.partial_path, sample=1000): + LOG.error(f"invalid partial archive: {self.partial_path}") + return self.create_partial_archive() + df = pq.ParquetDataset(self.partial_path).read().to_pandas() + + order_key = self._collection._schema.metadata[ORDER_KEY] + archive_after = self._collection._schema.metadata[ARCHIVE_AFTER] + + partial_max = df[order_key].max().to_pydatetime() + + since = partial_max - archive_after + since = max([since, self.start]) # don't allow to query before the item's start + df = df[df[order_key] < since].copy() + + _df = self.from_db(since=since) + + if _df is not None: + df = pd.concat([df, _df]) + self.to_archive(ddf=dd.from_pandas(df, npartitions=1), is_partial=True) + else: + # The update to the partial returned no rows, but the partial + # still exists, so we'll continue with whatever was calling this. + # We don't need to re-write the partial or really do anything. + pass + return True + + def create_partial_archive(self) -> bool: + _df = self.from_db() + if _df is None: + # Returned no rows, but the period is not closed, so we + # don't want to mark as empty. Do nothing. + return False + return self.to_archive(ddf=dd.from_pandas(_df, npartitions=1), is_partial=True) + + # --- ORM / Data handlers--- + def to_dict(self) -> dict[str, Any]: + return self._to_dict() + + def from_db(self, since: datetime | None = None) -> pd.DataFrame | None: + if self._collection.data_type == DFCollectionType.LEDGER: + assert since is None, "Shouldn't pass since for Ledger item" + assert self._collection.pg_config is not None + return self.from_postgres_ledger() + else: + return self.from_db_standard(since=since) + + def from_db_standard(self, since: datetime | None = None) -> pd.DataFrame | None: + assert ( + self._collection.data_type != DFCollectionType.LEDGER + ), "Can't call from_postgres_standard for Ledger DFCollectionItem" + + start, finish = self.start, self.finish + LOG.debug( + f"{self._collection.data_type.value}.from_postgres(" + f"start={start.strftime(DT_STR)}, " + f"finish={finish.strftime(DT_STR)})" + ) + coll = self._collection + schema = coll._schema + pg_config = coll.pg_config + assert pg_config, "Must provide PostgresConfig" + + start = since or start + order_key = schema.metadata[ORDER_KEY] + cols = list(schema.columns.keys()) + [schema.index.name] + cols_str = ", ".join(cols) + + try: + res = pg_config.execute_sql_query( + query=f""" + SELECT {cols_str} + FROM {coll.data_type.value} + WHERE {order_key} >= %s AND {order_key} < %s; + """, + params=[start, finish], + ) + except AssertionError as e: + capture_exception(error=e) + LOG.error(f"_from_postgres Exception: {e}") + return None + + if not res: + LOG.warning("_from_postgres query returned nothing") + # Return an empty df.DataFrame with the correct columns + return empty_dataframe_from_schema(coll._schema) + + df = pd.DataFrame.from_records(res).set_index(coll._schema.index.name) + df = self.validate_df(df=df) + + if df is None: + LOG.warning("_from_postgres query results failed validation") + # Schema validation can fail... + return None + + return df + + def from_postgres_ledger(self) -> pd.DataFrame | None: + assert ( + self._collection.data_type == DFCollectionType.LEDGER + ), "Can only call from_postgres_ledger on Ledger DFCollectionItem" + + start, finish = self.start, self.finish + LOG.info( + f"{self._collection.data_type.value}.from_postgres_ledger(" + f"start={start.strftime(DT_STR)}, " + f"finish={finish.strftime(DT_STR)})" + ) + + coll = self._collection + assert coll.pg_config, "Must provide PostgresConfig" + pg_config: PostgresConfig = coll.pg_config + + limit = 20000 + offset = 0 + res = [] + while True: + LOG.info( + f"{self._collection.data_type.value}.from_postgres_ledger({limit=}, {offset=})" + ) + chunk = pg_config.execute_sql_query( + query=f""" + SELECT lt.id AS tx_id, lt.created, lt.ext_description, lt.tag, + le.id AS entry_id, le.direction, le.amount, le.account_id, + la.display_name, la.qualified_name, la.account_type, + la.normal_balance, la.reference_type, la.reference_uuid, + la.currency + FROM ledger_transaction AS lt + LEFT JOIN ledger_entry AS le + ON lt.id = le.transaction_id + LEFT JOIN ledger_account AS la + ON la.uuid = le.account_id + WHERE lt.created >= %s AND lt.created < %s + AND le.id IS NOT NULL + ORDER BY lt.created + LIMIT {limit} OFFSET {offset}; + """, + params=[start, finish], + ) + res.extend(chunk) + if not chunk: + break + offset += limit + + if len(res) == 0: + return None + + # Note (AND le.id IS NOT NULL): It is possible we have transactions with + # no ledger entries. This is because the transaction creation failed + # for some reason. The ledger is not unbalanced, it is just an orphan + # transaction. Just skip those here. + + tx_df = TxSchema.validate( + check_obj=pd.DataFrame.from_records(res).set_index("entry_id"), + lazy=True, + ) + + tx_ids = list(tx_df["tx_id"].unique()) + metadata_res = [] + # "MySQL server has gone away" if this is too big + conn = pg_config.make_connection() + c: Cursor = conn.cursor() + for chunk in chunked(tx_ids, n=5_000): + c.execute( + query=""" + SELECT ltm.transaction_id AS tx_id, + ltm.id AS tx_metadata_id, + ltm.key, ltm.value + FROM ledger_transactionmetadata AS ltm + WHERE ltm.transaction_id = ANY(%s); + """, + params=[chunk], + ) + metadata_res += c.fetchall() + + conn.close() + + tx_meta = ( + pd.DataFrame( + TxMetaSchema.validate( + check_obj=pd.DataFrame.from_records(metadata_res).set_index( + ["tx_id", "tx_metadata_id"] + ), + lazy=True, + ).pivot(columns="key", values="value"), + # This makes sure we expand to have all the possible columns + columns=[e.value for e in TransactionMetadataColumns], + ) + .groupby("tx_id") + .first() + ) + + df = tx_df.merge(tx_meta, how="left", left_on="tx_id", right_index=True) + df = self.validate_df(df=df) + + if df is None: + # Schema validation can fail... + return None + + return df + + def to_archive( + self, + ddf: dd.DataFrame, + is_partial: bool = False, + overwrite: bool = False, + ) -> bool: + """ + :returns: bool (saved_successful) + """ + assert isinstance(ddf, dd.DataFrame), "must pass dask df" + + client: DaskClient | None = self._collection._client + # client = None + + if client: + row_len = client.compute(collections=ddf.shape[0], sync=True) + else: + row_len = len(ddf.index) + is_empty = row_len == 0 + + if is_partial: + return self.to_archive_numbered_partial(ddf=ddf) + else: + return self._to_archive( + ddf=ddf, + is_empty=is_empty, + overwrite=overwrite, + ) + + def _to_archive( + self, + ddf: dd.DataFrame | None, + is_empty: bool, + overwrite: bool = False, + ) -> bool: + """ + For archiving an item. Will write an empty file if ddf is empty. This + is NOT for writing partials. + + :returns: bool (saved_successful) + """ + + if ddf is None: + return False + + should_archive = self.should_archive() + if not should_archive: + LOG.warning(f"Cannot create archive for such new data: {self.path}") + return False + + if overwrite is False: + has_archive = self.has_archive(include_empty=True) + if has_archive: + LOG.warning(f"archive already exists: {self.path}") + return False + + if is_empty: + # Create an .empty only if the Item is "archiveable" (which we checked above) + self.set_empty() + return True + + # Incase the file saving is interrupted, or otherwise fails + # save it to a tmp file first, then rename once we can confirm + # that it successfully loads + tmp_path = self.tmp_path() + try: + schema = self._collection._schema + assert schema + assert schema.metadata + partition = schema.metadata.get(PARTITION_ON) + + ddf.to_parquet( + path=tmp_path, + partition_on=partition, + engine="pyarrow", + overwrite=True, + write_metadata_file=True, + compression="brotli", + ) + + except (pa.ArrowInvalid, pa.ArrowIOError, OSError) as e: + LOG.exception(e) + self.delete_archive(tmp_path) + return False + + # It was saved, but the file seems to be corrupt + if not self.valid_archive(tmp_path): + LOG.error(f"not valid archive: {tmp_path}") + self.delete_archive(tmp_path) + # File did not save correctly so return it as saved=False + return False + + # To debug, just set this key to auto expire in 5 seconds + # RC.set(name=f"_to_archive:{self.path.as_posix()}", value=1, ex=15) + # with RC.lock(f"_to_archive:{self.path.as_posix()}:lock", timeout=15): + + if os.path.isfile(tmp_path): + # If the file was saved okay, seems okay, rename it + os.replace(tmp_path, self.path) + os.remove(tmp_path) + + if os.path.isdir(tmp_path): + if os.path.exists(self.path.as_posix()): + if overwrite: + subprocess.call(["rm", "-r", self.path.as_posix()]) + time.sleep(1) + else: + LOG.error(f"already exists: {self.path.as_posix()}") + return False + + if platform == "darwin": + subprocess.call(["mv", tmp_path.as_posix(), self.path.as_posix()]) + else: + # -T will (should) cause the mv to fail if path wasn't successfully deleted + subprocess.call(["mv", "-T", tmp_path.as_posix(), self.path.as_posix()]) + return True + + def to_archive_numbered_partial(self, ddf: dd.DataFrame | None = None) -> bool: + """ + For partial files/dirs only. Writes the .partial file with a number + at the end (.partial.####) and then creates a symlink + from .partial -> .partial.#### + + :returns: bool (saved_successful) + """ + if ddf is None: + return False + + collection = self._collection + schema = collection._schema + assert schema + client: DaskClient | None = collection._client + + next_numbered_path = self.next_numbered_path(self.partial_path) + partial_path = self.partial_path + # finish = self.finish + + # Make sure these are in the same dir. b/c the symlink has to be + # relative, not an absolute path + assert ( + partial_path.parent == next_numbered_path.parent + ), "Can't have numbered_path in a different directory" + target = ( + next_numbered_path.name + ) # this is the symlink's target. it is a relative path (only the name) + + should_archive = self.should_archive() + assert should_archive is False, "Don't write partial if the item is archiveable" + + if client: + row_len = client.compute(collections=ddf.shape[0], sync=True) + else: + row_len = len(ddf.index) + + if row_len == 0: + LOG.warning("Skipping, don't partial save an empty dd.DataFrame") + return False + + try: + assert schema.metadata + partition = schema.metadata.get(PARTITION_ON) + ddf.to_parquet( + path=next_numbered_path, + partition_on=partition, + engine="pyarrow", + overwrite=True, + write_metadata_file=True, + compression="brotli", + ) + except (pa.ArrowInvalid, pa.ArrowIOError, OSError) as e: + LOG.exception(e) + self.delete_archive(next_numbered_path) + return False + + if platform == "darwin": + subprocess.call(["ln", "-sfn", target, partial_path]) + else: + subprocess.call(["ln", "-sfnT", target, partial_path]) + + return True + + def initial_load(self, overwrite: bool = False) -> bool: + + if overwrite is False: + assert not self.has_archive(include_empty=True), "already archived" + + assert self.should_archive(), "not ready to archive!" + + df: pd.DataFrame | None = self.from_db() + + if df is None: + self.set_empty() + return False + + ddf = dd.from_pandas(df, npartitions=1) + return self.to_archive(ddf=ddf, is_partial=False, overwrite=overwrite) + + def clear_corrupt_archive(self): + if self.has_archive(include_empty=False) and not self.valid_archive(self.path): + LOG.warning(f"invalid archive, deleting: {self.path}") + self.delete_archive(self.path) + + +class DFCollection(CollectionBase): + data_type: DFCollectionType | None = Field(default=None) + + # --- Private --- + pg_config: PostgresConfig | None = Field(default=None) + + def __repr__(self): + res = self.signature() + "\n" + if len(self.items) > 6: + items = self.items[:3] + ["..."] + self.items[-3:] + else: + items = self.items + + for i in items: + res += f" – {repr(i) if isinstance(i, DFCollectionItem) else i}\n" + + return res + + def signature(self): + arr = [ + 1 if i.has_archive(include_empty=True) else 0 + for i in self.items + if i.should_archive() + ] + repr_str = ( + f"items={len(self.items)}; start={self.start} @ {self.offset}; {int(sum(arr) / len(arr) * 100)}% " + f"archived" + ) + res = f"{self.__repr_name__()}({repr_str})" + return res + + @field_validator("data_type") + def check_data_type(cls, data_type: DFCollectionType | None, info: ValidationInfo): + if data_type is None: + raise ValueError("Must explicitly provide a data_type") + + if data_type not in DFCollectionTypeSchemas: + raise ValueError("Must provide a supported data_type") + + return data_type + + # --- Properties --- + @property + def items(self) -> list[DFCollectionItem]: + items = [] + for iv in self.interval_range: + cm = DFCollectionItem(start=iv[0]) + cm._collection = self + items.append(cm) + return items + + @property + def _schema(self) -> DataFrameSchema: + return DFCollectionTypeSchemas[self.data_type] + + # --- Methods --- + + def initial_load( + self, + client: DaskClient | None = None, + sync: bool = True, + since: datetime | None = None, + client_resources: dict[str, Any] | None = None, + timeout: float | None = None, + ) -> list[Future]: + # This can be used to just build all local archive files + # We typically want to go backwards first, so we can most quickly + # populate the last 90 days for example + + client = client or self._client + + LOG.info(f"{self.data_type.value}.initial_load({since=}, {sync=})") + + items = self.items + if since: + items = self.get_items(since=since) + + if client is None: + for item in reversed(items): + if item.has_archive(include_empty=True): + continue + if not item.should_archive(): + continue + item.initial_load() + return [] + + fs = [] + for item in items: + if item.has_archive(include_empty=True): + continue + if not item.should_archive(): + continue + f = dask.delayed(item.initial_load)() + fs.append(f) + + if sync: + fs = client.compute(fs, sync=False, priority=2, resources=client_resources) + _ = as_completed(fs, timeout=timeout) + return fs + + else: + return client.compute(fs, sync=True, priority=2, resources=client_resources) + + def fetch_force_rr_latest(self, sources) -> list[FilePath]: + LOG.info( + f"{self.data_type.value}.fetch_force_rr_latest(sources={len(sources)})" + ) + + # We only want 'partial-able' items (those that can not yet be archived). + rr_items = [ + i for i in self.items if not i.should_archive() and not i.is_empty() + ] + if rr_items: + # If the ARCHIVE_AFTER time is > the collection offset (which it is always currently), + # then there typically wouldn't be more than 1 un-archivable item. + _start = rr_items[0].start + _end = rr_items[-1].finish + rr_duration = (_end - _start).total_seconds() + + # TODO: Do we want to be smarter about any rr selects max durations? + # allowing 2x the length of the offset. If we have more than this not archived, + # we want to run the archive first, not fetch from rr + archive_after = self._schema.metadata[ARCHIVE_AFTER] + allowed_rr_duration = ( + (pd.Timedelta(self.offset) * 2) + archive_after + ).total_seconds() + if rr_duration > allowed_rr_duration: + raise ValueError( + f"rr select duration exceeds {pd.Timedelta(allowed_rr_duration)}" + ) + + for rr_item in rr_items: + if ( + rr_item.has_partial_archive() + and self.data_type != DFCollectionType.LEDGER + ): + saved = rr_item.update_partial_archive() + else: + saved = rr_item.create_partial_archive() + if saved: + sources.append(rr_item.partial_path) + + return sources + + def force_rr_latest( + self, + client: DaskClient, + client_resources: dict[str, Any] | None = None, + sync: bool = True, + ) -> list[Future]: + + # For forcing update of any partials asynchronously if desired + LOG.info(f"{self.data_type.value}.force_rr_latest({client=})") + + rr_items = [ + i for i in self.items if not i.should_archive() and not i.is_empty() + ] + fs = [] + for rr_item in rr_items: + if ( + rr_item.has_partial_archive() + and self.data_type != DFCollectionType.LEDGER + ): + fs.append(dask.delayed(rr_item.update_partial_archive)()) + else: + fs.append(dask.delayed(rr_item.create_partial_archive)()) + return client.compute(fs, sync=sync, priority=2, resources=client_resources) diff --git a/generalresearch/incite/collections/thl_marketplaces.py b/generalresearch/incite/collections/thl_marketplaces.py index fe2b01f..246fe87 100644 --- a/generalresearch/incite/collections/thl_marketplaces.py +++ b/generalresearch/incite/collections/thl_marketplaces.py @@ -1,6 +1,6 @@ from typing import Literal -from generalresearch.incite.collections import DFCollection, DFCollectionType +from generalresearch.incite.collections.base import DFCollection, DFCollectionType from generalresearch.incite.schemas.thl_marketplaces import ( InnovateSurveyHistorySchema, MorningSurveyTimeseriesSchema, diff --git a/generalresearch/incite/defaults.py b/generalresearch/incite/defaults.py index d4025fc..773555c 100644 --- a/generalresearch/incite/defaults.py +++ b/generalresearch/incite/defaults.py @@ -15,7 +15,7 @@ from generalresearch.incite.collections.thl_web import ( UserDFCollection, WallDFCollection, ) -from generalresearch.incite.mergers import MergeType +from generalresearch.incite.mergers.base import MergeType from generalresearch.incite.mergers.foundations.enriched_session import ( EnrichedSessionMerge, ) @@ -47,9 +47,7 @@ def session_df_collection( ) -def wall_df_collection( - ds: GRLDatasets, pg_config: PostgresConfig -) -> WallDFCollection: +def wall_df_collection(ds: GRLDatasets, pg_config: PostgresConfig) -> WallDFCollection: return WallDFCollection( offset="49h", pg_config=pg_config, @@ -58,9 +56,7 @@ def wall_df_collection( ) -def user_df_collection( - ds: GRLDatasets, pg_config: PostgresConfig -) -> UserDFCollection: +def user_df_collection(ds: GRLDatasets, pg_config: PostgresConfig) -> UserDFCollection: return UserDFCollection( offset="73h", pg_config=pg_config, diff --git a/generalresearch/incite/mergers/__init__.py b/generalresearch/incite/mergers/__init__.py index 810bedc..e69de29 100644 --- a/generalresearch/incite/mergers/__init__.py +++ b/generalresearch/incite/mergers/__init__.py @@ -1,301 +0,0 @@ -import logging -import os.path -import subprocess -from datetime import UTC, datetime -from enum import StrEnum -from sys import platform -from typing import Self - -import dask.dataframe as dd -import pandas as pd -from dask.distributed import Client -from pandera.pandas import DataFrameSchema -from pydantic import Field, ValidationInfo, field_validator, model_validator - -from generalresearch.incite.base import CollectionBase, CollectionItemBase -from generalresearch.incite.schemas import PARTITION_ON -from generalresearch.incite.schemas.mergers.foundations.enriched_session import ( - EnrichedSessionSchema, -) -from generalresearch.incite.schemas.mergers.foundations.enriched_task_adjust import ( - EnrichedTaskAdjustSchema, -) -from generalresearch.incite.schemas.mergers.foundations.enriched_wall import ( - EnrichedWallSchema, -) -from generalresearch.incite.schemas.mergers.foundations.user_id_product import ( - UserIdProductSchema, -) -from generalresearch.incite.schemas.mergers.pop_ledger import ( - PopLedgerSchema, -) -from generalresearch.incite.schemas.mergers.ym_survey_wall import ( - YMSurveyWallSchema, -) -from generalresearch.incite.schemas.mergers.ym_wall_summary import ( - YMWallSummarySchema, -) -from generalresearch.models.custom_types import AwareDatetimeISO - -LOG = logging.getLogger("incite") - - -class MergeType(StrEnum): - TEST = "test" - YM_SURVEY_WALL = "ym_survey_wall" - YM_WALL_SUMMARY = "ym_wall_summary" - - POP_LEDGER = "pop_ledger" - - # --- Foundations --- - USER_ID_PRODUCT = "user_id_product" - ENRICHED_WALL = "enriched_wall" - ENRICHED_SESSION = "enriched_session" - ENRICHED_TASK_ADJUST = "enriched_task_adjust" - - -MergeTypeSchemas = { - MergeType.YM_SURVEY_WALL: YMSurveyWallSchema, - MergeType.YM_WALL_SUMMARY: YMWallSummarySchema, - MergeType.POP_LEDGER: PopLedgerSchema, - # --- Foundations --- - MergeType.USER_ID_PRODUCT: UserIdProductSchema, - MergeType.ENRICHED_WALL: EnrichedWallSchema, - MergeType.ENRICHED_SESSION: EnrichedSessionSchema, - MergeType.ENRICHED_TASK_ADJUST: EnrichedTaskAdjustSchema, -} - - -class MergeCollectionItem(CollectionItemBase): - - # --- Properties --- - - @property - def finish(self) -> datetime: - # A MergeCollection can have offset = None - if self._collection.offset: - return ( - pd.Timestamp(self.start) + pd.Timedelta(self._collection.offset) - ).to_pydatetime() - else: - return datetime.now(tz=UTC).replace(microsecond=0) - - @property - def filename(self) -> str: - grouped_key = self._collection.grouped_key - offset = self._collection.offset - start = self.start.strftime("%Y-%m-%d-%H-%M-%S") - f = [self._collection.merge_type.name.lower()] - if offset: - f.append(offset) - if grouped_key: - f.append(grouped_key) - if self._collection.start is not None: - # This is a collection that is "looking back" 'offset' time (1 item). - f.append(start) - s = "-".join(f) - s += ".parquet" - return s - - # --- ORM / Data handlers--- - def to_dict(self, *args, **kwargs) -> dict: - res = self._to_dict() - res["group_by"] = self._collection.group_by - return res - - def to_archive( - self, - client: Client, - ddf: dd.DataFrame, - is_partial: bool = False, - ) -> bool: - assert is_partial is False, "use to_archive_symlink" - return self._to_archive(client=client, ddf=ddf, client_resources=None) - - def _to_archive( - self, client: Client, ddf: dd.DataFrame | None, client_resources=None - ) -> bool: - """ - For archiving an item. Will write an empty file if ddf is empty. - This is NOT for writing partials. - - :returns: bool (saved_successful) - """ - if ddf is None: - return False - - row_len: int = client.compute(collections=ddf.shape[0], sync=True) - assert row_len - assert row_len > 0, "empty ddf" - - tmp_path = self.tmp_path() - schema = self._collection._schema - assert schema.metadata - - partition = schema.metadata.get(PARTITION_ON) - f = ddf.to_parquet( - compute=False, - path=tmp_path, - partition_on=partition, - engine="pyarrow", - overwrite=True, - write_metadata_file=True, - compression="brotli", - ) - client.compute(f, sync=True, priority=2, resources=client_resources) - assert not os.path.exists( - self.path.as_posix() - ), f"already exits!: {self.path.as_posix()}" - - if platform == "darwin": - subprocess.call(["mv", tmp_path.as_posix(), self.path.as_posix()]) - else: - # -T will (should) cause the mv to fail if `path` wasn't successfully deleted - subprocess.call(["mv", "-T", tmp_path.as_posix(), self.path.as_posix()]) - return True - - def to_archive_symlink( - self, - client: Client, - ddf: dd.DataFrame, - is_partial: bool = False, - client_resources=None, - validate_after=True, - ) -> bool: - """ - This differs from to_archive(): - 1) to_parquet is run in this process. If the df is already - computed, there is no point in sending it to another worker - to write. - - 2) symlink to next_numbered_path is created whether or not - is_partial (to_archive only does this on partials) - - 3) we do not validate the written file. seems not useful to do - this, as the file will probably get overwritten on the next - loop anyway - """ - path = self.partial_path if is_partial else self.path - next_numbered_path = self.next_numbered_path(path) - collection = self._collection - LOG.warning(f"{collection.merge_type.value}.to_archive_symlink()") - - assert isinstance(ddf, dd.DataFrame), "must pass a dask df" - - # We should validate before or after!!! - # _validate_df(self.compute(ddf), coll._schema) - target = ( - next_numbered_path.name - ) # this is the symlink's target. it is a relative path (only the name) - - schema = self._collection._schema - partition = schema.metadata.get(PARTITION_ON, None) - f = ddf.to_parquet( - compute=False, - path=next_numbered_path.as_posix(), - partition_on=partition, - engine="pyarrow", - overwrite=True, - write_metadata_file=True, - compression="brotli", - ) - client.compute(f, sync=True, priority=2, resources=client_resources) - - if os.path.exists(path.as_posix()) and not os.path.islink(path.as_posix()): - # This will fail when going from the old way to using symlinks, - # if self.path already exists and is a directory. - raise ValueError( - f"first time we run this, make sure the path doesnt exist: {path.as_posix()}" - ) - - if platform == "darwin": - subprocess.call(["ln", "-sfn", target, path.as_posix()]) - else: - subprocess.call(["ln", "-sfnT", target, path.as_posix()]) - - if validate_after and not self.valid_archive(self.path): - LOG.error(f"{collection.merge_type.value} failed validation: {self.path}") - self.delete_archive(self.path) - return False - return True - - # todo: unclear what the common interface should be here ... ? - def fetch(self, *args, **kwargs) -> pd.DataFrame | dd.DataFrame: - raise NotImplementedError("implement in subclass") - - def build(self, *args, **kwargs) -> pd.DataFrame | dd.DataFrame: - raise NotImplementedError("implement in subclass") - - -class MergeCollection(CollectionBase): - """Mergers take instances of DFCollections, and/or other Mergers""" - - # In a merge, we can set offset = None which indicates that there is only 1 - # period/item where the range is 'start' until now. - offset: str | None = Field(default="72h") - # In a merge, we can set start = None which indicates that there is only 1 - # period/item where the range is (now - offset) until now. - start: AwareDatetimeISO | None = Field( - default=None, - description="This is the starting point in which data will" - " be retrieved in chunks from.", - frozen=True, - ) - - merge_type: MergeType | None = Field(default=None) - group_by: str | None = Field(default=None) - grouped_key: str | None = Field(default=None) - collection_item_class: type[MergeCollectionItem] = MergeCollectionItem - - @model_validator(mode="after") - def check_start_and_offset_nullable(self) -> Self: - if self.offset is None and self.start is None: - raise AssertionError("cannot set both start and offset to None") - return self - - @field_validator("merge_type") - def check_merge_type(cls, merge_type: MergeType | None, info: ValidationInfo): - if merge_type is None: - raise ValueError("Must explicitly provide a merge_type") - - if merge_type not in MergeTypeSchemas: - raise ValueError("Must provide a supported merge_type") - - return merge_type - - # --- Properties --- - @property - def interval_start(self) -> datetime | None: - # if self.start is None and self.offset is set, the inferred start is (now - offset) - if self.start is None: - return datetime.now(tz=UTC).replace(microsecond=0) - pd.Timedelta( - self.offset - ) - return self.start - - @property - def items(self) -> list[MergeCollectionItem]: - items = [] - for iv in self.interval_range: - cm = self.collection_item_class(start=iv[0]) - cm._collection = self - items.append(cm) - return items - - @property - def _schema(self) -> DataFrameSchema: - return MergeTypeSchemas[self.merge_type] - - def signature(self) -> str: - arr = [ - 1 if i.has_archive(include_empty=True) else 0 - for i in self.items - if i.should_archive() - ] - repr_str = ( - f"path={self.archive_path.as_posix()}; " - f"items={len(self.items)}; start={self.start} @ {self.offset}; {int(sum(arr) / len(arr) * 100)}% " - f"archived" - ) - res = f"{self.__repr_name__()}({repr_str})" - return res diff --git a/generalresearch/incite/mergers/base.py b/generalresearch/incite/mergers/base.py new file mode 100644 index 0000000..810bedc --- /dev/null +++ b/generalresearch/incite/mergers/base.py @@ -0,0 +1,301 @@ +import logging +import os.path +import subprocess +from datetime import UTC, datetime +from enum import StrEnum +from sys import platform +from typing import Self + +import dask.dataframe as dd +import pandas as pd +from dask.distributed import Client +from pandera.pandas import DataFrameSchema +from pydantic import Field, ValidationInfo, field_validator, model_validator + +from generalresearch.incite.base import CollectionBase, CollectionItemBase +from generalresearch.incite.schemas import PARTITION_ON +from generalresearch.incite.schemas.mergers.foundations.enriched_session import ( + EnrichedSessionSchema, +) +from generalresearch.incite.schemas.mergers.foundations.enriched_task_adjust import ( + EnrichedTaskAdjustSchema, +) +from generalresearch.incite.schemas.mergers.foundations.enriched_wall import ( + EnrichedWallSchema, +) +from generalresearch.incite.schemas.mergers.foundations.user_id_product import ( + UserIdProductSchema, +) +from generalresearch.incite.schemas.mergers.pop_ledger import ( + PopLedgerSchema, +) +from generalresearch.incite.schemas.mergers.ym_survey_wall import ( + YMSurveyWallSchema, +) +from generalresearch.incite.schemas.mergers.ym_wall_summary import ( + YMWallSummarySchema, +) +from generalresearch.models.custom_types import AwareDatetimeISO + +LOG = logging.getLogger("incite") + + +class MergeType(StrEnum): + TEST = "test" + YM_SURVEY_WALL = "ym_survey_wall" + YM_WALL_SUMMARY = "ym_wall_summary" + + POP_LEDGER = "pop_ledger" + + # --- Foundations --- + USER_ID_PRODUCT = "user_id_product" + ENRICHED_WALL = "enriched_wall" + ENRICHED_SESSION = "enriched_session" + ENRICHED_TASK_ADJUST = "enriched_task_adjust" + + +MergeTypeSchemas = { + MergeType.YM_SURVEY_WALL: YMSurveyWallSchema, + MergeType.YM_WALL_SUMMARY: YMWallSummarySchema, + MergeType.POP_LEDGER: PopLedgerSchema, + # --- Foundations --- + MergeType.USER_ID_PRODUCT: UserIdProductSchema, + MergeType.ENRICHED_WALL: EnrichedWallSchema, + MergeType.ENRICHED_SESSION: EnrichedSessionSchema, + MergeType.ENRICHED_TASK_ADJUST: EnrichedTaskAdjustSchema, +} + + +class MergeCollectionItem(CollectionItemBase): + + # --- Properties --- + + @property + def finish(self) -> datetime: + # A MergeCollection can have offset = None + if self._collection.offset: + return ( + pd.Timestamp(self.start) + pd.Timedelta(self._collection.offset) + ).to_pydatetime() + else: + return datetime.now(tz=UTC).replace(microsecond=0) + + @property + def filename(self) -> str: + grouped_key = self._collection.grouped_key + offset = self._collection.offset + start = self.start.strftime("%Y-%m-%d-%H-%M-%S") + f = [self._collection.merge_type.name.lower()] + if offset: + f.append(offset) + if grouped_key: + f.append(grouped_key) + if self._collection.start is not None: + # This is a collection that is "looking back" 'offset' time (1 item). + f.append(start) + s = "-".join(f) + s += ".parquet" + return s + + # --- ORM / Data handlers--- + def to_dict(self, *args, **kwargs) -> dict: + res = self._to_dict() + res["group_by"] = self._collection.group_by + return res + + def to_archive( + self, + client: Client, + ddf: dd.DataFrame, + is_partial: bool = False, + ) -> bool: + assert is_partial is False, "use to_archive_symlink" + return self._to_archive(client=client, ddf=ddf, client_resources=None) + + def _to_archive( + self, client: Client, ddf: dd.DataFrame | None, client_resources=None + ) -> bool: + """ + For archiving an item. Will write an empty file if ddf is empty. + This is NOT for writing partials. + + :returns: bool (saved_successful) + """ + if ddf is None: + return False + + row_len: int = client.compute(collections=ddf.shape[0], sync=True) + assert row_len + assert row_len > 0, "empty ddf" + + tmp_path = self.tmp_path() + schema = self._collection._schema + assert schema.metadata + + partition = schema.metadata.get(PARTITION_ON) + f = ddf.to_parquet( + compute=False, + path=tmp_path, + partition_on=partition, + engine="pyarrow", + overwrite=True, + write_metadata_file=True, + compression="brotli", + ) + client.compute(f, sync=True, priority=2, resources=client_resources) + assert not os.path.exists( + self.path.as_posix() + ), f"already exits!: {self.path.as_posix()}" + + if platform == "darwin": + subprocess.call(["mv", tmp_path.as_posix(), self.path.as_posix()]) + else: + # -T will (should) cause the mv to fail if `path` wasn't successfully deleted + subprocess.call(["mv", "-T", tmp_path.as_posix(), self.path.as_posix()]) + return True + + def to_archive_symlink( + self, + client: Client, + ddf: dd.DataFrame, + is_partial: bool = False, + client_resources=None, + validate_after=True, + ) -> bool: + """ + This differs from to_archive(): + 1) to_parquet is run in this process. If the df is already + computed, there is no point in sending it to another worker + to write. + + 2) symlink to next_numbered_path is created whether or not + is_partial (to_archive only does this on partials) + + 3) we do not validate the written file. seems not useful to do + this, as the file will probably get overwritten on the next + loop anyway + """ + path = self.partial_path if is_partial else self.path + next_numbered_path = self.next_numbered_path(path) + collection = self._collection + LOG.warning(f"{collection.merge_type.value}.to_archive_symlink()") + + assert isinstance(ddf, dd.DataFrame), "must pass a dask df" + + # We should validate before or after!!! + # _validate_df(self.compute(ddf), coll._schema) + target = ( + next_numbered_path.name + ) # this is the symlink's target. it is a relative path (only the name) + + schema = self._collection._schema + partition = schema.metadata.get(PARTITION_ON, None) + f = ddf.to_parquet( + compute=False, + path=next_numbered_path.as_posix(), + partition_on=partition, + engine="pyarrow", + overwrite=True, + write_metadata_file=True, + compression="brotli", + ) + client.compute(f, sync=True, priority=2, resources=client_resources) + + if os.path.exists(path.as_posix()) and not os.path.islink(path.as_posix()): + # This will fail when going from the old way to using symlinks, + # if self.path already exists and is a directory. + raise ValueError( + f"first time we run this, make sure the path doesnt exist: {path.as_posix()}" + ) + + if platform == "darwin": + subprocess.call(["ln", "-sfn", target, path.as_posix()]) + else: + subprocess.call(["ln", "-sfnT", target, path.as_posix()]) + + if validate_after and not self.valid_archive(self.path): + LOG.error(f"{collection.merge_type.value} failed validation: {self.path}") + self.delete_archive(self.path) + return False + return True + + # todo: unclear what the common interface should be here ... ? + def fetch(self, *args, **kwargs) -> pd.DataFrame | dd.DataFrame: + raise NotImplementedError("implement in subclass") + + def build(self, *args, **kwargs) -> pd.DataFrame | dd.DataFrame: + raise NotImplementedError("implement in subclass") + + +class MergeCollection(CollectionBase): + """Mergers take instances of DFCollections, and/or other Mergers""" + + # In a merge, we can set offset = None which indicates that there is only 1 + # period/item where the range is 'start' until now. + offset: str | None = Field(default="72h") + # In a merge, we can set start = None which indicates that there is only 1 + # period/item where the range is (now - offset) until now. + start: AwareDatetimeISO | None = Field( + default=None, + description="This is the starting point in which data will" + " be retrieved in chunks from.", + frozen=True, + ) + + merge_type: MergeType | None = Field(default=None) + group_by: str | None = Field(default=None) + grouped_key: str | None = Field(default=None) + collection_item_class: type[MergeCollectionItem] = MergeCollectionItem + + @model_validator(mode="after") + def check_start_and_offset_nullable(self) -> Self: + if self.offset is None and self.start is None: + raise AssertionError("cannot set both start and offset to None") + return self + + @field_validator("merge_type") + def check_merge_type(cls, merge_type: MergeType | None, info: ValidationInfo): + if merge_type is None: + raise ValueError("Must explicitly provide a merge_type") + + if merge_type not in MergeTypeSchemas: + raise ValueError("Must provide a supported merge_type") + + return merge_type + + # --- Properties --- + @property + def interval_start(self) -> datetime | None: + # if self.start is None and self.offset is set, the inferred start is (now - offset) + if self.start is None: + return datetime.now(tz=UTC).replace(microsecond=0) - pd.Timedelta( + self.offset + ) + return self.start + + @property + def items(self) -> list[MergeCollectionItem]: + items = [] + for iv in self.interval_range: + cm = self.collection_item_class(start=iv[0]) + cm._collection = self + items.append(cm) + return items + + @property + def _schema(self) -> DataFrameSchema: + return MergeTypeSchemas[self.merge_type] + + def signature(self) -> str: + arr = [ + 1 if i.has_archive(include_empty=True) else 0 + for i in self.items + if i.should_archive() + ] + repr_str = ( + f"path={self.archive_path.as_posix()}; " + f"items={len(self.items)}; start={self.start} @ {self.offset}; {int(sum(arr) / len(arr) * 100)}% " + f"archived" + ) + res = f"{self.__repr_name__()}({repr_str})" + return res diff --git a/generalresearch/incite/mergers/foundations/enriched_session.py b/generalresearch/incite/mergers/foundations/enriched_session.py index 049b1bc..4a300fc 100644 --- a/generalresearch/incite/mergers/foundations/enriched_session.py +++ b/generalresearch/incite/mergers/foundations/enriched_session.py @@ -14,7 +14,7 @@ from generalresearch.incite.collections.thl_web import ( SessionDFCollection, WallDFCollection, ) -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/mergers/foundations/enriched_task_adjust.py b/generalresearch/incite/mergers/foundations/enriched_task_adjust.py index e8a3654..f7c679f 100644 --- a/generalresearch/incite/mergers/foundations/enriched_task_adjust.py +++ b/generalresearch/incite/mergers/foundations/enriched_task_adjust.py @@ -12,7 +12,7 @@ from generalresearch.incite.collections.thl_web import ( TaskAdjustmentDFCollection, ) from generalresearch.incite.exceptions import BuildError, BuildItemsError -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/mergers/foundations/enriched_wall.py b/generalresearch/incite/mergers/foundations/enriched_wall.py index a74a556..9db5328 100644 --- a/generalresearch/incite/mergers/foundations/enriched_wall.py +++ b/generalresearch/incite/mergers/foundations/enriched_wall.py @@ -12,7 +12,7 @@ from generalresearch.incite.collections.thl_web import ( SessionDFCollection, WallDFCollection, ) -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, @@ -25,11 +25,11 @@ from generalresearch.incite.schemas.mergers.foundations.enriched_wall import ( EnrichedWallSchema, ) from generalresearch.models.custom_types import UUIDStr -from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig if TYPE_CHECKING: from generalresearch.models.admin.request import ReportRequest + from generalresearch.models.thl.user import User LOG = logging.getLogger("incite") diff --git a/generalresearch/incite/mergers/foundations/user_id_product.py b/generalresearch/incite/mergers/foundations/user_id_product.py index 863a741..3fb7b36 100644 --- a/generalresearch/incite/mergers/foundations/user_id_product.py +++ b/generalresearch/incite/mergers/foundations/user_id_product.py @@ -6,7 +6,7 @@ from typing import Any, Literal from distributed import Client from generalresearch.incite.collections.thl_web import UserDFCollection -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/mergers/pop_ledger.py b/generalresearch/incite/mergers/pop_ledger.py index b32503c..cf63cad 100644 --- a/generalresearch/incite/mergers/pop_ledger.py +++ b/generalresearch/incite/mergers/pop_ledger.py @@ -9,7 +9,7 @@ from distributed import Client from more_itertools import flatten from generalresearch.incite.collections.thl_web import LedgerDFCollection -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/mergers/ym_survey_wall.py b/generalresearch/incite/mergers/ym_survey_wall.py index a99e8ec..3cbf543 100644 --- a/generalresearch/incite/mergers/ym_survey_wall.py +++ b/generalresearch/incite/mergers/ym_survey_wall.py @@ -11,7 +11,7 @@ from sentry_sdk import capture_exception from generalresearch.incite.collections.thl_web import WallDFCollection from generalresearch.incite.exceptions import BuildError -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/mergers/ym_wall_summary.py b/generalresearch/incite/mergers/ym_wall_summary.py index 69ef5c5..a01443b 100644 --- a/generalresearch/incite/mergers/ym_wall_summary.py +++ b/generalresearch/incite/mergers/ym_wall_summary.py @@ -13,7 +13,7 @@ from generalresearch.incite.collections.thl_web import ( WallDFCollection, ) from generalresearch.incite.exceptions import FetchError -from generalresearch.incite.mergers import ( +from generalresearch.incite.mergers.base import ( MergeCollection, MergeCollectionItem, MergeType, diff --git a/generalresearch/incite/schemas/thl_web.py b/generalresearch/incite/schemas/thl_web.py index c1be202..30c7076 100644 --- a/generalresearch/incite/schemas/thl_web.py +++ b/generalresearch/incite/schemas/thl_web.py @@ -1,6 +1,7 @@ from datetime import UTC, datetime, timedelta import pandas as pd +from grip_client.enums import AccessType from pandera.pandas import Check, Column, DataFrameSchema, Index, MultiIndex from generalresearch.incite.schemas import ARCHIVE_AFTER, ORDER_KEY @@ -16,7 +17,6 @@ from generalresearch.models.thl.definitions import ( WallStatusCode2, ) from generalresearch.models.thl.ledger import TransactionMetadataColumns -from generalresearch.models.thl.maxmind.definitions import UserType IP_REGEX_PATTERN = ( r"^((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][" @@ -392,7 +392,7 @@ THLIPInfoSchema = DataFrameSchema( dtype=str, checks=[ Check.str_length(min_value=3, max_value=255), - Check.isin([e.value for e in UserType]), + Check.isin([e.value for e in AccessType]), ], nullable=True, ), diff --git a/generalresearch/models/network/label.py b/generalresearch/models/network/label.py index 60a6e58..b8fe4b0 100644 --- a/generalresearch/models/network/label.py +++ b/generalresearch/models/network/label.py @@ -51,6 +51,7 @@ class IPLabelSource(StrEnum): INTERNAL_USE = "internal_use" # An external "security" service flagged this IP + GRIP = "grip" SPUR = "spur" IPINFO = "ipinfo" MAXMIND = "maxmind" diff --git a/generalresearch/models/thl/definitions.py b/generalresearch/models/thl/definitions.py index 0217a80..c40df21 100644 --- a/generalresearch/models/thl/definitions.py +++ b/generalresearch/models/thl/definitions.py @@ -185,7 +185,7 @@ class SessionStatusCode2(IntEnum, metaclass=ReprEnumMeta): # Unable to parse either the bucket_id, request_id, or nudge_id from the url ENTRY_URL_MODIFICATION = 1 - # The client's IP failed maxmind lookup, or we failed to store it for some reason + # The client's IP failed GRIP lookup, or we failed to store it for some reason UNRECOGNIZED_IP = 2 # User is using an anonymous IP USER_IS_ANONYMOUS = 3 diff --git a/generalresearch/models/thl/ipinfo.py b/generalresearch/models/thl/ipinfo.py index e327bae..1e2be5b 100644 --- a/generalresearch/models/thl/ipinfo.py +++ b/generalresearch/models/thl/ipinfo.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime from typing import Any, Literal, Self from faker import Faker +from grip_client.enums import AccessType from pydantic import ( BaseModel, ConfigDict, @@ -19,7 +20,6 @@ from generalresearch.models.custom_types import ( CountryISOLike, IPvAnyAddressStr, ) -from generalresearch.models.thl.maxmind.definitions import UserType from generalresearch.pg_helper import PostgresConfig fake = Faker() @@ -173,11 +173,11 @@ class IPInformation(BaseModel): default=None, description="A score indicating the likelihood that the IP address is static.", ) - user_type: UserType | None = Field( + user_type: AccessType | None = Field( default=None, description="The type of user associated with the IP address " "(e.g., 'residential', 'business').", - examples=[UserType.SCHOOL], + examples=[AccessType.RESIDENTIAL], ) postal_code: str | None = Field( default=None, @@ -218,8 +218,8 @@ class IPInformation(BaseModel): @property def basic(self) -> bool: - # This could be almost any field, but we're checking here if maxmind - # insights was run on this record. If not, then most of the optional + # This could be almost any field, but we're checking here if GRIP + # was run on this record. If not, then most of the optional # fields will be None return self.is_anonymous is None @@ -253,7 +253,7 @@ class IPInformation(BaseModel): return d @classmethod - def from_mysql(cls, d: dict) -> Self: + def from_mysql(cls, d: dict[str, Any]) -> Self: d["updated"] = d["updated"].replace(tzinfo=UTC) return cls.model_validate(d) @@ -261,3 +261,13 @@ class IPInformation(BaseModel): class GeoIPInformation(IPInformation, IPGeoname): model_config = ConfigDict(extra="ignore") + + geoname_id: PositiveInt # type: ignore[reportIncompatibleVariableOverride] + + @field_validator("geoname_id", mode="before") + @classmethod + def _coerce_geoname_id(cls, v: PositiveInt | None): + if v is None: + raise ValueError("GeoIPInformation can't be constructed") + + return v diff --git a/generalresearch/models/thl/ledger.py b/generalresearch/models/thl/ledger.py index a9fbbb1..2b25d2e 100644 --- a/generalresearch/models/thl/ledger.py +++ b/generalresearch/models/thl/ledger.py @@ -22,12 +22,6 @@ from generalresearch.models.custom_types import ( UUIDStr, check_valid_uuid, ) -from generalresearch.models.thl.ledger_example import ( - _example_user_tx_adjustment, - _example_user_tx_bonus, - _example_user_tx_complete, - _example_user_tx_payout, -) from generalresearch.models.thl.pagination import Page from generalresearch.models.thl.payout_format import ( PayoutFormatType, @@ -36,6 +30,53 @@ from generalresearch.models.thl.payout_format import ( from generalresearch.utils.enum import ReprEnumMeta +def _example_user_tx_payout(schema: dict[str, Any]) -> None: + + schema["example"] = UserLedgerTransactionUserPayout( + product_id=uuid4().hex, + payout_id=uuid4().hex, + amount=-5, + description="HIT Reward", + payout_format="${payout/100:.2f}", + created=datetime.now(tz=UTC), + ).model_dump(mode="json") + + +def _example_user_tx_bonus(schema: dict[str, Any]) -> None: + + schema["example"] = UserLedgerTransactionUserBonus( + product_id=uuid4().hex, + amount=100, + description="Compensation Bonus", + payout_format="${payout/100:.2f}", + created=datetime.now(tz=UTC), + ).model_dump(mode="json") + + +def _example_user_tx_complete(schema: dict[str, Any]) -> None: + + schema["example"] = UserLedgerTransactionTaskComplete( + product_id=uuid4().hex, + amount=38, + description="Task Complete", + payout_format="${payout/100:.2f}", + created=datetime.now(tz=UTC), + tsid=uuid4().hex, + ).model_dump(mode="json") + + +def _example_user_tx_adjustment(schema: dict[str, Any]) -> None: + + schema["example"] = UserLedgerTransactionTaskAdjustment( + product_id=uuid4().hex, + amount=-38, + description="Task Adjustment", + payout_format="${payout/100:.2f}", + created=datetime.now(tz=UTC), + tsid=uuid4().hex, + ).model_dump(mode="json") + + class Direction(IntEnum, metaclass=ReprEnumMeta): """Entries on the debit side will increase debit normal accounts, while entries on the credit side will decrease them. Conversely, entries on @@ -393,7 +434,7 @@ class UserLedgerTransaction(BaseModel): # It is optional b/c we'll calculate this from the query balance_after: int | None = Field(default=None) - def create_url(self, product_id: str): + def create_url(self, product_id: str) -> str | None: raise NotImplementedError() @computed_field( @@ -431,7 +472,7 @@ class UserLedgerTransactionUserPayout(UserLedgerTransaction): examples=["a3848e0a53d64f68a74ced5f61b6eb68"], ) - def create_url(self, product_id: str): + def create_url(self, product_id: str) -> str | None: return f"https://fsb.generalresearch.com/{product_id}/cashout/{self.payout_id}/" @model_validator(mode="after") @@ -459,7 +500,7 @@ class UserLedgerTransactionUserBonus(UserLedgerTransaction): default="Compensation Bonus", ) - def create_url(self, product_id: str): + def create_url(self, product_id: str) -> str | None: return None @model_validator(mode="after") @@ -497,7 +538,7 @@ class UserLedgerTransactionTaskComplete(UserLedgerTransaction): examples=["a3848e0a53d64f68a74ced5f61b6eb68"], ) - def create_url(self, product_id: str): + def create_url(self, product_id: str) -> str | None: return f"https://fsb.generalresearch.com/{product_id}/status/{self.tsid}/" @model_validator(mode="after") @@ -528,7 +569,7 @@ class UserLedgerTransactionTaskAdjustment(UserLedgerTransaction): examples=["a3848e0a53d64f68a74ced5f61b6eb68"], ) - def create_url(self, product_id: str): + def create_url(self, product_id: str) -> str | None: return f"https://fsb.generalresearch.com/{product_id}/status/{self.tsid}/" diff --git a/generalresearch/models/thl/ledger_example.py b/generalresearch/models/thl/ledger_example.py deleted file mode 100644 index 0291691..0000000 --- a/generalresearch/models/thl/ledger_example.py +++ /dev/null @@ -1,64 +0,0 @@ -from __future__ import annotations - -from datetime import UTC, datetime -from typing import Any -from uuid import uuid4 - - -def _example_user_tx_payout(schema: dict[str, Any]) -> None: - from generalresearch.models.thl.ledger import ( - UserLedgerTransactionUserPayout, - ) - - schema["example"] = UserLedgerTransactionUserPayout( - product_id=uuid4().hex, - payout_id=uuid4().hex, - amount=-5, - description="HIT Reward", - payout_format="${payout/100:.2f}", - created=datetime.now(tz=UTC), - ).model_dump(mode="json") - - -def _example_user_tx_bonus(schema: dict[str, Any]) -> None: - from generalresearch.models.thl.ledger import ( - UserLedgerTransactionUserBonus, - ) - - schema["example"] = UserLedgerTransactionUserBonus( - product_id=uuid4().hex, - amount=100, - description="Compensation Bonus", - payout_format="${payout/100:.2f}", - created=datetime.now(tz=UTC), - ).model_dump(mode="json") - - -def _example_user_tx_complete(schema: dict[str, Any]) -> None: - from generalresearch.models.thl.ledger import ( - UserLedgerTransactionTaskComplete, - ) - - schema["example"] = UserLedgerTransactionTaskComplete( - product_id=uuid4().hex, - amount=38, - description="Task Complete", - payout_format="${payout/100:.2f}", - created=datetime.now(tz=UTC), - tsid=uuid4().hex, - ).model_dump(mode="json") - - -def _example_user_tx_adjustment(schema: dict[str, Any]) -> None: - from generalresearch.models.thl.ledger import ( - UserLedgerTransactionTaskAdjustment, - ) - - schema["example"] = UserLedgerTransactionTaskAdjustment( - product_id=uuid4().hex, - amount=-38, - description="Task Adjustment", - payout_format="${payout/100:.2f}", - created=datetime.now(tz=UTC), - tsid=uuid4().hex, - ).model_dump(mode="json") diff --git a/generalresearch/models/thl/maxmind/__init__.py b/generalresearch/models/thl/maxmind/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/generalresearch/models/thl/maxmind/definitions.py b/generalresearch/models/thl/maxmind/definitions.py deleted file mode 100644 index 01431c7..0000000 --- a/generalresearch/models/thl/maxmind/definitions.py +++ /dev/null @@ -1,22 +0,0 @@ -from enum import Enum - -from generalresearch.utils.enum import ReprEnumMeta - - -class UserType(Enum, metaclass=ReprEnumMeta): - # https://support.maxmind.com/hc/en-us/articles/4408430082971-IP-Trait-Risk-Data#h_01FN6V8JMQMWZGWNPPAW77ZPY4 - BUSINESS = "business" - CAFE = "cafe" - CELLULAR = "cellular" - COLLEGE = "college" - CDN = "content_delivery_network" - CPN = "consumer_privacy_network" - GOVERNMENT = "government" - HOSTING = "hosting" - LIBRARY = "library" - MILITARY = "military" - RESIDENTIAL = "residential" - ROUTER = "router" - SCHOOL = "school" - SEARCH_ENGINE = "search_engine_spider" - TRAVELER = "traveler" diff --git a/generalresearch/models/thl/session.py b/generalresearch/models/thl/session.py index e4e264f..d547cf4 100644 --- a/generalresearch/models/thl/session.py +++ b/generalresearch/models/thl/session.py @@ -296,9 +296,9 @@ class WallBase(BaseModel): finished: datetime | None = None, ) -> None: # This should be called by the wall manager in order to actually update db - from generalresearch import wall_status_codes + from generalresearch.wall_status_codes import annotate_status_code - status, status_code_1, status_code_2 = wall_status_codes.annotate_status_code( + status, status_code_1, status_code_2 = annotate_status_code( self.source, ext_status_code_1, ext_status_code_2, @@ -317,18 +317,18 @@ class WallBase(BaseModel): ) def is_soft_fail(self) -> bool: - from generalresearch import wall_status_codes + from generalresearch.wall_status_codes import is_soft_fail assert self.status is not None, "status should not be None" assert self.status_code_1 is not None, "status_code_1 should not be None" - return wall_status_codes.is_soft_fail(self) + return is_soft_fail(self) def stop_marketplace_session(self) -> bool: - from generalresearch import wall_status_codes + from generalresearch.wall_status_codes import stop_marketplace_session assert self.status is not None, "status should not be None" assert self.status_code_1 is not None, "status_code_1 should not be None" - return wall_status_codes.stop_marketplace_session(self) + return stop_marketplace_session(self) def get_status_after_adjustment(self) -> Status: if self.adjusted_status in { @@ -349,10 +349,13 @@ class WallBase(BaseModel): WallAdjustedStatus.CPI_ADJUSTMENT, }: return self.adjusted_cpi + elif self.adjusted_status == WallAdjustedStatus.ADJUSTED_TO_FAIL: return Decimal(0) + elif self.status == Status.COMPLETE: return self.cpi + else: return Decimal(0) @@ -442,7 +445,7 @@ class Wall(WallBase): d = self.model_dump(mode="json", exclude={"elapsed"}) return json.dumps(d) - def model_dump_mysql(self, *args, **kwargs) -> dict: + def model_dump_mysql(self, *args, **kwargs) -> dict[str, Any]: # Generate a dictionary representation of the model, with special handling for datetimes d = self.model_dump(mode="json", exclude={"elapsed"}, *args, **kwargs) d["started"] = self.started.replace(tzinfo=None) @@ -496,13 +499,13 @@ class WallOut(WallBase): ) # Serialize user_cpi to an int - @field_serializer("user_cpi", return_type=int) - def serialize_user_cpi(self, v: Decimal, _info): + @field_serializer("user_cpi", return_type=int | None) + def serialize_user_cpi(self, v: Decimal | None) -> int | None: return decimal_to_int_cents(v) # If user_cpi is an int, put it back to a decimal @field_validator("user_cpi", mode="before") - def deserialize_user_cpi(cls, v): + def deserialize_user_cpi(cls, v: Decimal | None) -> Decimal | None: if isinstance(v, int): return int_cents_to_decimal(v) return v @@ -510,7 +513,7 @@ class WallOut(WallBase): # noinspection PyNestedDecorators @field_validator("user_cpi", mode="after") @classmethod - def check_cpi_decimal_places(cls, v: Decimal) -> Decimal: + def check_cpi_decimal_places(cls, v: Decimal | None) -> Decimal | None: if v is not None: assert ( v.as_tuple().exponent >= -5 diff --git a/generalresearch/models/thl/user_iphistory.py b/generalresearch/models/thl/user_iphistory.py index 257c41a..2773629 100644 --- a/generalresearch/models/thl/user_iphistory.py +++ b/generalresearch/models/thl/user_iphistory.py @@ -5,6 +5,7 @@ from datetime import UTC, datetime, timedelta from typing import Self from faker import Faker +from grip_client.enums import AccessType from pydantic import ( BaseModel, ConfigDict, @@ -22,7 +23,6 @@ from generalresearch.models.thl.ipinfo import ( GeoIPInformation, normalize_ip, ) -from generalresearch.models.thl.maxmind.definitions import UserType from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig @@ -53,7 +53,11 @@ class UserIPRecord(BaseModel): ) @property - def user_type(self) -> UserType | None: + def user_type(self) -> AccessType | None: + return self.information.user_type if self.information else None + + @property + def access_type(self) -> AccessType | None: return self.information.user_type if self.information else None @property @@ -204,7 +208,6 @@ class UserIPHistory(BaseModel): if res.get(x.ip): x.information = res[x.ip] - def collapse_ip_records(self): """ - Records where sequential ipv6 addresses are in the same /64 block, diff --git a/pyproject.toml b/pyproject.toml index 03a1a1f..94f5073 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,4 +59,8 @@ addopts = "-v --tb=short" target-version = "py314" exclude = [ "generalresearch/thl_django", -] \ No newline at end of file +] + +[tool.pylint.messages_control] +disable = ["all"] +enable = ["cyclic-import"] \ No newline at end of file diff --git a/test_utils/grliq/conftest.py b/test_utils/grliq/conftest.py index 9a3bc56..891b73c 100644 --- a/test_utils/grliq/conftest.py +++ b/test_utils/grliq/conftest.py @@ -2,13 +2,13 @@ from __future__ import annotations from collections.abc import Callable from datetime import UTC, datetime, timedelta +from typing import Any from uuid import uuid4 import pytest from pydantic import PostgresDsn from generalresearch.config import GRLBaseSettings -from generalresearch.grliq.managers import DUMMY_GRLIQ_DATA from generalresearch.grliq.managers.forensic_data import ( GrlIqDataManager, ) @@ -19,6 +19,10 @@ from generalresearch.grliq.managers.forensic_results import ( GrlIqCategoryResultsReader, ) from generalresearch.grliq.models.forensic_data import GrlIqData +from generalresearch.grliq.models.forensic_result import ( + GrlIqCheckerResults, + GrlIqForensicCategoryResult, +) from generalresearch.pg_helper import PostgresConfig # === Miscellaneous === @@ -75,11 +79,42 @@ def grliq_crr(grliq_db: PostgresConfig) -> GrlIqCategoryResultsReader: # === Models === +@pytest.fixture(scope="session") +def grliq_data_list() -> list[dict[str, Any]]: + return [ + { + "data": GrlIqData.model_validate_json( + """{"mid": "3722ed29314940fabd37b42d808dcf5a", "uuid": "b11441da5a854dfbb8401d4c32e56db5", "phase": "offerwall-enter", "events": null, "vendor": "Google Inc.", "app_name": "Netscape", "calendar": "gregory", "language": "en-US", "platform": "Linux x86_64", "timezone": "America/Mexico_City", "client_ip": "131.196.250.250", "timestamp": "2025-02-27T16:05:34-06:00", "webrtc_ip": "131.196.250.250", "created_at": "2025-02-27T22:05:35.370589Z", "language_2": "en-US", "language_3": null, "platform_2": "Linux x86_64", "platform_3": null, "prefetched": true, "product_id": "d0606a0b5d034a8d81b1e3579d1f76fd", "webgl_flag": true, "webgl_hash": "da27e1b9b660057a3f5e185d3f5deabe", "canvas_hash": "14ed764326ec454d976c322261d99f16", "color_gamut": "3", "country_iso": "mx", "inner_width": 612, "outer_width": 1813, "product_sub": "20030107", "audio_codecs": "1,1,1,1,1,3,1,3,1,3,3,1,1,3,3,3,3,1,3,3,3,2,1,1", "cookie_check": "", "graphics_api": "WebKit WebGL", "inner_height": 1174, "mouse_events": null, "ontouchstart": false, "outer_height": 1261, "plugins_hash": "4c05fa2f766a444d4f253ead792c8b0e|2", "screen_width": 2560, "video_codecs": "1,3,3,3,3,3,3,3,3,3,1,1,1,1,1,1,3,1,1,1,3,3,1", "webgl_hash_2": "fc73fd5db75e2c36222fe34251be3971", "webrtc_error": false, "window_opera": false, "battery_level": 0.9, "canvas_hash_2": "bd11ebbf5c26fd20e0217820b4159752", "dynamic_range": false, "error_message": "Cannot read", "forced_colors": false, "math_result_1": "1.9275814160560204e-50", "math_result_2": "1.6182817135715877", "screen_height": 1440, "webgl_check_1": true, "webgl_context": "webgl2", "window_chrome": true, "connection_rtt": 150, "history_length": 16, "user_agent_str": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "web_sql_exists": false, "calender_locale": "en-US", "connection_type": "", "inverted_colors": true, "navigator_brave": false, "product_user_id": "d1d55df1-959e-4740-b77c-fa1f4fc457ae", "request_headers": {"host": "test", "accept": "*/*", "connection": "keep-alive", "user-agent": "python-httpx/0.27.0", "content-length": "3646", "accept-encoding": "gzip, deflate", "x-forwarded-for": "131.196.250.250"}, "timezone_offset": 360, "webrtc_local_ip": "50486637-6b64-4812-b10a-0a75337c31bd.local", "battery_charging": true, "client_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "131.196.250.250", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "mx", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "max_touch_points": 0, "numbering_system": "latn", "path_fingerprint": 3252, "prefers_contrast": "0", "rendering_engine": "WebKit", "timezone_success": "pass", "user_agent_hints": {"model": null, "brands": [{"brand": "Google Chrome", "version": "131"}, {"brand": "Chromium", "version": "131"}, {"brand": "Not_A Brand", "version": "24"}], "mobile": false, "bitness": "64", "platform": "Linux", "brands_full": [{"brand": "Google Chrome", "version": "131.0.6778.204"}, {"brand": "Chromium", "version": "131.0.6778.204"}, {"brand": "Not_A Brand", "version": "24.0.0.0"}], "architecture": "x86", "platform_version": "6.2.0"}, "user_agent_str_2": null, "webgl_extensions": "EXT_clip_control|EXT_color_buffer_float|EXT_color_buffer_half_float|EXT_conservative_depth|EXT_depth_clamp|EXT_disjoint_timer_query_webgl2|EXT_float_blend|EXT_polygon_offset_clamp|EXT_render_snorm|EXT_texture_compression_bptc|EXT_texture_compression_rgtc|EXT_texture_filter_anisotropic|EXT_texture_mirror_clamp_to_edge|EXT_texture_norm16|KHR_parallel_shader_compile|NV_shader_noperspective_interpolation|OES_draw_buffers_indexed|OES_sample_variables|OES_shader_multisample_interpolation|OES_texture_float_linear|OVR_multiview2|WEBGL_blend_func_extended|WEBGL_clip_cull_distance|WEBGL_compressed_texture_astc|WEBGL_compressed_texture_etc|WEBGL_compressed_texture_etc1|WEBGL_compressed_texture_s3tc|WEBGL_compressed_texture_s3tc_srgb|WEBGL_debug_renderer_info|WEBGL_debug_shaders|WEBGL_lose_context|WEBGL_multi_draw|WEBGL_polygon_mode|WEBGL_provoking_vertex|WEBGL_stencil_texturing", "webrtc_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "131.196.250.250", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "mx", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "chrome_extensions": "", "execution_time_ms": 371.0999999642372, "graphics_renderer": "WebGL 2.0 (OpenGL ES 3.0 Chromium)", "keyboard_detected": true, "mime_types_length": 2, "request_fs_exists": true, "audio_context_flag": "pass", "audio_context_hash": "9307303774dec3248c18a939392090da", "canvas_fingerprint": 258, "canvas_pixel_check": false, "device_pixel_ratio": 1.0, "indexedDbData_blob": true, "navigator_keys_len": 79, "no_edge_pdf_plugin": false, "screen_avail_width": 2560, "webdriver_detected": false, "window_orientation": 0, "connection_downlink": 10.0, "navigator_webdriver": false, "non_native_function": false, "screen_avail_height": 1400, "supported_fonts_str": "72|768|262144|1073741824|0|0|540672|73728|7340032|1342177280|117446656|256|16|0|543|4290797636|1677723648|4168998400|0|1048576|262144|268500994|1342177280|262144|125829376|37888000|0|435363842|0|2147483648|109543424|1880099872|268435471", "text_2d_fingerprint": "bfcce91c9e71d11af7b14dbee4c75f83", "webrtc_is_supported": "pass", "canvas_support_level": "full", "do_not_track_enabled": "1", "hardware_concurrency": 12, "keyboard_layout_size": 48, "prefers_color_scheme": false, "webgl_max_anisotropy": 16, "battery_charging_time": 0.0, "browser_by_properties": "c", "eval_to_string_length": 33, "performance_loop_time": 0.09999996423721313, "session_storage_check": "pass", "unmasked_vendor_webgl": "Google Inc. (Intel)", "hardware_concurrency_2": 12, "hardware_concurrency_3": null, "localStorage_available": true, "memory_jsHeapSizeLimit": 4294705152, "mozilla_web_app_exists": false, "navigator_deviceMemory": 8.0, "navigator_java_enabled": false, "prefers_reduced_motion": false, "storage_estimate_quota": 1178717110272, "webdriver_detected_msg": "", "window_active_x_object": false, "window_external_exists": true, "color_depth_pixel_depth": "24-24", "indexedDbData_available": true, "navigator_cookieEnabled": true, "unmasked_renderer_webgl": "ANGLE (Intel, Mesa Intel(R) Graphics (RPL-P), OpenGL 4.6)", "battery_discharging_time": 0.0, "connection_effectiveType": "4g", "non_native_function_flag": "", "speech_synthesis_voice_1": "Google Bahasa Indonesia", "window_client_information": true, "audio_compressor_reduction": 20.538288116455078, "navigator_mediaDevices_len": 3, "audio_intensity_fingerprint": 124.04347527516074, "speech_synthesis_voice_hash": "8010ee3313813de521e48e63bd5a6f13", "microsoft_credentials_exists": false, "window_installTrigger_exists": false, "speech_synthesis_voices_count": 19, "webgl_shading_language_version": "WebGL GLSL ES 3.00 (OpenGL ES GLSL ES 3.0 Chromium)", "error_message_stack_access_count": 0, "speech_synthesis_avail_voices_count": 19, "error_message_stack_access_count_worker": 0}""" + ), + "result_data": GrlIqCheckerResults.model_validate_json( + """{"uuid": "b11441da5a854dfbb8401d4c32e56db5", "check_codecs": {"score": 0}, "check_timezone": {"score": 0}, "check_timestamp": {"score": 0}, "check_user_type": {"score": 0}, "check_ip_changes": {"score": 0}, "check_ip_country": {"score": 0}, "check_environment": {"score": 0}, "check_ip_timezone": {"score": 0}, "check_isp_changes": {"score": 0}, "check_useragent_js": {"score": 0}, "check_required_fonts": {"score": 0}, "check_user_anonymous": {"score": 0}, "check_webrtc_success": {"score": 0}, "check_seen_timestamps": {"msg": "duplicate timestamp", "score": 100}, "check_country_timezone": {"score": 0}, "check_prohibited_fonts": {"score": 0}, "check_timezone_changes": {"score": 0}, "check_execution_time_ms": {"msg": "duplicate execution_time_ms", "score": 100}, "check_fingerprint_reuse": {"score": 0}, "check_fingerprint_cycling": {"score": 0}, "check_ip_webrtc_ip_detail": {"score": 0}, "check_environment_critical": {"score": 0}, "check_useragent_other_enums": {"score": 0}, "check_useragent_ip_properties": {"score": 0}, "check_useragent_data_properties": {"score": 0}, "check_useragent_device_family_brand": {"score": 0}}""" + ), + "category_result": GrlIqForensicCategoryResult.model_validate_json( + """{"uuid": "b11441da5a854dfbb8401d4c32e56db5", "is_bot": 0, "is_tampered": 100, "is_velocity": 0, "is_anonymous": 0, "suspicious_ip": 0, "is_oscillating": 0, "is_teleporting": 0, "is_inconsistent": 0, "platform_ip_inconsistent": 0}""" + ), + "fraud_score": 100, + "is_attempt_allowed": False, + }, + { + "data": GrlIqData.model_validate_json( + """{"mid": "35f6f5c30bc74ea7ac4aca7b40a02352", "uuid": "d54509f2f310499f8ab74839b10b2a41", "phase": "offerwall-enter", "events": null, "vendor": "Google Inc.", "app_name": "Netscape", "calendar": "gregory", "language": "en-US", "platform": "Linux x86_64", "timezone": "America/Los_Angeles", "client_ip": "104.9.125.144", "timestamp": "2025-02-28T11:34:39-08:00", "webrtc_ip": "172.56.209.195", "created_at": "2025-02-28T19:34:39.681872Z", "language_2": "en-US", "language_3": null, "platform_2": "Linux x86_64", "platform_3": null, "prefetched": true, "product_id": "d0606a0b5d034a8d81b1e3579d1f76fd", "webgl_flag": true, "webgl_hash": "da27e1b9b660057a3f5e185d3f5deabe", "canvas_hash": "e6e4d17da26050ce85ad00d3c6ea999e", "color_gamut": "3", "country_iso": "us", "inner_width": 841, "outer_width": 1680, "product_sub": "20030107", "audio_codecs": "1,1,1,1,1,3,1,3,1,3,3,1,1,3,3,3,3,1,3,3,3,2,1,1", "cookie_check": "", "graphics_api": "WebKit WebGL", "inner_height": 891, "mouse_events": null, "ontouchstart": false, "outer_height": 978, "plugins_hash": "4c05fa2f766a444d4f253ead792c8b0e|2", "screen_width": 1680, "video_codecs": "1,3,3,3,3,3,3,3,3,3,1,1,1,1,1,1,3,1,1,1,3,3,1", "webgl_hash_2": "fc73fd5db75e2c36222fe34251be3971", "webrtc_error": false, "window_opera": false, "battery_level": 0.41, "canvas_hash_2": "e0559d49b1864985cafc0d1c3a6b053c", "dynamic_range": false, "error_message": "Cannot read", "forced_colors": false, "math_result_1": "1.9275814160560204e-50", "math_result_2": "1.6182817135715877", "screen_height": 1050, "webgl_check_1": true, "webgl_context": "webgl2", "window_chrome": true, "connection_rtt": 100, "history_length": 11, "user_agent_str": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "web_sql_exists": false, "calender_locale": "en-US", "connection_type": "", "inverted_colors": true, "navigator_brave": false, "product_user_id": "test-unit", "request_headers": {"dnt": "1", "host": "127.0.0.1:8081", "accept": "application/json, lk/null q=0.1", "origin": "http://127.0.0.1:8080", "referer": "http://127.0.0.1:8080/", "sec-ch-ua": "\\"Google Chrome\\";v=\\"131\\", \\"Chromium\\";v=\\"131\\", \\"Not_A Brand\\";v=\\"24\\"", "connection": "keep-alive", "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "content-type": "application/json", "content-length": "3313", "sec-fetch-dest": "empty", "sec-fetch-mode": "cors", "sec-fetch-site": "same-site", "accept-encoding": "gzip, deflate, br, zstd", "accept-language": "en-US,en;q=0.9", "sec-ch-ua-mobile": "?0", "sec-ch-ua-platform": "\\"Linux\\""}, "timezone_offset": 480, "webrtc_local_ip": "10.253.217.45,[2607:fb91:20c5:c6af:cda0:10b4:830a:a85e]", "battery_charging": false, "client_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "104.9.125.144", "isp": "AT&T Internet", "latitude": 37.3897, "city_name": "Mountain View", "longitude": -122.083, "time_zone": "America/Los_Angeles", "user_type": "residential", "country_iso": "us", "postal_code": "94041", "is_anonymous": false, "accuracy_radius": 5, "static_ip_score": 40.3, "subdivision_1_iso": "CA", "subdivision_2_iso": null, "subdivision_1_name": "California", "subdivision_2_name": null, "registered_country_iso": "us"}, "max_touch_points": 0, "numbering_system": "latn", "path_fingerprint": 3252, "prefers_contrast": "0", "rendering_engine": "WebKit", "timezone_success": "pass", "user_agent_hints": {"model": null, "brands": [{"brand": "Google Chrome", "version": "131"}, {"brand": "Chromium", "version": "131"}, {"brand": "Not_A Brand", "version": "24"}], "mobile": false, "bitness": "64", "platform": "Linux", "brands_full": [{"brand": "Google Chrome", "version": "131.0.6778.204"}, {"brand": "Chromium", "version": "131.0.6778.204"}, {"brand": "Not_A Brand", "version": "24.0.0.0"}], "architecture": "x86", "platform_version": "6.2.0"}, "user_agent_str_2": null, "webgl_extensions": "EXT_clip_control|EXT_color_buffer_float|EXT_color_buffer_half_float|EXT_conservative_depth|EXT_depth_clamp|EXT_disjoint_timer_query_webgl2|EXT_float_blend|EXT_polygon_offset_clamp|EXT_render_snorm|EXT_texture_compression_bptc|EXT_texture_compression_rgtc|EXT_texture_filter_anisotropic|EXT_texture_mirror_clamp_to_edge|EXT_texture_norm16|KHR_parallel_shader_compile|NV_shader_noperspective_interpolation|OES_draw_buffers_indexed|OES_sample_variables|OES_shader_multisample_interpolation|OES_texture_float_linear|OVR_multiview2|WEBGL_blend_func_extended|WEBGL_clip_cull_distance|WEBGL_compressed_texture_astc|WEBGL_compressed_texture_etc|WEBGL_compressed_texture_etc1|WEBGL_compressed_texture_s3tc|WEBGL_compressed_texture_s3tc_srgb|WEBGL_debug_renderer_info|WEBGL_debug_shaders|WEBGL_lose_context|WEBGL_multi_draw|WEBGL_polygon_mode|WEBGL_provoking_vertex|WEBGL_stencil_texturing", "webrtc_ip_detail": {"continent_code": "EU", "continent_name": "Europe", "country_name": "France", "is_in_european_union": true, "ip": "172.56.209.195", "isp": null, "latitude": null, "city_name": null, "longitude": null, "time_zone": null, "user_type": null, "country_iso": "us", "postal_code": null, "is_anonymous": null, "accuracy_radius": null, "static_ip_score": null, "subdivision_1_iso": null, "subdivision_2_iso": null, "subdivision_1_name": null, "subdivision_2_name": null, "registered_country_iso": null}, "chrome_extensions": "", "execution_time_ms": 924.5, "graphics_renderer": "WebGL 2.0 (OpenGL ES 3.0 Chromium)", "keyboard_detected": true, "mime_types_length": 2, "request_fs_exists": true, "audio_context_flag": "pass", "audio_context_hash": "9307303774dec3248c18a939392090da", "canvas_fingerprint": 258, "canvas_pixel_check": false, "device_pixel_ratio": 1.0, "indexedDbData_blob": true, "navigator_keys_len": 79, "no_edge_pdf_plugin": false, "screen_avail_width": 1680, "webdriver_detected": false, "window_orientation": 0, "connection_downlink": 10.0, "navigator_webdriver": false, "non_native_function": false, "screen_avail_height": 1010, "supported_fonts_str": "72|17152|327680|1073741824|0|0|540736|73728|7340032|1342177280|117446657|256|16|0|262687|4290797636|1677723648|4168998400|0|1048576|262144|268500994|1342177280|262144|125829376|37888000|0|435363842|0|2147483648|109543680|1880099888|301989903", "text_2d_fingerprint": "bfcce91c9e71d11af7b14dbee4c75f83", "webrtc_is_supported": "pass", "canvas_support_level": "full", "do_not_track_enabled": "1", "hardware_concurrency": 12, "keyboard_layout_size": 48, "prefers_color_scheme": false, "webgl_max_anisotropy": 16, "battery_charging_time": 0.0, "browser_by_properties": "c", "eval_to_string_length": 33, "performance_loop_time": 0.09999999962747097, "session_storage_check": "pass", "unmasked_vendor_webgl": "Google Inc. (Intel)", "hardware_concurrency_2": 12, "hardware_concurrency_3": null, "localStorage_available": true, "memory_jsHeapSizeLimit": 4294705152, "mozilla_web_app_exists": false, "navigator_deviceMemory": 8.0, "navigator_java_enabled": false, "prefers_reduced_motion": false, "storage_estimate_quota": 1178717110272, "webdriver_detected_msg": "", "window_active_x_object": false, "window_external_exists": true, "color_depth_pixel_depth": "24-24", "indexedDbData_available": true, "navigator_cookieEnabled": true, "unmasked_renderer_webgl": "ANGLE (Intel, Mesa Intel(R) Graphics (RPL-P), OpenGL 4.6)", "battery_discharging_time": 4844.0, "connection_effectiveType": "4g", "non_native_function_flag": "", "speech_synthesis_voice_1": "Google Bahasa Indonesia", "window_client_information": true, "audio_compressor_reduction": 20.538288116455078, "navigator_mediaDevices_len": 8, "audio_intensity_fingerprint": 124.04347527516074, "speech_synthesis_voice_hash": "8010ee3313813de521e48e63bd5a6f13", "microsoft_credentials_exists": false, "window_installTrigger_exists": false, "speech_synthesis_voices_count": 19, "webgl_shading_language_version": "WebGL GLSL ES 3.00 (OpenGL ES GLSL ES 3.0 Chromium)", "error_message_stack_access_count": 2, "speech_synthesis_avail_voices_count": 19, "error_message_stack_access_count_worker": 2}""" + ), + "result_data": GrlIqCheckerResults.model_validate_json( + """{"uuid": "d54509f2f310499f8ab74839b10b2a41", "check_codecs": {"score": 0}, "check_timezone": {"score": 0}, "check_timestamp": {"score": 0}, "check_user_type": {"score": 0}, "check_ip_changes": {"score": 0}, "check_ip_country": {"score": 0}, "check_environment": {"msg": "error_message_stack_access_count: 2", "score": 100}, "check_ip_timezone": {"score": 0}, "check_isp_changes": {"score": 0}, "check_useragent_js": {"score": 0}, "check_required_fonts": {"score": 0}, "check_user_anonymous": {"score": 0}, "check_webrtc_success": {"score": 0}, "check_seen_timestamps": {"score": 0}, "check_country_timezone": {"score": 0}, "check_prohibited_fonts": {"score": 0}, "check_timezone_changes": {"score": 0}, "check_execution_time_ms": {"score": 0}, "check_fingerprint_reuse": {"score": 0}, "check_fingerprint_cycling": {"score": 0}, "check_ip_webrtc_ip_detail": {"score": 0}, "check_environment_critical": {"score": 0}, "check_useragent_other_enums": {"score": 0}, "check_useragent_ip_properties": {"score": 0}, "check_useragent_data_properties": {"score": 0}, "check_useragent_device_family_brand": {"score": 0}}""" + ), + "category_result": GrlIqForensicCategoryResult.model_validate_json( + """{"uuid": "d54509f2f310499f8ab74839b10b2a41", "is_bot": 0, "is_tampered": 0, "is_velocity": 0, "is_anonymous": 0, "suspicious_ip": 0, "is_oscillating": 0, "is_teleporting": 0, "is_inconsistent": 10, "platform_ip_inconsistent": 0}""" + ), + "fraud_score": 10, + "is_attempt_allowed": True, + }, + ] + + @pytest.fixture(scope="function") -def grliq_data() -> GrlIqData: - from generalresearch.grliq.managers import DUMMY_GRLIQ_DATA +def grliq_data(grliq_data_list: list[dict[str, Any]]) -> GrlIqData: - g: GrlIqData = DUMMY_GRLIQ_DATA[1]["data"] + g: GrlIqData = grliq_data_list[1]["data"] g.id = None g.uuid = uuid4().hex @@ -89,7 +124,9 @@ def grliq_data() -> GrlIqData: @pytest.fixture -def grliq_data_factory(grliq_dm: GrlIqDataManager) -> Callable[..., GrlIqData]: +def grliq_data_factory( + grliq_dm: GrlIqDataManager, grliq_data_list: list[dict[str, Any]] +) -> Callable[..., GrlIqData]: def _inner( is_attempt_allowed: bool = True, @@ -109,9 +146,8 @@ def grliq_data_factory(grliq_dm: GrlIqDataManager) -> Callable[..., GrlIqData]: :param mid: the thl_session:uuid / mid for the attempt. :return: """ - import copy - res: GrlIqData = copy.deepcopy(DUMMY_GRLIQ_DATA[int(is_attempt_allowed)]) + res: GrlIqData = grliq_data_list[int(is_attempt_allowed)]["data"] product_id = product_id or uuid4().hex product_user_id = product_user_id or uuid4().hex diff --git a/test_utils/incite/mergers/conftest.py b/test_utils/incite/mergers/conftest.py index c0f0bcf..1f88804 100644 --- a/test_utils/incite/mergers/conftest.py +++ b/test_utils/incite/mergers/conftest.py @@ -6,7 +6,7 @@ from datetime import datetime, timedelta import pytest from generalresearch.incite.base import GRLDatasets -from generalresearch.incite.mergers import MergeType +from generalresearch.incite.mergers.base import MergeType from generalresearch.incite.mergers.foundations.enriched_session import ( EnrichedSessionMerge, ) -- cgit v1.2.3 From ee97ff88beeb18dea2b630f6c2c805a6f260ad4f Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Sun, 30 Aug 2026 22:42:33 -0700 Subject: adding grip-client --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) (limited to 'pyproject.toml') diff --git a/pyproject.toml b/pyproject.toml index 94f5073..71ba993 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "scipy", "sentry-sdk", "slackclient", + "grip-client", "tldextract", "ua-parser", "user-agents", -- cgit v1.2.3 From df4445609608833f0c67d0633439ea2220966638 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Mon, 31 Aug 2026 12:45:23 -0700 Subject: adding dask pattern --- Jenkinsfile | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'pyproject.toml') diff --git a/Jenkinsfile b/Jenkinsfile index c44e57b..60838ca 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -54,7 +54,7 @@ pipeline { sh "/usr/local/bin/${PYTHON_VERSION} -m venv ${VENV}-${PYTHON_VERSION}" sh "${VENV}-${PYTHON_VERSION}/bin/pip install -U setuptools wheel pip" sh "${VENV}-${PYTHON_VERSION}/bin/pip install '.'" - sh "${VENV}-${PYTHON_VERSION}/bin/pip install '.[django]'" + sh "${VENV}-${PYTHON_VERSION}/bin/pip install '.[django,dask]'" } } } diff --git a/pyproject.toml b/pyproject.toml index 71ba993..5037085 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ ] [project.optional-dependencies] django = ["Django>=5.2", "psycopg>=3.1"] - +dask = ["dask>=2026.7.1", "distributed>=2026.7.1"] [tool.setuptools.packages.find] where = ["."] -- cgit v1.2.3 From 5598426e67f1a502aca68fd566d34216b2315482 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Mon, 31 Aug 2026 12:47:44 -0700 Subject: explicit adding pandas --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) (limited to 'pyproject.toml') diff --git a/pyproject.toml b/pyproject.toml index 5037085..bb23838 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dependencies = [ "limits", "more-itertools", "numpy", + "pandas", "pandera", "protobuf", "pyarrow", -- cgit v1.2.3 From f8f1f07b193845d92c7f6ef8ae95b9696db6330f Mon Sep 17 00:00:00 2001 From: stuppie Date: Fri, 4 Sep 2026 12:54:13 -0600 Subject: fix a lot of tests --- generalresearch/currency.py | 12 ++--- generalresearch/grliq/models/forensic_data.py | 14 ++--- .../incite/mergers/foundations/enriched_wall.py | 3 -- pyproject.toml | 2 +- test_utils/managers/gr/conftest.py | 2 + test_utils/managers/thl/conftest.py | 5 ++ test_utils/models/gr/conftest.py | 14 ++--- tests/conftest.py | 2 - tests/models/custom_types/test_dsn.py | 5 +- tests/models/gr/test_authentication.py | 63 ++++++++++------------ tests/models/gr/test_business.py | 25 ++++----- tests/models/gr/test_team.py | 8 ++- tests/models/test_finance.py | 14 ++--- .../thl/test_contest/test_leaderboard_contest.py | 8 +-- tests/models/thl/test_payout_format.py | 8 +-- tests/models/thl/test_product.py | 21 ++------ 16 files changed, 84 insertions(+), 122 deletions(-) (limited to 'pyproject.toml') diff --git a/generalresearch/currency.py b/generalresearch/currency.py index 716cb0f..7a9d037 100644 --- a/generalresearch/currency.py +++ b/generalresearch/currency.py @@ -29,12 +29,12 @@ class USDCent(int): if isinstance(value, float): warnings.warn( - "USDCent init with a float. Rounding behavior may " "be unexpected" + "USDCent init with a float. Rounding behavior may be unexpected" ) if isinstance(value, Decimal): warnings.warn( - "USDCent init with a Decimal. Rounding behavior may " "be unexpected" + "USDCent init with a Decimal. Rounding behavior may be unexpected" ) if value < 0: @@ -61,7 +61,7 @@ class USDCent(int): res = super().__abs__() return self.__class__(res) - def __truediv__(self): + def __truediv__(self, value): raise ValueError("Division not allowed for USDCent") def __str__(self): @@ -97,12 +97,12 @@ class USDMill(int): if isinstance(value, float): warnings.warn( - "USDMill init with a float. Rounding behavior " "may be unexpected" + "USDMill init with a float. Rounding behavior may be unexpected" ) if isinstance(value, Decimal): warnings.warn( - "USDMill init with a Decimal. Rounding behavior " "may be unexpected" + "USDMill init with a Decimal. Rounding behavior may be unexpected" ) if value < 0: @@ -129,7 +129,7 @@ class USDMill(int): res = super().__abs__() return self.__class__(res) - def __truediv__(self): + def __truediv__(self, value): raise ValueError("Division not allowed for USDMill") def __str__(self): diff --git a/generalresearch/grliq/models/forensic_data.py b/generalresearch/grliq/models/forensic_data.py index 6a07774..9d69e41 100644 --- a/generalresearch/grliq/models/forensic_data.py +++ b/generalresearch/grliq/models/forensic_data.py @@ -53,9 +53,9 @@ from generalresearch.models.custom_types import ( IPvAnyAddressStr, UUIDStr, ) +from generalresearch.models.thl.ipinfo import GeoIPInformation if TYPE_CHECKING: - from generalresearch.models.thl.ipinfo import GeoIPInformation from generalresearch.models.thl.session import Session fake = Faker() @@ -776,14 +776,14 @@ class GrlIqData(BaseModel): # product_id and product_user_id are parsed from the post body. make sure # they match the session whose mid was specified assert self.product_id == session.user.product_id, "product_id mismatch" - assert ( - self.product_user_id == session.user.product_user_id - ), "product_user_id mismatch" + assert self.product_user_id == session.user.product_user_id, ( + "product_user_id mismatch" + ) # validate the Session's mid is "recent" - assert (datetime.now(tz=UTC) - session.started) < timedelta( - minutes=90 - ), "expired session" + assert (datetime.now(tz=UTC) - session.started) < timedelta(minutes=90), ( + "expired session" + ) def model_dump_sql(self, **kwargs) -> dict[str, Any]: d = {} diff --git a/generalresearch/incite/mergers/foundations/enriched_wall.py b/generalresearch/incite/mergers/foundations/enriched_wall.py index 396c2be..70139c2 100644 --- a/generalresearch/incite/mergers/foundations/enriched_wall.py +++ b/generalresearch/incite/mergers/foundations/enriched_wall.py @@ -40,7 +40,6 @@ class EnrichedWallMergeItem(MergeCollectionItem): session_coll: SessionDFCollection, pg_config: PostgresConfig, client: Client | None = None, - client_resources: dict[str, Any] | None = None, ) -> None: ir: pd.Interval = self.interval @@ -160,7 +159,6 @@ class EnrichedWallMergeItem(MergeCollectionItem): ddf=ddf, is_partial=True, validate_after=False, - client_resources=client_resources, ) else: df = self.validate_df(df=df) @@ -169,7 +167,6 @@ class EnrichedWallMergeItem(MergeCollectionItem): client, ddf=ddf, is_partial=False, - client_resources=client_resources, ) diff --git a/pyproject.toml b/pyproject.toml index bb23838..13fa584 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,12 +9,12 @@ description = "Python Utilities for General Research" readme = "README.md" requires-python = ">=3.8" dependencies = [ - "fastapi", "Faker", "PyMySQL", "psycopg", "cachetools", "decorator", + "influxdb", "limits", "more-itertools", "numpy", diff --git a/test_utils/managers/gr/conftest.py b/test_utils/managers/gr/conftest.py index cc1053c..09e08f5 100644 --- a/test_utils/managers/gr/conftest.py +++ b/test_utils/managers/gr/conftest.py @@ -29,6 +29,8 @@ if TYPE_CHECKING: @pytest.fixture(scope="session") def gr_redis_config_db() -> str: + # need to update 'databases' in /etc/redis/redis.conf + # or this won't work and you'll have no indication why ... return str(randint(99, 1_023)) diff --git a/test_utils/managers/thl/conftest.py b/test_utils/managers/thl/conftest.py index 8ca4383..98dd574 100644 --- a/test_utils/managers/thl/conftest.py +++ b/test_utils/managers/thl/conftest.py @@ -92,6 +92,11 @@ def thl_redis_config( r.flushdb() +@pytest.fixture(scope="session") +def thl_redis_client(thl_redis_config): + return thl_redis_config.create_redis_client() + + @pytest.fixture(scope="session") def thl_web_rr(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: _dsn = django_db_factory("generalresearch.thl_django") diff --git a/test_utils/models/gr/conftest.py b/test_utils/models/gr/conftest.py index 1dbea0c..a48656b 100644 --- a/test_utils/models/gr/conftest.py +++ b/test_utils/models/gr/conftest.py @@ -12,7 +12,7 @@ from pydantic_extra_types.phone_numbers import PhoneNumber from generalresearch.models.custom_types import UUIDStr if TYPE_CHECKING: - from generalresearch.managers.gr.authentication import GRTokenManager, GRUserManager + from generalresearch.managers.gr.authentication import GRUserManager from generalresearch.managers.gr.business import ( BusinessAddressManager, BusinessBankAccountManager, @@ -289,9 +289,9 @@ def gr_user_token_factory( gr_user.prefetch_token(pg_config=gr_db) res = gr_user.token - assert ( - res is not None - ), "GRToken should exist after creation and prefetching" + assert res is not None, ( + "GRToken should exist after creation and prefetching" + ) return res else: @@ -335,8 +335,10 @@ def gr_membership_factory( @pytest.fixture() -def gr_membership(gr_membership_factory: Callable[..., Membership]) -> Membership: - return gr_membership_factory(save=True) +def gr_membership( + gr_membership_factory: Callable[..., Membership], gr_team: Team, gr_user: GRUser +) -> Membership: + return gr_membership_factory(gr_team=gr_team, gr_user=gr_user, save=True) @pytest.fixture() diff --git a/tests/conftest.py b/tests/conftest.py index 4777e15..b69d7ea 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -12,7 +12,6 @@ pytest_plugins = [ "test_utils.managers.contest.conftest", "test_utils.managers.gr.conftest", "test_utils.managers.ledger.conftest", - "test_utils.managers.network.conftest", "test_utils.managers.thl.conftest", "test_utils.managers.upk.conftest", # -- Models @@ -20,7 +19,6 @@ pytest_plugins = [ "test_utils.models.contest.conftest", "test_utils.models.gr.conftest", "test_utils.models.ledger.conftest", - "test_utils.models.network.conftest", "test_utils.models.thl.conftest", "test_utils.models.upk.conftest", # -- Marketplaces diff --git a/tests/models/custom_types/test_dsn.py b/tests/models/custom_types/test_dsn.py index 2aae579..eff02d3 100644 --- a/tests/models/custom_types/test_dsn.py +++ b/tests/models/custom_types/test_dsn.py @@ -1,14 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING from uuid import uuid4 import pytest from pydantic import BaseModel, Field, MySQLDsn, ValidationError from pydantic_core import Url -if TYPE_CHECKING: - from generalresearch.models.custom_types import DaskDsn, SentryDsn +from generalresearch.models.custom_types import DaskDsn, SentryDsn # --- Test Pydantic Models --- @@ -23,7 +21,6 @@ class SettingsModel(BaseModel): class TestDaskDsn: - def test_base(self): from dask.distributed import Client diff --git a/tests/models/gr/test_authentication.py b/tests/models/gr/test_authentication.py index 059a0a4..881571c 100644 --- a/tests/models/gr/test_authentication.py +++ b/tests/models/gr/test_authentication.py @@ -10,7 +10,6 @@ from typing import TYPE_CHECKING from uuid import uuid4 import pytest -from redis import Redis from generalresearch.models.gr.authentication import Claims, GRToken, GRUser from generalresearch.models.gr.team import Team @@ -26,7 +25,6 @@ SSO_ISSUER = "" class TestGRUser: - def test_init(self, gr_user: GRUser): assert isinstance(gr_user, GRUser) @@ -43,7 +41,7 @@ class TestGRUser: def test_teams( self, gr_user: GRUser, - membership: Membership, + gr_membership: Membership, gr_db: PostgresConfig, gr_redis_config: RedisConfig, ): @@ -60,16 +58,16 @@ class TestGRUser: self, gr_user_token: GRToken, gr_user: GRUser, - membership: Membership, + gr_membership: Membership, product_factory: Callable[..., Product], - membership_factory: Callable[..., Membership], - team: Team, + gr_membership_factory: Callable[..., Membership], + gr_team: Team, thl_web_rr: PostgresConfig, gr_redis_config: RedisConfig, gr_db: PostgresConfig, ): - product_factory(team=team) - membership_factory(team=team, gr_user=gr_user) + product_factory(team=gr_team) + gr_membership_factory(team=gr_team, gr_user=gr_user) gr_user.prefetch_teams( pg_config=gr_db, @@ -82,8 +80,8 @@ class TestGRUser: self, gr_user: GRUser, product_factory: Callable[..., Product], - team: Team, - membership: Membership, + gr_team: Team, + gr_membership: Membership, gr_db: PostgresConfig, thl_web_rr: PostgresConfig, gr_redis_config: RedisConfig, @@ -94,15 +92,15 @@ class TestGRUser: # Create a new Team membership, and then create a Product that # is part of that team - membership.prefetch_team(pg_config=gr_db, redis_config=gr_redis_config) - assert isinstance(membership.team, Team) + gr_membership.prefetch_team(pg_config=gr_db, redis_config=gr_redis_config) + assert isinstance(gr_membership.team, Team) - p: Product = product_factory(team=team) + p: Product = product_factory(team=gr_team) assert p.id_int - assert team.uuid == membership.team.uuid - assert p.team_id == team.uuid - assert p.team_uuid == membership.team.uuid - assert gr_user.id == membership.user_id + assert gr_team.uuid == gr_membership.team.uuid + assert p.team_id == gr_team.uuid + assert p.team_uuid == gr_membership.team.uuid + assert gr_user.id == gr_membership.user_id gr_user.prefetch_products( pg_config=gr_db, @@ -115,7 +113,6 @@ class TestGRUser: class TestGRUserMethods: - def test_cache_key(self, gr_user: GRUser): assert isinstance(gr_user.cache_key, str) assert ":" in gr_user.cache_key @@ -124,13 +121,13 @@ class TestGRUserMethods: def test_to_redis( self, gr_user: GRUser, - team: Team, + gr_team: Team, gr_business: Business, product_factory: Callable[..., Product], - membership_factory: Callable[..., Membership], + gr_membership_factory: Callable[..., Membership], ): - product_factory(team=team, business=gr_business) - membership_factory(team=team, gr_user=gr_user) + product_factory(team=gr_team, business=gr_business) + gr_membership_factory(team=gr_team, gr_user=gr_user) res = gr_user.to_redis() assert isinstance(res, str) @@ -171,16 +168,16 @@ class TestGRUserMethods: gr_db: PostgresConfig, thl_web_rr: PostgresConfig, product_factory: Callable[..., Product], - team: Team, - membership_factory: Callable[..., Membership], + gr_team: Team, + gr_membership_factory: Callable[..., Membership], thl_redis_config: RedisConfig, ): from generalresearch.models.gr.authentication import GRUser client = gr_redis_config.create_redis_client() - p1 = product_factory(team=team) - membership_factory(team=team, gr_user=gr_user) + p1 = product_factory(team=gr_team) + gr_membership_factory(team=gr_team, gr_user=gr_user) gr_user.set_cache( pg_config=gr_db, thl_web_rr=thl_web_rr, redis_config=gr_redis_config @@ -206,10 +203,10 @@ class TestGRUserMethods: gr_db: PostgresConfig, thl_web_rr: PostgresConfig, product_factory: Callable[..., Product], - team: Team, + gr_team: Team, gr_redis_config: RedisConfig, ): - product_factory(team=team) + product_factory(team=gr_team) client = gr_redis_config.create_redis_client() gr_user.set_cache( @@ -227,10 +224,10 @@ class TestGRUserMethods: thl_web_rr: PostgresConfig, product_factory: Callable[..., Product], gr_business: Business, - team: Team, + gr_team: Team, gr_redis_config: RedisConfig, ): - product_factory(team=team, business=gr_business) + product_factory(team=gr_team, business=gr_business) gr_user.set_cache( pg_config=gr_db, thl_web_rr=thl_web_rr, redis_config=gr_redis_config @@ -247,10 +244,10 @@ class TestGRUserMethods: gr_db: PostgresConfig, thl_web_rr: PostgresConfig, product_factory: Callable[..., Product], - team: Team, + gr_team: Team, gr_redis_config: RedisConfig, ): - product_factory(team=team) + product_factory(team=gr_team) gr_user.set_cache( pg_config=gr_db, thl_web_rr=thl_web_rr, redis_config=gr_redis_config @@ -262,7 +259,6 @@ class TestGRUserMethods: class TestGRToken: - @pytest.fixture def gr_token(self, gr_user: GRUser): now = datetime.now(tz=UTC) @@ -290,7 +286,6 @@ class TestGRToken: class TestClaims: - def test_init(self): d = { diff --git a/tests/models/gr/test_business.py b/tests/models/gr/test_business.py index 030a214..5d0de4f 100644 --- a/tests/models/gr/test_business.py +++ b/tests/models/gr/test_business.py @@ -119,11 +119,11 @@ class TestBusiness: def duration(self) -> timedelta | None: return None - def test_init(self, business: Business): + def test_init(self, gr_business: Business): - assert isinstance(business, Business) - assert isinstance(business.id, int) - assert isinstance(business.uuid, str) + assert isinstance(gr_business, Business) + assert isinstance(gr_business.id, int) + assert isinstance(gr_business.uuid, str) def test_str_and_repr( self, @@ -208,17 +208,17 @@ class TestBusiness: def test_addresses( self, - business: Business, + gr_business: Business, gr_db: PostgresConfig, ): from generalresearch.models.gr.business import BusinessAddress - assert business.addresses is None + assert gr_business.addresses is None - business.prefetch_addresses(pg_config=gr_db) - assert isinstance(business.addresses, list) - assert len(business.addresses) == 1 - assert isinstance(business.addresses[0], BusinessAddress) + gr_business.prefetch_addresses(pg_config=gr_db) + assert isinstance(gr_business.addresses, list) + assert len(gr_business.addresses) == 1 + assert isinstance(gr_business.addresses[0], BusinessAddress) def test_teams( self, @@ -674,8 +674,6 @@ class TestBusinessBalance: started=start + timedelta(days=2), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) - brokerage_product_payout_event_factory( product=u1.product, amount=USDCent(5), @@ -770,7 +768,6 @@ class TestBusinessBalance: wall_req_cpi=Decimal("2.50"), started=start + timedelta(days=2), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=u1.product, @@ -887,7 +884,6 @@ class TestBusinessBalance: wall_req_cpi=Decimal(".75"), started=start + timedelta(days=1), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=u1.product, amount=USDCent(71), @@ -1041,7 +1037,6 @@ class TestBusinessBalance: wall_req_cpi=Decimal("2.50"), started=start + timedelta(days=2), ) - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=u1.product, diff --git a/tests/models/gr/test_team.py b/tests/models/gr/test_team.py index 8ebedb6..b5f1781 100644 --- a/tests/models/gr/test_team.py +++ b/tests/models/gr/test_team.py @@ -42,7 +42,6 @@ if TYPE_CHECKING: class TestTeam: - def test_init(self, gr_team: Team): assert isinstance(gr_team, Team) @@ -54,7 +53,7 @@ class TestTeam: ): assert gr_team.memberships is None - gr_team.prefetch_memberships(membership_manager=gr_membership_manager) + gr_team.prefetch_memberships(gr_membership_manager=gr_membership_manager) assert isinstance(gr_team.memberships, list) assert len(gr_team.memberships) == 0 @@ -67,7 +66,7 @@ class TestTeam: ): assert gr_team.memberships is None - gr_team.prefetch_memberships(membership_manager=gr_membership_manager) + gr_team.prefetch_memberships(gr_membership_manager=gr_membership_manager) assert isinstance(gr_team.memberships, list) assert len(gr_team.memberships) == 1 assert gr_team.memberships[0].user_id == gr_user.id @@ -75,7 +74,7 @@ class TestTeam: # Create another new Membership gr_membership_manager.create(team=gr_team, gr_user=gr_user_factory()) assert len(gr_team.memberships) == 1 - gr_team.prefetch_memberships(membership_manager=gr_membership_manager) + gr_team.prefetch_memberships(gr_membership_manager=gr_membership_manager) assert len(gr_team.memberships) == 2 def test_gr_users( @@ -146,7 +145,6 @@ class TestTeam: class TestTeamMethods: - def test_cache_key(self, gr_team: Team): assert isinstance(gr_team.cache_key, str) assert ":" in gr_team.cache_key diff --git a/tests/models/test_finance.py b/tests/models/test_finance.py index 502c596..a1da961 100644 --- a/tests/models/test_finance.py +++ b/tests/models/test_finance.py @@ -13,9 +13,6 @@ import pytest from dask.distributed import Client as DaskClient # noinspection PyUnresolvedReferences -from distributed.utils_test import ( - client_no_amm, -) from faker import Faker from generalresearch.incite.schemas.mergers.pop_ledger import ( @@ -26,8 +23,6 @@ from generalresearch.models.thl.finance import ( POPFinancial, ProductBalances, ) -from test_utils.incite.collections.conftest import ledger_collection -from test_utils.incite.mergers.conftest import pop_ledger_merge if TYPE_CHECKING: from generalresearch.incite.collections.thl_web import LedgerDFCollection @@ -43,7 +38,6 @@ fake = Faker() class TestProductBalanceInitialize: - def test_unknown_fields(self): with pytest.raises(expected_exception=ValueError): ProductBalances.model_validate( @@ -251,7 +245,6 @@ class TestProductBalanceInitialize: class TestBusinessBalanceInitialize: - def test_validate_product_ids(self): instance1 = ProductBalances.model_validate( {"bp_payment.CREDIT": 500, "bp_adjustment.DEBIT": 40} @@ -668,9 +661,11 @@ class TestBusinessBalanceInitialize: ), ) class TestProductFinanceData: - def test_base( self, + ledger_collection: LedgerDFCollection, + pop_ledger_merge, + client_no_amm, duration: timedelta, product: Product, user_factory: Callable[..., User], @@ -681,9 +676,9 @@ class TestProductFinanceData: # -- Build & Setup u: User = user_factory(product=product, created=ledger_collection.start) + assert u.product for item in ledger_collection.items: - for _ in range(3): rand_item_time = fake.date_time_between( start_date=item.start, @@ -737,7 +732,6 @@ class TestProductFinanceData: class TestPOPFinancialData: - def test_base( self, client_no_amm: DaskClient, diff --git a/tests/models/thl/test_contest/test_leaderboard_contest.py b/tests/models/thl/test_contest/test_leaderboard_contest.py index c49776b..f787bdf 100644 --- a/tests/models/thl/test_contest/test_leaderboard_contest.py +++ b/tests/models/thl/test_contest/test_leaderboard_contest.py @@ -33,7 +33,7 @@ class TestLeaderboardContest(TestContest): @pytest.fixture def leaderboard_contest( - self, product: Product, thl_redis: Redis, user_manager: UserManager + self, product: Product, thl_redis_client: Redis, user_manager: UserManager ) -> LeaderboardContest: board_key = f"leaderboard:{product.uuid}:us:weekly:2025-05-26:complete_count" @@ -67,14 +67,14 @@ class TestLeaderboardContest(TestContest): ), ], ) - c._redis_client = thl_redis + c._redis_client = thl_redis_client c._user_manager = user_manager return c def test_init( self, leaderboard_contest: LeaderboardContest, - thl_redis: Redis, + thl_redis_client: Redis, user_1: User, user_2: User, ): @@ -82,7 +82,7 @@ class TestLeaderboardContest(TestContest): assert leaderboard_contest.end_condition.ends_at is not None lbm = LeaderboardManager( - redis_client=thl_redis, + redis_client=thl_redis_client, board_code=model.board_code, country_iso=model.country_iso, freq=model.freq, diff --git a/tests/models/thl/test_payout_format.py b/tests/models/thl/test_payout_format.py index 56eafe3..fe7aea5 100644 --- a/tests/models/thl/test_payout_format.py +++ b/tests/models/thl/test_payout_format.py @@ -1,20 +1,14 @@ from __future__ import annotations -from typing import TYPE_CHECKING - import pytest from pydantic import BaseModel from generalresearch.models.thl.payout_format import ( PayoutFormatField, + PayoutFormatType, format_payout_format, ) -if TYPE_CHECKING: - from generalresearch.models.thl.payout_format import ( - PayoutFormatType, - ) - class PayoutFormatTestClass(BaseModel): payout_format: PayoutFormatType = PayoutFormatField diff --git a/tests/models/thl/test_product.py b/tests/models/thl/test_product.py index 223430f..25affcf 100644 --- a/tests/models/thl/test_product.py +++ b/tests/models/thl/test_product.py @@ -48,7 +48,6 @@ if TYPE_CHECKING: class TestProduct: - def test_init(self): # By default, just a Pydantic instance doesn't have an id_int instance = Product.model_validate( @@ -70,13 +69,13 @@ class TestProduct: # By default, just a Pydantic instance doesn't have an id_int instance = product_factory() assert isinstance(instance.id_int, int) + assert isinstance(instance, Product) res = instance.model_dump_json() - assert isinstance(res, Product) # we json skip & exclude - res = instance.model_dump() - assert isinstance(res, Product) + p = Product.model_validate_json(res) + assert isinstance(p, Product) def test_redirect_url(self): p = Product.model_validate( @@ -150,12 +149,6 @@ class TestProduct: redirect_url="https://www.google.com/hey", ) - assert isinstance(p.payout_config.payout_transformation, PayoutTransformation) - assert isinstance( - p.payout_config.payout_transformation.kwargs, - PayoutTransformationPercentArgs, - ) - p.payout_config.payout_transformation = PayoutTransformation.model_validate( { "f": "payout_transformation_percent", @@ -598,7 +591,6 @@ class TestGlobalProductConfigFor: class TestProductFinancials: - @pytest.fixture def start(self) -> datetime: return datetime(year=2018, month=3, day=14, hour=0, tzinfo=UTC) @@ -639,7 +631,6 @@ class TestProductFinancials: u1: User = user_factory(product=p1) bp_wallet = thl_ledger_manager.get_account_or_create_bp_wallet(product=p1) thl_ledger_manager.get_account_or_create_user_wallet(user=u1) - brokerage_product_payout_event_manager.set_account_lookup_table(thl_lm=thl_lm) assert ( len( @@ -818,7 +809,6 @@ class TestProductFinancials: class TestProductBalance: - @pytest.fixture def start(self) -> datetime: return datetime(year=2018, month=3, day=14, hour=0, tzinfo=UTC) @@ -867,7 +857,6 @@ class TestProductBalance: pop_ledger_merge.build(client=client_no_amm, ledger_coll=ledger_collection) # 2. Payout and build Parquets 2nd time - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=product, amount=USDCent(71), @@ -928,7 +917,6 @@ class TestProductBalance: # 2. Payout and build Parquets 2nd time but this payout is "now" # so it hasn't already been archived - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=product, amount=USDCent(71), @@ -947,7 +935,6 @@ class TestProductBalance: class TestProductPOPFinancial: - @pytest.fixture def start(self) -> datetime: return datetime(year=2018, month=3, day=14, hour=0, tzinfo=UTC) @@ -1020,7 +1007,6 @@ class TestProductPOPFinancial: class TestProductCache: - @pytest.fixture def start(self) -> datetime: return datetime(year=2018, month=3, day=14, hour=0, tzinfo=UTC) @@ -1143,7 +1129,6 @@ class TestProductCache: ) # 2. Payout - payout_event_manager.set_account_lookup_table(thl_lm=thl_ledger_manager) brokerage_product_payout_event_factory( product=product, amount=USDCent(71), -- cgit v1.2.3