From 77e64ac954a7738b93a85fefde25fd6436e75737 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Thu, 20 Aug 2026 11:20:00 -0700 Subject: TestPostgresDjangoCreation, pytz to toml --- .gitignore | 3 +- Jenkinsfile | 41 ------------ generalresearch/managers/leaderboard/__init__.py | 4 +- generalresearch/models/custom_types.py | 14 +++-- pyproject.toml | 1 + test_utils/conftest.py | 79 +++++++++++++++--------- tests/models/custom_types/test_aware_datetime.py | 2 +- tests/test_postgres.py | 26 +++++++- 8 files changed, 91 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index c7c1d0b..db79a59 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ generalresearch/resources/brokerage_trust_calculated.csv tests/.env.test .env.* .DS_Store -build/ \ No newline at end of file +build/ +*.egg-info \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index a684caf..e829ba9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,47 +32,6 @@ pipeline { stages { stage('Setup DB') { - steps { - script { - env.DB_NAME = 'unittest-thl-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.THL_WEB_RW_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.DB_NAME}" - env.THL_WEB_RR_DB = env.THL_WEB_RW_DB - env.THL_WEB_RO_DB = env.THL_WEB_RW_DB - echo "Using database: ${env.DB_NAME}" - - env.SPECTRUM_DB_NAME = 'unittest-thl-spectrum-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.SPECTRUM_RW_DB = "mariadb://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_MARIA_HOST}/${env.SPECTRUM_DB_NAME}" - env.SPECTRUM_RR_DB = env.SPECTRUM_RW_DB - echo "Using database: ${env.SPECTRUM_DB_NAME}" - - env.GRLIQ_DB_NAME = 'unittest-grliq-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GRLIQ_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GRLIQ_DB_NAME}" - echo "Using database: ${env.GRLIQ_DB_NAME}" - - env.GR_DB_NAME = 'unittest-gr-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GR_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GR_DB_NAME}" - echo "Using database: ${env.GR_DB_NAME}" - } - - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres < str: InternalHostname = Annotated[str, AfterValidator(validate_hostname)] +class PostgresDict(MultiHostHost): + """The path part of this host, or `None`.""" + + # Databasename + name: str | None + + def convert_datetime_to_iso_8601_with_z_suffix(dt: datetime) -> str: # By default, datetimes are serialized with the %f optional. We don't # want that because then the deserialization fails if the datetime diff --git a/pyproject.toml b/pyproject.toml index 79b2382..dd2e649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pytest", "pylibmc", "pymemcache", + "pytz", "redis", "requests", "scipy", diff --git a/test_utils/conftest.py b/test_utils/conftest.py index c2c46c5..9c80065 100644 --- a/test_utils/conftest.py +++ b/test_utils/conftest.py @@ -11,9 +11,10 @@ import redis from _pytest.config import Config from dotenv import load_dotenv from pydantic import MariaDBDsn, PostgresDsn, TypeAdapter +from pydantic_core import MultiHostHost from redis import Redis -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig from generalresearch.sql_helper import SqlHelper @@ -115,32 +116,54 @@ def postgres_instance(settings: "GRLBaseSettings") -> Generator[PostgresDsn]: conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() - cur.execute(SQL("DROP DATABASE {}").format(Identifier(db_name))) + cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name))) cur.close() conn.close() -@pytest.fixture -def postgres_instance_host( +@pytest.fixture(scope="session") +def postgres_instance_dict( postgres_instance: PostgresDsn, -) -> Generator[InternalHostname]: - host = postgres_instance.hosts()[0]["host"] +) -> Generator[PostgresDict]: + host = postgres_instance.hosts()[0] assert host is not None + msg = "Must have full Postgres details" + assert host["host"], msg + assert host["username"], msg + assert host["password"], msg + + assert postgres_instance.path + + yield PostgresDict( + username=host["username"], + password=host["password"], + host=host["host"], + name=postgres_instance.path.lstrip("/"), + port=5432, + ) + + +@pytest.fixture(scope="session") +def postgres_instance_host( + postgres_instance_dict: PostgresDict, +) -> Generator[InternalHostname]: adapter = TypeAdapter(InternalHostname) - value = adapter.validate_python(host) + value = adapter.validate_python(postgres_instance_dict["host"]) yield value @pytest.fixture(scope="session") -def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: +def django_db_factory( + postgres_instance: PostgresDsn, postgres_instance_dict: PostgresDict +) -> Callable[..., PostgresDsn]: import django from django.apps import apps from django.conf import settings as django_settings from django.core.management import call_command - def _inner(): + def _inner(django_project: str = "generalresearch.thl_django"): # 1. Bootstrapping Django settings if not django_settings.configured: @@ -148,40 +171,38 @@ def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: DATABASES={ "default": { "ENGINE": "django.db.backends.postgresql", - # PostgresDsn stores path as "/dbname" - "NAME": str(postgres_instance.path).lstrip("/"), - "USER": postgres_instance["username"], - "PASSWORD": postgres_instance["password"], - "HOST": postgres_instance["host"], - "PORT": "5432", + "NAME": postgres_instance_dict["name"], + "USER": postgres_instance_dict["username"], + "PASSWORD": postgres_instance_dict["password"], + "HOST": postgres_instance_dict["host"], + "PORT": postgres_instance_dict["port"], } }, INSTALLED_APPS=[ "django.contrib.postgres", "django.contrib.contenttypes", - "generalresearch.thl_django", + django_project, ], ) django.setup() - for model in apps.get_models(): - print(f"Discovered model: {model._meta.label}") + # for model in apps.get_models(): + # print(f"Discovered model: {model._meta.label}") # 2. Run migrations directly during fixture activation call_command("migrate") + # 3. Return the Dsn so the factory gives a way to connect + return postgres_instance + return _inner @pytest.fixture(scope="session") -def thl_web_rr(postgres_instance: PostgresDsn, django_db_setup) -> PostgresConfig: - - # Run Migrations now. - # generalresearch/thl_django - django_db_setup() +def thl_web_rr(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: return PostgresConfig( - dsn=postgres_instance, + dsn=django_db_factory("generalresearch.thl_django"), connect_timeout=1, statement_timeout=5, ) @@ -193,11 +214,13 @@ def thl_web_rw(thl_web_rr: PostgresConfig) -> PostgresConfig: @pytest.fixture(scope="session") -def gr_db(postgres_instance: PostgresDsn) -> PostgresConfig: +def gr_db(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: - # Run Migrations, somehow pull from other repo... - django_db_setup() - return PostgresConfig(dsn=postgres_instance, connect_timeout=1, statement_timeout=5) + return PostgresConfig( + dsn=django_db_factory("gr_carer"), + connect_timeout=1, + statement_timeout=5, + ) @pytest.fixture(scope="session") diff --git a/tests/models/custom_types/test_aware_datetime.py b/tests/models/custom_types/test_aware_datetime.py index a23413c..14d1343 100644 --- a/tests/models/custom_types/test_aware_datetime.py +++ b/tests/models/custom_types/test_aware_datetime.py @@ -4,7 +4,7 @@ from typing import Optional import pytest import pytz -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError from generalresearch.models.custom_types import AwareDatetimeISO diff --git a/tests/test_postgres.py b/tests/test_postgres.py index 92eed71..3b3ddd0 100644 --- a/tests/test_postgres.py +++ b/tests/test_postgres.py @@ -1,9 +1,10 @@ import socket import subprocess +from typing import Callable from pydantic import PostgresDsn -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig @@ -42,3 +43,26 @@ class TestPostgresDSN: ) res = config.execute_sql_query(query="SELECT 1;") assert len(res) == 1 + + +class TestPostgresDjangoCreation: + + def test_ping(self, postgres_instance_dict: PostgresDict): + assert can_ping(host=postgres_instance_dict["host"]) + + def test_django_creation( + self, + django_db_factory: Callable[..., None], + ): + + dsn = django_db_factory() + assert isinstance(dsn, PostgresDsn) + + def test_django_tables(self, thl_web_rw: PostgresConfig): + res = thl_web_rw.execute_sql_query(query=""" + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + assert len(res) == 1 + assert res[0]["count"] == 56 -- cgit v1.2.3