From c6c439970d2167e7afce5f99fef11ab0126162e2 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Thu, 20 Aug 2026 09:30:19 -0700 Subject: postgresql auto creation into fixtures. simple tests for db conn checks. wrapper fixture for InternalHostnam (pydantic has MultiHost without host attr annoyances) --- tests/test_postgres.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tests/test_postgres.py (limited to 'tests/test_postgres.py') diff --git a/tests/test_postgres.py b/tests/test_postgres.py new file mode 100644 index 0000000..92eed71 --- /dev/null +++ b/tests/test_postgres.py @@ -0,0 +1,44 @@ +import socket +import subprocess + +from pydantic import PostgresDsn + +from generalresearch.models.custom_types import InternalHostname +from generalresearch.pg_helper import PostgresConfig + + +def is_port_open(host: InternalHostname, port: int = 5432, timeout: int = 3): + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except (socket.timeout, ConnectionRefusedError, OSError): + return False + + +def can_ping(host: InternalHostname): + return ( + subprocess.call( + ["ping", "-c", "1", str(host)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + == 0 + ) + + +class TestPostgresDSN: + + def test_ping(self, postgres_instance_host: InternalHostname): + assert can_ping(host=postgres_instance_host) + + def test_port(self, postgres_instance_host: InternalHostname): + assert is_port_open(host=postgres_instance_host) + + def test_conn(self, postgres_instance: PostgresDsn): + config = PostgresConfig( + dsn=postgres_instance, + connect_timeout=1, + statement_timeout=1, + ) + res = config.execute_sql_query(query="SELECT 1;") + assert len(res) == 1 -- cgit v1.2.3 From 77e64ac954a7738b93a85fefde25fd6436e75737 Mon Sep 17 00:00:00 2001 From: Max Nanis Date: Thu, 20 Aug 2026 11:20:00 -0700 Subject: TestPostgresDjangoCreation, pytz to toml --- .gitignore | 3 +- Jenkinsfile | 41 ------------ generalresearch/managers/leaderboard/__init__.py | 4 +- generalresearch/models/custom_types.py | 14 +++-- pyproject.toml | 1 + test_utils/conftest.py | 79 +++++++++++++++--------- tests/models/custom_types/test_aware_datetime.py | 2 +- tests/test_postgres.py | 26 +++++++- 8 files changed, 91 insertions(+), 79 deletions(-) (limited to 'tests/test_postgres.py') diff --git a/.gitignore b/.gitignore index c7c1d0b..db79a59 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ generalresearch/resources/brokerage_trust_calculated.csv tests/.env.test .env.* .DS_Store -build/ \ No newline at end of file +build/ +*.egg-info \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index a684caf..e829ba9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,47 +32,6 @@ pipeline { stages { stage('Setup DB') { - steps { - script { - env.DB_NAME = 'unittest-thl-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.THL_WEB_RW_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.DB_NAME}" - env.THL_WEB_RR_DB = env.THL_WEB_RW_DB - env.THL_WEB_RO_DB = env.THL_WEB_RW_DB - echo "Using database: ${env.DB_NAME}" - - env.SPECTRUM_DB_NAME = 'unittest-thl-spectrum-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.SPECTRUM_RW_DB = "mariadb://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_MARIA_HOST}/${env.SPECTRUM_DB_NAME}" - env.SPECTRUM_RR_DB = env.SPECTRUM_RW_DB - echo "Using database: ${env.SPECTRUM_DB_NAME}" - - env.GRLIQ_DB_NAME = 'unittest-grliq-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GRLIQ_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GRLIQ_DB_NAME}" - echo "Using database: ${env.GRLIQ_DB_NAME}" - - env.GR_DB_NAME = 'unittest-gr-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GR_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GR_DB_NAME}" - echo "Using database: ${env.GR_DB_NAME}" - } - - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres < str: InternalHostname = Annotated[str, AfterValidator(validate_hostname)] +class PostgresDict(MultiHostHost): + """The path part of this host, or `None`.""" + + # Databasename + name: str | None + + def convert_datetime_to_iso_8601_with_z_suffix(dt: datetime) -> str: # By default, datetimes are serialized with the %f optional. We don't # want that because then the deserialization fails if the datetime diff --git a/pyproject.toml b/pyproject.toml index 79b2382..dd2e649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pytest", "pylibmc", "pymemcache", + "pytz", "redis", "requests", "scipy", diff --git a/test_utils/conftest.py b/test_utils/conftest.py index c2c46c5..9c80065 100644 --- a/test_utils/conftest.py +++ b/test_utils/conftest.py @@ -11,9 +11,10 @@ import redis from _pytest.config import Config from dotenv import load_dotenv from pydantic import MariaDBDsn, PostgresDsn, TypeAdapter +from pydantic_core import MultiHostHost from redis import Redis -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig from generalresearch.sql_helper import SqlHelper @@ -115,32 +116,54 @@ def postgres_instance(settings: "GRLBaseSettings") -> Generator[PostgresDsn]: conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() - cur.execute(SQL("DROP DATABASE {}").format(Identifier(db_name))) + cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name))) cur.close() conn.close() -@pytest.fixture -def postgres_instance_host( +@pytest.fixture(scope="session") +def postgres_instance_dict( postgres_instance: PostgresDsn, -) -> Generator[InternalHostname]: - host = postgres_instance.hosts()[0]["host"] +) -> Generator[PostgresDict]: + host = postgres_instance.hosts()[0] assert host is not None + msg = "Must have full Postgres details" + assert host["host"], msg + assert host["username"], msg + assert host["password"], msg + + assert postgres_instance.path + + yield PostgresDict( + username=host["username"], + password=host["password"], + host=host["host"], + name=postgres_instance.path.lstrip("/"), + port=5432, + ) + + +@pytest.fixture(scope="session") +def postgres_instance_host( + postgres_instance_dict: PostgresDict, +) -> Generator[InternalHostname]: adapter = TypeAdapter(InternalHostname) - value = adapter.validate_python(host) + value = adapter.validate_python(postgres_instance_dict["host"]) yield value @pytest.fixture(scope="session") -def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: +def django_db_factory( + postgres_instance: PostgresDsn, postgres_instance_dict: PostgresDict +) -> Callable[..., PostgresDsn]: import django from django.apps import apps from django.conf import settings as django_settings from django.core.management import call_command - def _inner(): + def _inner(django_project: str = "generalresearch.thl_django"): # 1. Bootstrapping Django settings if not django_settings.configured: @@ -148,40 +171,38 @@ def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: DATABASES={ "default": { "ENGINE": "django.db.backends.postgresql", - # PostgresDsn stores path as "/dbname" - "NAME": str(postgres_instance.path).lstrip("/"), - "USER": postgres_instance["username"], - "PASSWORD": postgres_instance["password"], - "HOST": postgres_instance["host"], - "PORT": "5432", + "NAME": postgres_instance_dict["name"], + "USER": postgres_instance_dict["username"], + "PASSWORD": postgres_instance_dict["password"], + "HOST": postgres_instance_dict["host"], + "PORT": postgres_instance_dict["port"], } }, INSTALLED_APPS=[ "django.contrib.postgres", "django.contrib.contenttypes", - "generalresearch.thl_django", + django_project, ], ) django.setup() - for model in apps.get_models(): - print(f"Discovered model: {model._meta.label}") + # for model in apps.get_models(): + # print(f"Discovered model: {model._meta.label}") # 2. Run migrations directly during fixture activation call_command("migrate") + # 3. Return the Dsn so the factory gives a way to connect + return postgres_instance + return _inner @pytest.fixture(scope="session") -def thl_web_rr(postgres_instance: PostgresDsn, django_db_setup) -> PostgresConfig: - - # Run Migrations now. - # generalresearch/thl_django - django_db_setup() +def thl_web_rr(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: return PostgresConfig( - dsn=postgres_instance, + dsn=django_db_factory("generalresearch.thl_django"), connect_timeout=1, statement_timeout=5, ) @@ -193,11 +214,13 @@ def thl_web_rw(thl_web_rr: PostgresConfig) -> PostgresConfig: @pytest.fixture(scope="session") -def gr_db(postgres_instance: PostgresDsn) -> PostgresConfig: +def gr_db(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: - # Run Migrations, somehow pull from other repo... - django_db_setup() - return PostgresConfig(dsn=postgres_instance, connect_timeout=1, statement_timeout=5) + return PostgresConfig( + dsn=django_db_factory("gr_carer"), + connect_timeout=1, + statement_timeout=5, + ) @pytest.fixture(scope="session") diff --git a/tests/models/custom_types/test_aware_datetime.py b/tests/models/custom_types/test_aware_datetime.py index a23413c..14d1343 100644 --- a/tests/models/custom_types/test_aware_datetime.py +++ b/tests/models/custom_types/test_aware_datetime.py @@ -4,7 +4,7 @@ from typing import Optional import pytest import pytz -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError from generalresearch.models.custom_types import AwareDatetimeISO diff --git a/tests/test_postgres.py b/tests/test_postgres.py index 92eed71..3b3ddd0 100644 --- a/tests/test_postgres.py +++ b/tests/test_postgres.py @@ -1,9 +1,10 @@ import socket import subprocess +from typing import Callable from pydantic import PostgresDsn -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig @@ -42,3 +43,26 @@ class TestPostgresDSN: ) res = config.execute_sql_query(query="SELECT 1;") assert len(res) == 1 + + +class TestPostgresDjangoCreation: + + def test_ping(self, postgres_instance_dict: PostgresDict): + assert can_ping(host=postgres_instance_dict["host"]) + + def test_django_creation( + self, + django_db_factory: Callable[..., None], + ): + + dsn = django_db_factory() + assert isinstance(dsn, PostgresDsn) + + def test_django_tables(self, thl_web_rw: PostgresConfig): + res = thl_web_rw.execute_sql_query(query=""" + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + assert len(res) == 1 + assert res[0]["count"] == 56 -- cgit v1.2.3