diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | Jenkinsfile | 41 | ||||
| -rw-r--r-- | generalresearch/managers/leaderboard/__init__.py | 4 | ||||
| -rw-r--r-- | generalresearch/models/custom_types.py | 14 | ||||
| -rw-r--r-- | pyproject.toml | 1 | ||||
| -rw-r--r-- | test_utils/conftest.py | 79 | ||||
| -rw-r--r-- | tests/models/custom_types/test_aware_datetime.py | 2 | ||||
| -rw-r--r-- | tests/test_postgres.py | 26 |
8 files changed, 91 insertions, 79 deletions
@@ -8,4 +8,5 @@ generalresearch/resources/brokerage_trust_calculated.csv tests/.env.test .env.* .DS_Store -build/
\ No newline at end of file +build/ +*.egg-info
\ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index a684caf..e829ba9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,47 +32,6 @@ pipeline { stages { stage('Setup DB') { - steps { - script { - env.DB_NAME = 'unittest-thl-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.THL_WEB_RW_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.DB_NAME}" - env.THL_WEB_RR_DB = env.THL_WEB_RW_DB - env.THL_WEB_RO_DB = env.THL_WEB_RW_DB - echo "Using database: ${env.DB_NAME}" - - env.SPECTRUM_DB_NAME = 'unittest-thl-spectrum-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.SPECTRUM_RW_DB = "mariadb://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_MARIA_HOST}/${env.SPECTRUM_DB_NAME}" - env.SPECTRUM_RR_DB = env.SPECTRUM_RW_DB - echo "Using database: ${env.SPECTRUM_DB_NAME}" - - env.GRLIQ_DB_NAME = 'unittest-grliq-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GRLIQ_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GRLIQ_DB_NAME}" - echo "Using database: ${env.GRLIQ_DB_NAME}" - - env.GR_DB_NAME = 'unittest-gr-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GR_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GR_DB_NAME}" - echo "Using database: ${env.GR_DB_NAME}" - } - - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.GRLIQ_DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.GR_DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - mysql -h ${env.DB_MARIA_HOST} -u ${env.DB_USER} -p${env.DB_PASSWORD} --ssl=0 -e 'CREATE DATABASE `${env.SPECTRUM_DB_NAME}`;' - """ - script { env.REDIS_DB = new Random().nextInt(1024).toString() env.REDIS = "${env.REDIS}:6379/${env.REDIS_DB}" diff --git a/generalresearch/managers/leaderboard/__init__.py b/generalresearch/managers/leaderboard/__init__.py index 8468cdc..aae4a05 100644 --- a/generalresearch/managers/leaderboard/__init__.py +++ b/generalresearch/managers/leaderboard/__init__.py @@ -1,8 +1,8 @@ from typing import Dict -from zoneinfo import ZoneInfo import pytz -from cachetools import cached, LRUCache +from cachetools import LRUCache, cached +from zoneinfo import ZoneInfo @cached(cache=LRUCache(maxsize=1)) diff --git a/generalresearch/models/custom_types.py b/generalresearch/models/custom_types.py index 2f40dbb..84bf8e3 100644 --- a/generalresearch/models/custom_types.py +++ b/generalresearch/models/custom_types.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import re +import sys as _sys from datetime import datetime, timedelta, timezone from typing import Any, Literal from uuid import UUID @@ -18,15 +19,11 @@ from pydantic import ( from pydantic.functional_serializers import PlainSerializer from pydantic.functional_validators import AfterValidator, BeforeValidator from pydantic.networks import IPvAnyNetwork, UrlConstraints -from pydantic_core import Url +from pydantic_core import MultiHostHost, Url from typing_extensions import Annotated from generalresearch.models import DeviceType, Source -# if TYPE_CHECKING: -# from generalresearch.models import DeviceType - - HOSTNAME_REGEX = re.compile( r"^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" ) @@ -41,6 +38,13 @@ def validate_hostname(v: str) -> str: InternalHostname = Annotated[str, AfterValidator(validate_hostname)] +class PostgresDict(MultiHostHost): + """The path part of this host, or `None`.""" + + # Databasename + name: str | None + + def convert_datetime_to_iso_8601_with_z_suffix(dt: datetime) -> str: # By default, datetimes are serialized with the %f optional. We don't # want that because then the deserialization fails if the datetime diff --git a/pyproject.toml b/pyproject.toml index 79b2382..dd2e649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pytest", "pylibmc", "pymemcache", + "pytz", "redis", "requests", "scipy", diff --git a/test_utils/conftest.py b/test_utils/conftest.py index c2c46c5..9c80065 100644 --- a/test_utils/conftest.py +++ b/test_utils/conftest.py @@ -11,9 +11,10 @@ import redis from _pytest.config import Config from dotenv import load_dotenv from pydantic import MariaDBDsn, PostgresDsn, TypeAdapter +from pydantic_core import MultiHostHost from redis import Redis -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig from generalresearch.sql_helper import SqlHelper @@ -115,32 +116,54 @@ def postgres_instance(settings: "GRLBaseSettings") -> Generator[PostgresDsn]: conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() - cur.execute(SQL("DROP DATABASE {}").format(Identifier(db_name))) + cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name))) cur.close() conn.close() -@pytest.fixture -def postgres_instance_host( +@pytest.fixture(scope="session") +def postgres_instance_dict( postgres_instance: PostgresDsn, -) -> Generator[InternalHostname]: - host = postgres_instance.hosts()[0]["host"] +) -> Generator[PostgresDict]: + host = postgres_instance.hosts()[0] assert host is not None + msg = "Must have full Postgres details" + assert host["host"], msg + assert host["username"], msg + assert host["password"], msg + + assert postgres_instance.path + + yield PostgresDict( + username=host["username"], + password=host["password"], + host=host["host"], + name=postgres_instance.path.lstrip("/"), + port=5432, + ) + + +@pytest.fixture(scope="session") +def postgres_instance_host( + postgres_instance_dict: PostgresDict, +) -> Generator[InternalHostname]: adapter = TypeAdapter(InternalHostname) - value = adapter.validate_python(host) + value = adapter.validate_python(postgres_instance_dict["host"]) yield value @pytest.fixture(scope="session") -def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: +def django_db_factory( + postgres_instance: PostgresDsn, postgres_instance_dict: PostgresDict +) -> Callable[..., PostgresDsn]: import django from django.apps import apps from django.conf import settings as django_settings from django.core.management import call_command - def _inner(): + def _inner(django_project: str = "generalresearch.thl_django"): # 1. Bootstrapping Django settings if not django_settings.configured: @@ -148,40 +171,38 @@ def django_db_setup(postgres_instance: PostgresDsn) -> Callable[..., None]: DATABASES={ "default": { "ENGINE": "django.db.backends.postgresql", - # PostgresDsn stores path as "/dbname" - "NAME": str(postgres_instance.path).lstrip("/"), - "USER": postgres_instance["username"], - "PASSWORD": postgres_instance["password"], - "HOST": postgres_instance["host"], - "PORT": "5432", + "NAME": postgres_instance_dict["name"], + "USER": postgres_instance_dict["username"], + "PASSWORD": postgres_instance_dict["password"], + "HOST": postgres_instance_dict["host"], + "PORT": postgres_instance_dict["port"], } }, INSTALLED_APPS=[ "django.contrib.postgres", "django.contrib.contenttypes", - "generalresearch.thl_django", + django_project, ], ) django.setup() - for model in apps.get_models(): - print(f"Discovered model: {model._meta.label}") + # for model in apps.get_models(): + # print(f"Discovered model: {model._meta.label}") # 2. Run migrations directly during fixture activation call_command("migrate") + # 3. Return the Dsn so the factory gives a way to connect + return postgres_instance + return _inner @pytest.fixture(scope="session") -def thl_web_rr(postgres_instance: PostgresDsn, django_db_setup) -> PostgresConfig: - - # Run Migrations now. - # generalresearch/thl_django - django_db_setup() +def thl_web_rr(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: return PostgresConfig( - dsn=postgres_instance, + dsn=django_db_factory("generalresearch.thl_django"), connect_timeout=1, statement_timeout=5, ) @@ -193,11 +214,13 @@ def thl_web_rw(thl_web_rr: PostgresConfig) -> PostgresConfig: @pytest.fixture(scope="session") -def gr_db(postgres_instance: PostgresDsn) -> PostgresConfig: +def gr_db(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: - # Run Migrations, somehow pull from other repo... - django_db_setup() - return PostgresConfig(dsn=postgres_instance, connect_timeout=1, statement_timeout=5) + return PostgresConfig( + dsn=django_db_factory("gr_carer"), + connect_timeout=1, + statement_timeout=5, + ) @pytest.fixture(scope="session") diff --git a/tests/models/custom_types/test_aware_datetime.py b/tests/models/custom_types/test_aware_datetime.py index a23413c..14d1343 100644 --- a/tests/models/custom_types/test_aware_datetime.py +++ b/tests/models/custom_types/test_aware_datetime.py @@ -4,7 +4,7 @@ from typing import Optional import pytest import pytz -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError from generalresearch.models.custom_types import AwareDatetimeISO diff --git a/tests/test_postgres.py b/tests/test_postgres.py index 92eed71..3b3ddd0 100644 --- a/tests/test_postgres.py +++ b/tests/test_postgres.py @@ -1,9 +1,10 @@ import socket import subprocess +from typing import Callable from pydantic import PostgresDsn -from generalresearch.models.custom_types import InternalHostname +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig @@ -42,3 +43,26 @@ class TestPostgresDSN: ) res = config.execute_sql_query(query="SELECT 1;") assert len(res) == 1 + + +class TestPostgresDjangoCreation: + + def test_ping(self, postgres_instance_dict: PostgresDict): + assert can_ping(host=postgres_instance_dict["host"]) + + def test_django_creation( + self, + django_db_factory: Callable[..., None], + ): + + dsn = django_db_factory() + assert isinstance(dsn, PostgresDsn) + + def test_django_tables(self, thl_web_rw: PostgresConfig): + res = thl_web_rw.execute_sql_query(query=""" + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + assert len(res) == 1 + assert res[0]["count"] == 56 |