diff options
| -rw-r--r-- | .gitignore | 3 | ||||
| -rw-r--r-- | Jenkinsfile | 41 | ||||
| -rw-r--r-- | generalresearch/config.py | 18 | ||||
| -rw-r--r-- | generalresearch/managers/leaderboard/__init__.py | 4 | ||||
| -rw-r--r-- | generalresearch/models/custom_types.py | 41 | ||||
| -rw-r--r-- | pyproject.toml | 1 | ||||
| -rw-r--r-- | test_utils/conftest.py | 205 | ||||
| -rw-r--r-- | test_utils/incite/conftest.py | 3 | ||||
| -rw-r--r-- | tests/incite/collections/test_df_collection_item_thl_web.py | 26 | ||||
| -rw-r--r-- | tests/incite/collections/test_df_collection_thl_marketplaces.py | 2 | ||||
| -rw-r--r-- | tests/incite/collections/test_df_collection_thl_web.py | 27 | ||||
| -rw-r--r-- | tests/models/custom_types/test_aware_datetime.py | 2 | ||||
| -rw-r--r-- | tests/pytest.ini | 3 | ||||
| -rw-r--r-- | tests/test_postgres.py | 68 |
14 files changed, 255 insertions, 189 deletions
@@ -8,4 +8,5 @@ generalresearch/resources/brokerage_trust_calculated.csv tests/.env.test .env.* .DS_Store -build/
\ No newline at end of file +build/ +*.egg-info
\ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index a684caf..e829ba9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,47 +32,6 @@ pipeline { stages { stage('Setup DB') { - steps { - script { - env.DB_NAME = 'unittest-thl-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.THL_WEB_RW_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.DB_NAME}" - env.THL_WEB_RR_DB = env.THL_WEB_RW_DB - env.THL_WEB_RO_DB = env.THL_WEB_RW_DB - echo "Using database: ${env.DB_NAME}" - - env.SPECTRUM_DB_NAME = 'unittest-thl-spectrum-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.SPECTRUM_RW_DB = "mariadb://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_MARIA_HOST}/${env.SPECTRUM_DB_NAME}" - env.SPECTRUM_RR_DB = env.SPECTRUM_RW_DB - echo "Using database: ${env.SPECTRUM_DB_NAME}" - - env.GRLIQ_DB_NAME = 'unittest-grliq-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GRLIQ_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GRLIQ_DB_NAME}" - echo "Using database: ${env.GRLIQ_DB_NAME}" - - env.GR_DB_NAME = 'unittest-gr-' + UUID.randomUUID().toString().replace('-', '').take(12) - env.GR_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.GR_DB_NAME}" - echo "Using database: ${env.GR_DB_NAME}" - } - - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.GRLIQ_DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF - CREATE DATABASE "${env.GR_DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8'; - EOF - """ - sh """ - mysql -h ${env.DB_MARIA_HOST} -u ${env.DB_USER} -p${env.DB_PASSWORD} --ssl=0 -e 'CREATE DATABASE `${env.SPECTRUM_DB_NAME}`;' - """ - script { env.REDIS_DB = new Random().nextInt(1024).toString() env.REDIS = "${env.REDIS}:6379/${env.REDIS_DB}" diff --git a/generalresearch/config.py b/generalresearch/config.py index 551f75a..af80069 100644 --- a/generalresearch/config.py +++ b/generalresearch/config.py @@ -5,9 +5,9 @@ from datetime import datetime, timezone from pathlib import Path from pydantic import DirectoryPath, Field, MariaDBDsn, PostgresDsn, RedisDsn -from pydantic_settings import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict -from generalresearch.models.custom_types import DaskDsn, SentryDsn +from generalresearch.models.custom_types import DaskDsn, InternalHostname, SentryDsn os.environ["DISABLE_PANDERA_IMPORT_WARNING"] = "True" @@ -39,8 +39,22 @@ def is_debug() -> bool: class GRLBaseSettings(BaseSettings): + model_config = SettingsConfigDict( + env_file=(".env.test", ".env.testing", ".env.staging", ".env.prod"), + env_file_encoding="utf-8", + extra="allow", + ) + debug: bool = Field(default=True) + # --- Pytest --- + + testing_postgres: InternalHostname | None = Field(default=None) + testing_postgres_user: str | None = Field(default=None) + testing_postgres_pass: str | None = Field(default=None) + + # --- + redis: RedisDsn | None = Field(default=None) redis_timeout: float = Field(default=0.10) diff --git a/generalresearch/managers/leaderboard/__init__.py b/generalresearch/managers/leaderboard/__init__.py index 8468cdc..aae4a05 100644 --- a/generalresearch/managers/leaderboard/__init__.py +++ b/generalresearch/managers/leaderboard/__init__.py @@ -1,8 +1,8 @@ from typing import Dict -from zoneinfo import ZoneInfo import pytz -from cachetools import cached, LRUCache +from cachetools import LRUCache, cached +from zoneinfo import ZoneInfo @cached(cache=LRUCache(maxsize=1)) diff --git a/generalresearch/models/custom_types.py b/generalresearch/models/custom_types.py index ea96741..84bf8e3 100644 --- a/generalresearch/models/custom_types.py +++ b/generalresearch/models/custom_types.py @@ -1,6 +1,10 @@ +from __future__ import annotations + import json +import re +import sys as _sys from datetime import datetime, timedelta, timezone -from typing import Any, Literal, Optional, Set +from typing import Any, Literal from uuid import UUID from pydantic import ( @@ -14,14 +18,31 @@ from pydantic import ( ) from pydantic.functional_serializers import PlainSerializer from pydantic.functional_validators import AfterValidator, BeforeValidator -from pydantic.networks import UrlConstraints, IPvAnyNetwork -from pydantic_core import Url +from pydantic.networks import IPvAnyNetwork, UrlConstraints +from pydantic_core import MultiHostHost, Url from typing_extensions import Annotated from generalresearch.models import DeviceType, Source -# if TYPE_CHECKING: -# from generalresearch.models import DeviceType +HOSTNAME_REGEX = re.compile( + r"^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(\.[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" +) + + +def validate_hostname(v: str) -> str: + if not HOSTNAME_REGEX.match(v): + raise ValueError("Invalid internal hostname format") + return v + + +InternalHostname = Annotated[str, AfterValidator(validate_hostname)] + + +class PostgresDict(MultiHostHost): + """The path part of this host, or `None`.""" + + # Databasename + name: str | None def convert_datetime_to_iso_8601_with_z_suffix(dt: datetime) -> str: @@ -31,7 +52,7 @@ def convert_datetime_to_iso_8601_with_z_suffix(dt: datetime) -> str: return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") -def convert_str_dt(v: Any) -> Optional[AwareDatetime]: +def convert_str_dt(v: Any) -> AwareDatetime | None: # By default, pydantic is unable to handle tz-aware isoformat str. Attempt # to parse a str that was dumped using the iso8601 format with Z suffix. if v is not None and type(v) is str: @@ -158,7 +179,7 @@ from_comma_sep_str = BeforeValidator( # This is a set of DeviceType, that serializes and de-serializes into a # (sorted) comma-separated str -DeviceTypes = Annotated[Set[DeviceType], enum_to_comma_sep_str, from_comma_sep_str] +DeviceTypes = Annotated[set[DeviceType], enum_to_comma_sep_str, from_comma_sep_str] # This is a set of alphanumeric strings, that serializes and de-serializes # into a (sorted) comma-separated str @@ -223,9 +244,9 @@ InfluxDsn = Annotated[ ), ] -AlphaNumStrSet = Annotated[Set[AlphaNumStr], to_comma_sep_str, from_comma_sep_str] -IPLikeStrSet = Annotated[Set[IPLikeStr], to_comma_sep_str, from_comma_sep_str] -UUIDStrSet = Annotated[Set[UUIDStr], to_comma_sep_str, from_comma_sep_str] +AlphaNumStrSet = Annotated[set[AlphaNumStr], to_comma_sep_str, from_comma_sep_str] +IPLikeStrSet = Annotated[set[IPLikeStr], to_comma_sep_str, from_comma_sep_str] +UUIDStrSet = Annotated[set[UUIDStr], to_comma_sep_str, from_comma_sep_str] list_models_to_json_str = PlainSerializer( lambda x: json.dumps([y.model_dump(mode="json") for y in x]), diff --git a/pyproject.toml b/pyproject.toml index 79b2382..dd2e649 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "pytest", "pylibmc", "pymemcache", + "pytz", "redis", "requests", "scipy", diff --git a/test_utils/conftest.py b/test_utils/conftest.py index 232c1fc..9c80065 100644 --- a/test_utils/conftest.py +++ b/test_utils/conftest.py @@ -1,22 +1,20 @@ import os import shutil -import sys from datetime import datetime, timedelta, timezone from os.path import join as pjoin from pathlib import Path from typing import TYPE_CHECKING, Callable, Generator from uuid import uuid4 -import django import pytest import redis from _pytest.config import Config -from django.conf import settings as django_settings -from django.core.management import call_command from dotenv import load_dotenv -from pydantic import MariaDBDsn, PostgresDsn +from pydantic import MariaDBDsn, PostgresDsn, TypeAdapter +from pydantic_core import MultiHostHost from redis import Redis +from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig from generalresearch.redis_helper import RedisConfig from generalresearch.sql_helper import SqlHelper @@ -28,7 +26,7 @@ if TYPE_CHECKING: @pytest.fixture(scope="session") -def env_file_path(pytestconfig: Config) -> str: +def env_file_path(pytestconfig: Config) -> Path: root_path = pytestconfig.rootpath env_file = ".env.test" @@ -40,18 +38,16 @@ def env_file_path(pytestconfig: Config) -> str: for env_path in candidates: if os.path.exists(env_path): load_dotenv(dotenv_path=env_path, override=True) - return os.path.normpath(env_path) + return Path(os.path.normpath(env_path)) raise AssertionError(f"No .env.test file found in: {', '.join(candidates)}") @pytest.fixture(scope="session") -def settings(env_file_path: str) -> "GRLBaseSettings": +def settings(env_file_path: Path) -> "GRLBaseSettings": from generalresearch.config import GRLBaseSettings - print(f"{env_file_path=}") - - s = GRLBaseSettings(_env_file=env_file_path) + s = GRLBaseSettings() if s.thl_mkpl_rr_db is not None: if s.spectrum_rw_db is None: @@ -71,14 +67,28 @@ def settings(env_file_path: str) -> "GRLBaseSettings": def postgres_instance(settings: "GRLBaseSettings") -> Generator[PostgresDsn]: """Create a ephemeral postgresql instance for us to use during pytest. - This is simplified, and only based off a single host. We don't want to - create multiple migrated tmp databases for each rw/rr/ro connection + This does not create any tables, or schema definitions within the instance. + What this does is simply: + + 1. Create a database on a known, consistent, staging or unittest + defined Postgres server. + + 2. Return the PostgresDsn of that table + + 3. On shutdown, go ahead and delete that database after the + tests have finished. """ - assert settings.thl_web_rw_db - # assert settings.thl_web_rw_db.host + msg = "Must define Postgres test settings" + assert settings.testing_postgres, msg + assert settings.testing_postgres_user, msg + assert settings.testing_postgres_pass, msg - dsn: PostgresDsn = settings.thl_web_rw_db + db_uri, db_user, db_pass = ( + settings.testing_postgres, + settings.testing_postgres_user, + settings.testing_postgres_pass, + ) # Connect to default DB to create the new one from psycopg import connect @@ -86,40 +96,74 @@ def postgres_instance(settings: "GRLBaseSettings") -> Generator[PostgresDsn]: now = datetime.now(timezone.utc) ts: str = now.strftime("%Y-%m-%d") - db_name = f"unittest-{ts}-{uuid4().hex[:6]}" - print("XXX", str(dsn)) - conn = connect(str(dsn)) + + db_path_connect = f"postgres://{db_user}:{db_pass}@{db_uri}" + db_path = f"{db_path_connect}/{db_name}" + + # The DATABASE does NOT yet exist on the Postgres SERVER, thus + # we first must connect only to the SERVER (eg: default postgres path used) + conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() cur.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name))) cur.close() conn.close() - host = dsn.hosts()[0] - db_url = ( - f"postgres://{host['username']}:{host['password']}@{host['host']}/{db_name}" - ) - - yield PostgresDsn(db_url) + yield PostgresDsn(db_path) # Teardown: drop the DB after the session - conn = connect(str(dsn)) + conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() - # cur.execute(SQL("DROP DATABASE {}").format(Identifier(db_name))) + cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name))) cur.close() conn.close() @pytest.fixture(scope="session") -def django_db_setup(settings: "GRLBaseSettings") -> Callable[..., None]: +def postgres_instance_dict( + postgres_instance: PostgresDsn, +) -> Generator[PostgresDict]: + host = postgres_instance.hosts()[0] + assert host is not None + + msg = "Must have full Postgres details" + assert host["host"], msg + assert host["username"], msg + assert host["password"], msg + + assert postgres_instance.path + + yield PostgresDict( + username=host["username"], + password=host["password"], + host=host["host"], + name=postgres_instance.path.lstrip("/"), + port=5432, + ) + + +@pytest.fixture(scope="session") +def postgres_instance_host( + postgres_instance_dict: PostgresDict, +) -> Generator[InternalHostname]: + adapter = TypeAdapter(InternalHostname) + value = adapter.validate_python(postgres_instance_dict["host"]) + yield value + + +@pytest.fixture(scope="session") +def django_db_factory( + postgres_instance: PostgresDsn, postgres_instance_dict: PostgresDict +) -> Callable[..., PostgresDsn]: - def _inner(): + import django + from django.apps import apps + from django.conf import settings as django_settings + from django.core.management import call_command - assert settings.thl_web_rw_db - dsn: PostgresDsn = settings.thl_web_rw_db - host = dsn.hosts()[0] + def _inner(django_project: str = "generalresearch.thl_django"): # 1. Bootstrapping Django settings if not django_settings.configured: @@ -127,93 +171,70 @@ def django_db_setup(settings: "GRLBaseSettings") -> Callable[..., None]: DATABASES={ "default": { "ENGINE": "django.db.backends.postgresql", - # PostgresDsn stores path as "/dbname" - "NAME": str(dsn.path).lstrip("/"), - "USER": host["username"], - "PASSWORD": host["password"], - "HOST": host["host"], - "PORT": "5432", + "NAME": postgres_instance_dict["name"], + "USER": postgres_instance_dict["username"], + "PASSWORD": postgres_instance_dict["password"], + "HOST": postgres_instance_dict["host"], + "PORT": postgres_instance_dict["port"], } }, INSTALLED_APPS=[ "django.contrib.postgres", "django.contrib.contenttypes", - "generalresearch.thl_django", + django_project, ], ) django.setup() - from django.apps import apps - - for model in apps.get_models(): - print(f"Discovered model: {model._meta.label}") + # for model in apps.get_models(): + # print(f"Discovered model: {model._meta.label}") # 2. Run migrations directly during fixture activation call_command("migrate") + # 3. Return the Dsn so the factory gives a way to connect + return postgres_instance + return _inner @pytest.fixture(scope="session") -def thl_web_rr( - settings: "GRLBaseSettings", postgres_instance: PostgresDsn, django_db_setup -) -> PostgresConfig: - dsn = settings.thl_web_rr_db - assert dsn - assert dsn.path - - if dsn.path not in ["/", "/postgres"]: - assert "/unittest-" in dsn.path - - db_path = postgres_instance.path - host = dsn.hosts()[0] - db_url = f"postgres://{host['username']}:{host['password']}@{host['host']}{db_path}" - - # Run Migrations now. - django_db_setup() +def thl_web_rr(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: return PostgresConfig( - dsn=PostgresDsn(db_url), + dsn=django_db_factory("generalresearch.thl_django"), connect_timeout=1, statement_timeout=5, ) @pytest.fixture(scope="session") -def thl_web_rw( - settings: "GRLBaseSettings", postgres_instance: PostgresDsn, django_db_setup -) -> PostgresConfig: - dsn = settings.thl_web_rw_db - assert dsn - assert dsn.path +def thl_web_rw(thl_web_rr: PostgresConfig) -> PostgresConfig: + return thl_web_rr - if dsn.path not in ["/", "/postgres"]: - assert "/unittest-" in dsn.path - db_path = postgres_instance.path - host = dsn.hosts()[0] - db_url = f"postgres://{host['username']}:{host['password']}@{host['host']}{db_path}" - - # Run Migrations now. - django_db_setup() +@pytest.fixture(scope="session") +def gr_db(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: return PostgresConfig( - dsn=PostgresDsn(db_url), + dsn=django_db_factory("gr_carer"), connect_timeout=1, statement_timeout=5, ) @pytest.fixture(scope="session") -def gr_db(settings: "GRLBaseSettings") -> PostgresConfig: - dsn = settings.gr_db - assert dsn - assert dsn.path +def grliq_db(postgres_instance: PostgresDsn) -> PostgresConfig: - if dsn.path not in ["/", "/postgres"]: - assert "/unittest-" in dsn.path + # test_words = {"localhost", "127.0.0.1", "unittest", "grliq-test"} + # assert any(w in str(postgres_config.dsn) for w in test_words), "check grliq postgres_config" + # assert "grliqdeceezpocymo" not in str(postgres_config.dsn), "check grliq postgres_config" - return PostgresConfig(dsn=settings.gr_db, connect_timeout=5, statement_timeout=2) + return PostgresConfig( + dsn=postgres_instance, + connect_timeout=1, + statement_timeout=5, + ) @pytest.fixture(scope="session") @@ -234,26 +255,6 @@ def spectrum_rw(settings: "GRLBaseSettings") -> SqlHelper: @pytest.fixture(scope="session") -def grliq_db(settings: "GRLBaseSettings") -> PostgresConfig: - dsn = settings.grliq_db - assert dsn - assert dsn.path - - if dsn.path not in ["/", "/postgres"]: - assert "/unittest-" in dsn.path - - # test_words = {"localhost", "127.0.0.1", "unittest", "grliq-test"} - # assert any(w in str(postgres_config.dsn) for w in test_words), "check grliq postgres_config" - # assert "grliqdeceezpocymo" not in str(postgres_config.dsn), "check grliq postgres_config" - - return PostgresConfig( - dsn=settings.grliq_db, - connect_timeout=2, - statement_timeout=2, - ) - - -@pytest.fixture(scope="session") def thl_redis(settings: "GRLBaseSettings") -> "Redis": # todo: this should get replaced with redisconfig (in most places) # I'm not sure where this would be? in the domain name? diff --git a/test_utils/incite/conftest.py b/test_utils/incite/conftest.py index 058093e..0e2f7bd 100644 --- a/test_utils/incite/conftest.py +++ b/test_utils/incite/conftest.py @@ -10,9 +10,6 @@ import pytest from _pytest.fixtures import SubRequest from faker import Faker -# from test_utils.managers.ledger.conftest import session_with_tx_factory -# from test_utils.models.conftest import session_factory - if TYPE_CHECKING: from generalresearch.config import GRLBaseSettings from generalresearch.incite.base import GRLDatasets diff --git a/tests/incite/collections/test_df_collection_item_thl_web.py b/tests/incite/collections/test_df_collection_item_thl_web.py index a858fbe..8b8bcbe 100644 --- a/tests/incite/collections/test_df_collection_item_thl_web.py +++ b/tests/incite/collections/test_df_collection_item_thl_web.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from collections.abc import Generator from datetime import datetime, timedelta, timezone from itertools import product as iter_product from os.path import join as pjoin @@ -12,13 +15,7 @@ from distributed import Client, Scheduler, Worker # noinspection PyUnresolvedReferences from distributed.utils_test import ( - cleanup, - client, - client_no_amm, - cluster_fixture, gen_cluster, - loop, - loop_in_thread, ) from faker import Faker from pandera.pandas import DataFrameSchema @@ -34,7 +31,6 @@ from generalresearch.models.thl.product import Product from generalresearch.models.thl.user import User from generalresearch.pg_helper import PostgresConfig from generalresearch.sql_helper import PostgresDsn -from test_utils.incite.conftest import incite_item_factory, mnt_filepath if TYPE_CHECKING: from generalresearch.incite.base import GRLDatasets @@ -56,12 +52,12 @@ unsupported_mock_types = { } -def combo_object(): +def combo_object() -> Generator[str, None, None]: for x in iter_product( df_collections, ["15min", "45min", "1H"], ): - yield x + yield from x class TestDFCollectionItemBase: @@ -199,7 +195,7 @@ class TestDFCollectionItemMethod: client_no_amm, incite_item_factory, delete_df_collection, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): assert 1 + 1 == 2 @@ -768,7 +764,7 @@ class TestDFCollectionItemFunctionalTest: product: Product, incite_item_factory, delete_df_collection, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): from generalresearch.models.thl.user import User @@ -818,7 +814,7 @@ class TestDFCollectionItemFunctionalTest: df_collection_data_type, incite_item_factory, delete_df_collection, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): """A functional test to write some Parquet files for the DFCollection and then confirm that the files get written @@ -866,7 +862,7 @@ class TestDFCollectionItemFunctionalTest: df_collection_data_type, incite_item_factory, delete_df_collection, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): from generalresearch.models.thl.user import User @@ -919,7 +915,7 @@ class TestDFCollectionItemFunctionalTest: product: Product, offset: str, duration: timedelta, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): """Don't allow creating an archive for data that will likely be overwritten or updated @@ -960,7 +956,7 @@ class TestDFCollectionItemFunctionalTest: user: User, offset: str, duration: timedelta, - mnt_filepath: "GRLDatasets", + mnt_filepath: GRLDatasets, ): delete_df_collection(coll=df_collection) diff --git a/tests/incite/collections/test_df_collection_thl_marketplaces.py b/tests/incite/collections/test_df_collection_thl_marketplaces.py index 8ce8acc..981f62e 100644 --- a/tests/incite/collections/test_df_collection_thl_marketplaces.py +++ b/tests/incite/collections/test_df_collection_thl_marketplaces.py @@ -28,7 +28,7 @@ def combo_object(): ], ["5min", "6H", "30D"], ): - yield x + yield from x @pytest.mark.parametrize("df_coll, offset", combo_object()) diff --git a/tests/incite/collections/test_df_collection_thl_web.py b/tests/incite/collections/test_df_collection_thl_web.py index c64dac8..b09d44c 100644 --- a/tests/incite/collections/test_df_collection_thl_web.py +++ b/tests/incite/collections/test_df_collection_thl_web.py @@ -1,3 +1,6 @@ +from __future__ import annotations + +from collections.abc import Generator from datetime import datetime from itertools import product from typing import TYPE_CHECKING @@ -11,9 +14,13 @@ from generalresearch.incite.collections import DFCollection, DFCollectionType if TYPE_CHECKING: from generalresearch.incite.base import GRLDatasets + from generalresearch.incite.collections import ( + DFCollectionItem, + DFCollectionType, + ) -def combo_object(): +def combo_object() -> Generator[tuple, None, None]: for x in product( [ DFCollectionType.USER, @@ -25,7 +32,7 @@ def combo_object(): ], ["30min", "1H"], ): - yield x + yield from x @pytest.mark.parametrize( @@ -33,7 +40,9 @@ def combo_object(): ) class TestDFCollection_thl_web: - def test_init(self, df_collection_data_type, offset: str, df_collection): + def test_init( + self, df_collection_data_type: DFCollectionType, offset: str, df_collection + ): assert isinstance(df_collection_data_type, DFCollectionType) assert isinstance(df_collection, DFCollection) @@ -43,12 +52,12 @@ class TestDFCollection_thl_web: ) class TestDFCollection_thl_web_Properties: - def test_items(self, df_collection_data_type, offset: str, df_collection): + def test_items(self, df_collection): assert isinstance(df_collection.items, list) for i in df_collection.items: assert i._collection == df_collection - def test__schema(self, df_collection_data_type, offset: str, df_collection): + def test__schema(self, df_collection): assert isinstance(df_collection._schema, DataFrameSchema) @@ -58,16 +67,16 @@ class TestDFCollection_thl_web_Properties: class TestDFCollection_thl_web_BaseProperties: @pytest.mark.skip - def test__interval_range(self, df_collection_data_type, offset: str, df_collection): + def test__interval_range(self, df_collection): pass - def test_interval_start(self, df_collection_data_type, offset: str, df_collection): + def test_interval_start(self, df_collection): assert isinstance(df_collection.interval_start, datetime) - def test_interval_range(self, df_collection_data_type, offset: str, df_collection): + def test_interval_range(self, df_collection): assert isinstance(df_collection.interval_range, list) - def test_progress(self, df_collection_data_type, offset: str, df_collection): + def test_progress(self, df_collection): assert isinstance(df_collection.progress, pd.DataFrame) diff --git a/tests/models/custom_types/test_aware_datetime.py b/tests/models/custom_types/test_aware_datetime.py index a23413c..14d1343 100644 --- a/tests/models/custom_types/test_aware_datetime.py +++ b/tests/models/custom_types/test_aware_datetime.py @@ -4,7 +4,7 @@ from typing import Optional import pytest import pytz -from pydantic import BaseModel, ValidationError, Field +from pydantic import BaseModel, Field, ValidationError from generalresearch.models.custom_types import AwareDatetimeISO diff --git a/tests/pytest.ini b/tests/pytest.ini index d280de0..1a5c089 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,2 +1 @@ -[pytest] -asyncio_mode = auto
\ No newline at end of file +[pytest]
\ No newline at end of file diff --git a/tests/test_postgres.py b/tests/test_postgres.py new file mode 100644 index 0000000..3b3ddd0 --- /dev/null +++ b/tests/test_postgres.py @@ -0,0 +1,68 @@ +import socket +import subprocess +from typing import Callable + +from pydantic import PostgresDsn + +from generalresearch.models.custom_types import InternalHostname, PostgresDict +from generalresearch.pg_helper import PostgresConfig + + +def is_port_open(host: InternalHostname, port: int = 5432, timeout: int = 3): + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except (socket.timeout, ConnectionRefusedError, OSError): + return False + + +def can_ping(host: InternalHostname): + return ( + subprocess.call( + ["ping", "-c", "1", str(host)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + == 0 + ) + + +class TestPostgresDSN: + + def test_ping(self, postgres_instance_host: InternalHostname): + assert can_ping(host=postgres_instance_host) + + def test_port(self, postgres_instance_host: InternalHostname): + assert is_port_open(host=postgres_instance_host) + + def test_conn(self, postgres_instance: PostgresDsn): + config = PostgresConfig( + dsn=postgres_instance, + connect_timeout=1, + statement_timeout=1, + ) + res = config.execute_sql_query(query="SELECT 1;") + assert len(res) == 1 + + +class TestPostgresDjangoCreation: + + def test_ping(self, postgres_instance_dict: PostgresDict): + assert can_ping(host=postgres_instance_dict["host"]) + + def test_django_creation( + self, + django_db_factory: Callable[..., None], + ): + + dsn = django_db_factory() + assert isinstance(dsn, PostgresDsn) + + def test_django_tables(self, thl_web_rw: PostgresConfig): + res = thl_web_rw.execute_sql_query(query=""" + SELECT COUNT(*) + FROM information_schema.tables + WHERE table_schema = 'public'; + """) + assert len(res) == 1 + assert res[0]["count"] == 56 |