from __future__ import annotations import os import subprocess import sys from collections.abc import Callable, Generator from datetime import UTC, datetime from pathlib import Path from random import randint from typing import TYPE_CHECKING, Any from uuid import uuid4 import pytest import redis from fastapi.testclient import TestClient from generalresearch.models.custom_types import InternalHostname, PostgresDict from generalresearch.pg_helper import PostgresConfig, PostgresDsn from generalresearch.redis_helper import RedisConfig from mypy_boto3_mturk import MTurkClient from pydantic import TypeAdapter from pytest import TempPathFactory from jb.decorators import CLIENT_CONFIG, get_redis_config from jb.main import app from tests import generate_amt_id if TYPE_CHECKING: from jb.settings import Settings pytest_plugins = [ "tests.fixtures.amt", "tests.fixtures.flow", "tests.fixtures.http", "tests.fixtures.managers", "tests.fixtures.models", ] # --- IDs and Identifiers --- @pytest.fixture def amt_hit_id() -> str: return generate_amt_id() @pytest.fixture def amt_hit_type_id() -> str: return generate_amt_id() @pytest.fixture def amt_assignment_id() -> str: return generate_amt_id() @pytest.fixture def amt_worker_id() -> str: return generate_amt_id(length=21) @pytest.fixture def amt_group_id() -> str: return generate_amt_id() @pytest.fixture def tsid() -> str: return uuid4().hex @pytest.fixture def tsid1() -> str: return uuid4().hex @pytest.fixture def tsid2() -> str: return uuid4().hex @pytest.fixture def pe_id() -> str: # payout event / cashout request UUID return uuid4().hex # --- Settings --- @pytest.fixture(scope="session") def settings() -> Settings: from jb.settings import Settings as JBSettings return JBSettings() # --- Database Connectors --- @pytest.fixture(scope="session") def postgres_instance(settings: Settings) -> Generator[PostgresDsn]: """Create a ephemeral postgresql instance for us to use during pytest. This does not create any tables, or schema definitions within the instance. What this does is simply: 1. Create a database on a known, consistent, staging or unittest defined Postgres server. 2. Return the PostgresDsn of that table 3. On shutdown, go ahead and delete that database after the tests have finished. """ msg = "Must define Postgres test settings" assert settings.testing_postgres, msg assert settings.testing_postgres_user, msg assert settings.testing_postgres_pass, msg db_uri, db_user, db_pass = ( settings.testing_postgres, settings.testing_postgres_user, settings.testing_postgres_pass, ) # Connect to default DB to create the new one from psycopg import connect from psycopg.sql import SQL, Identifier now = datetime.now(UTC) ts: str = now.strftime("%Y-%m-%d") db_name = f"unittest-{ts}-{uuid4().hex[:6]}" db_path_connect = f"postgres://{db_user}:{db_pass}@{db_uri}" db_path = f"{db_path_connect}/{db_name}" # The DATABASE does NOT yet exist on the Postgres SERVER, thus # we first must connect only to the SERVER (eg: default postgres path used) conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() cur.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name))) cur.close() conn.close() yield PostgresDsn(db_path) # Teardown: drop the DB after the session conn = connect(f"{db_path_connect}/postgres") conn.autocommit = True cur = conn.cursor() cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name))) cur.close() conn.close() @pytest.fixture(scope="session") def django_settings_file( postgres_instance_dict: PostgresDict, tmp_path_factory: TempPathFactory, ) -> Callable[..., tuple[str, Path]]: def _inner(extra_installed_apps: list[str] | None = None) -> tuple[str, Path]: installed_apps = [ "django.contrib.postgres", "django.contrib.contenttypes", ] + (extra_installed_apps or []) settings_dir = tmp_path_factory.mktemp("django-settings") settings_module = "test_settings" settings_content = f"""DATABASES = {{ "default": {{ "ENGINE": "django.db.backends.postgresql", "NAME": {postgres_instance_dict["name"]!r}, "USER": {postgres_instance_dict["username"]!r}, "PASSWORD": {postgres_instance_dict["password"]!r}, "HOST": {postgres_instance_dict["host"]!r}, "PORT": {postgres_instance_dict["port"]!r}, }} }} INSTALLED_APPS = {installed_apps!r} DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" LANGUAGE_CODE = "en-us" TIME_ZONE = "UTC" USE_I18N = True USE_L10N = True USE_TZ = True """ settings_file_path = settings_dir / f"{settings_module}.py" settings_file_path.write_text(settings_content, encoding="utf-8") return settings_module, settings_dir return _inner @pytest.fixture(scope="session") def postgres_instance_dict( postgres_instance: PostgresDsn, ) -> Generator[PostgresDict]: host = postgres_instance.hosts()[0] assert host is not None msg = "Must have full Postgres details" assert host["host"], msg assert host["username"], msg assert host["password"], msg assert postgres_instance.path yield PostgresDict( username=host["username"], password=host["password"], host=host["host"], name=postgres_instance.path.lstrip("/"), port=5432, ) @pytest.fixture(scope="session") def postgres_instance_host( postgres_instance_dict: PostgresDict, ) -> Generator[InternalHostname]: adapter = TypeAdapter(InternalHostname) value = adapter.validate_python(postgres_instance_dict["host"]) yield value @pytest.fixture(scope="session") def django_db_factory( postgres_instance: PostgresDsn, django_settings_file: Callable[..., tuple[str, Path]], postgres_instance_dict: PostgresDict, tmp_path_factory: TempPathFactory, ) -> Callable[..., PostgresDsn | None]: _ran = {} def _inner( django_project: str = "carer.mtwerk", ) -> PostgresDsn | None: if _ran.get(django_project, False): print(f"Already ran django_db_factory:{django_project}") return postgres_instance _ran[django_project] = True _cwd = None _manage_path = "generalresearch.thl_django.app.manage" _settings_module, _settings_dir = django_settings_file( extra_installed_apps=[ "carer.mtwerk", ], ) pythonpath = str(_settings_dir) if existing_pythonpath := os.environ.get("PYTHONPATH"): pythonpath += os.pathsep + existing_pythonpath env = { **os.environ, "DJANGO_SETTINGS_MODULE": _settings_module, "PYTHONPATH": pythonpath, } # we check right after. if we check now, we won't print if bad res1 = subprocess.run( # noqa: PLW1510 [ sys.executable, "-m", _manage_path, "makemigrations", f"--settings={_settings_module}", ], cwd=str(_cwd) if _cwd is not None else None, env=env, capture_output=True, text=True, ) if res1.returncode != 0: print("STDOUT:", res1.stdout) print("STDERR:", res1.stderr) res1.check_returncode() res2 = subprocess.run( # noqa: PLW1510 [ sys.executable, "-m", _manage_path, "migrate", f"--settings={_settings_module}", ], env=env, cwd=str(_cwd) if _cwd is not None else None, capture_output=True, text=True, ) if res2.returncode != 0: print("STDOUT:", res2.stdout) print("STDERR:", res2.stderr) res2.check_returncode() # 3. Return the Dsn so the factory gives a way to connect return postgres_instance return _inner @pytest.fixture(scope="session") def pg_config(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig: _dsn = django_db_factory() return PostgresConfig( dsn=_dsn, connect_timeout=1, statement_timeout=5, ) # --- Redis --- @pytest.fixture(scope="session") def redis_config_db() -> str: # need to update 'databases' in /etc/redis/redis.conf # or this won't work and you'll have no indication why ... return str(randint(99, 1_023)) @pytest.fixture(scope="session") def redis_config(settings: Settings, redis_config_db: str) -> Generator[RedisConfig]: assert "unittest" in str(settings.testing_redis) or "127.0.0.1" in str( settings.testing_redis ) uri = f"redis://{settings.testing_redis}/{redis_config_db}" res = subprocess.run( ["redis-cli", "-u", uri, "SET", "jenkins_lock", "1", "NX", "EX", "3600"], check=True, text=True, capture_output=True, ) if res.stdout.strip() != "OK": raise ValueError("Redis already locked... aborting.") yield RedisConfig( dsn=uri, decode_responses=True, socket_timeout=settings.redis_timeout, socket_connect_timeout=settings.redis_timeout, ) r = redis.from_url(uri) r.flushdb() # --- Connectors --- @pytest.fixture(scope="session") def amt_client(settings: Settings) -> MTurkClient: import boto3 client = boto3.client( service_name="mturk", region_name="us-east-1", endpoint_url=str(settings.amt_endpoint), aws_access_key_id=settings.amt_access_id, aws_secret_access_key=settings.amt_secret_key, config=CLIENT_CONFIG, ) # Confirm we're only using the Sandbox for any unittests assert "mturk-requester-sandbox" in client.meta.endpoint_url return client