basic tests should be working. Jenkins p1

14 files changed, 221 insertions, 148 deletions

diff --git a/Jenkinsfile b/Jenkinsfile
index eb53bfd..0632ee8 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -10,95 +10,47 @@ pipeline {
         pollSCM('H */3 * * *')
     }
 
+    options {
+        skipDefaultCheckout()
+    }
+
     environment {
         DATA_SRC = "${env.WORKSPACE}/mnt/"
-
-        AMT_JB_CARER_VENV = "${env.WORKSPACE}/amt-jb-carer-venv"
-        AMT_JB_VENV = "${env.WORKSPACE}/amt-jb-venv"
+        VENV = "${env.WORKSPACE}/amt-jb-venv"
     }
 
     stages {
-        stage('Setup DB') {
-            steps {
-                script {
-                    env.DB_NAME = 'unittest-amt-jb-' + UUID.randomUUID().toString().replace('-', '').take(12)
-                    env.AMT_JB_DB = "postgres://${env.DB_USER}:${env.DB_PASSWORD}@${env.DB_POSTGRESQL_HOST}/${env.DB_NAME}"
-                    echo "Using database: ${env.DB_NAME}"
-                }
 
-                sh """
-                PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF
-                CREATE DATABASE "${env.DB_NAME}" WITH TEMPLATE = template0 ENCODING = 'UTF8';
-                EOF
-                """
-            }
-        }
-
-        stage('env') {
+        stage('setup') {
             steps {
-                withCredentials([file(
-                    credentialsId: 'amt-jb-app-env',
-                    variable: 'ENV_FILE_PATH')]) {
-                    sh "cp \$ENV_FILE_PATH ${WORKSPACE}/.env.test"
+                dir("generalresearch-${VER}") {
+                    deleteDir()
+                    unstash 'source'
+
+                    withCredentials([file(
+                        credentialsId: '971e1f48-09ce-4446-9155-a52c1adb6249',
+                        variable: 'ENV_TEST_FILE')]) {
+                        sh 'cp $ENV_TEST_FILE .env.test'
                     }
-            }
-        }
-
-        stage('setup:amt-jb-carer') {
-            steps {
-                checkout scmGit(
-                    branches: [[name: 'master']],
-                    extensions: [ cloneOption(shallow: true) ],
-                    userRemoteConfigs: [
-                        [credentialsId:  'abdeb570-b708-44f3-b857-8a6b06ed9822',
-                         url: 'ssh://code.g-r-l.com:6611/panels/amt-jb']
-                    ],
-                )
-
-                dir('carer/') {
-                    sh 'python3.13 -m venv $AMT_JB_CARER_VENV'
-                    sh '$AMT_JB_CARER_VENV/bin/pip install -U setuptools wheel pip'
-                    sh '$AMT_JB_CARER_VENV/bin/pip install -r requirements.txt'
-                    sh '$AMT_JB_CARER_VENV/bin/python3.13 manage.py migrate --settings=carer.settings.unittest'
+                    sh "/usr/local/bin/python3.14 -m venv ${VENV}"
+                    sh "${VENV}/bin/pip install -U setuptools wheel pip"
+                    sh "${VENV}/bin/pip install -r requirements.txt"
+                    sh "${VENV}/bin/pip install django"
                 }
             }
         }
 
-        /*  Okay, finally we can setup the virtual environment for the actual
-            project itself. gr-api FastAPI doesn't manage any of it's own
-            database so it doesn't need to do any migrations or anything.
-        */
-        stage('setup:amt-jb') {
-            steps {
-                sh 'python3.13 -m venv $AMT_JB_VENV'
-                sh '$AMT_JB_VENV/bin/pip install -U setuptools wheel pip'
-                withCredentials([sshUserPrivateKey(
-                    credentialsId: 'abdeb570-b708-44f3-b857-8a6b06ed9822',
-                    keyFileVariable: 'SSH_PRIVATE_KEY')]) {
-                    sh """
-                        eval \$(ssh-agent) && ssh-add ${SSH_PRIVATE_KEY} && \
-                        ${AMT_JB_VENV}/bin/pip install -r requirements.txt
-                    """
-                    }
-            }
-        }
-
         stage('tests') {
             steps {
-                sh '$AMT_JB_VENV/bin/pytest -v tests'
+                sh '$VENV/bin/pytest tests/test_postgres.py -vs'
             }
         }
+
     }
     post {
         always {
             echo 'One way or another, I have finished'
             deleteDir() /* clean up our workspace */
-
-            sh """
-            PGPASSWORD=${env.DB_PASSWORD} psql -h ${env.DB_POSTGRESQL_HOST} -U ${env.DB_USER} -d postgres <<EOF
-            DROP DATABASE "${env.DB_NAME}";
-            EOF
-            """
         }
     }
 }
diff --git a/carer/carer/__init__.py b/carer/app/__init__.py
index e69de29..e69de29 100644
--- a/carer/carer/__init__.py
+++ b/carer/app/__init__.py
diff --git a/carer/app/manage.py b/carer/app/manage.py
new file mode 100644
index 0000000..9ae482e
--- /dev/null
+++ b/carer/app/manage.py
@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "carer.app.settings")
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)
diff --git a/carer/app/settings.py b/carer/app/settings.py
new file mode 100644
index 0000000..55dd93b
--- /dev/null
+++ b/carer/app/settings.py
@@ -0,0 +1,24 @@
+import os
+
+INSTALLED_APPS = [
+    "django.contrib.postgres",
+    "django.contrib.contenttypes",
+    "carer.mtwerk",
+]
+
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": os.environ.get("DB_NAME", "thl-test"),
+        "USER": os.environ.get("DB_USER", "postgres"),
+        "PASSWORD": os.environ.get("DB_PASSWORD", "password"),
+        "HOST": os.environ.get("DB_HOST", "127.0.0.1"),
+        "PORT": "5432",
+    }
+}
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+LANGUAGE_CODE = "en-us"
+TIME_ZONE = "UTC"
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
diff --git a/carer/apps.py b/carer/apps.py
new file mode 100644
index 0000000..c96687b
--- /dev/null
+++ b/carer/apps.py
@@ -0,0 +1,9 @@
+from django.apps import AppConfig
+
+
+class MTwerkSchemaConfig(AppConfig):
+    name = "carer.mtwerk"
+    label = "mtwerk"
+
+    def ready(self):
+        from .mtwerk import models  # pycharm: keep
diff --git a/carer/carer/settings/base.py b/carer/carer/settings/base.py
deleted file mode 100644
index c83e0fe..0000000
--- a/carer/carer/settings/base.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from pathlib import Path
-import os
-
-# Build paths inside the project like this: BASE_DIR / 'subdir'.
-BASE_DIR = Path(__file__).resolve().parent.parent
-
-SECRET_KEY = os.environ.get("CARER_SECRET_KEY")
-
-INSTALLED_APPS = [
-    "carer.mtwerk",
-]
-
-LANGUAGE_CODE = "en-us"
-TIME_ZONE = "UTC"
-USE_I18N = True
-USE_TZ = True
-
-DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
diff --git a/carer/carer/settings/unittest.py b/carer/carer/settings/unittest.py
deleted file mode 100644
index a147794..0000000
--- a/carer/carer/settings/unittest.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import os
-
-# noinspection PyUnresolvedReferences
-from .base import *
-
-DEBUG = True
-ALLOWED_HOSTS = ["*"]
-
-DEFAULT_HTTP_PROTOCOL = "https"
-CSRF_COOKIE_HTTPONLY = True  # Get from page, never cookie
-
-DB_NAME = os.environ.get("DB_NAME")
-DB_USER = os.environ.get("DB_USER")
-DB_PASSWORD = os.environ.get("DB_PASSWORD")
-DB_POSTGRESQL_HOST = os.environ.get("DB_POSTGRESQL_HOST")
-
-DATABASES = dict(
-    default={
-        "ENGINE": "django.db.backends.postgresql",
-        "NAME": DB_NAME,
-        "USER": DB_USER,
-        "PASSWORD": DB_PASSWORD,
-        "HOST": DB_POSTGRESQL_HOST,
-        "PORT": "5432",
-        "ENCODING": "utf8mb4",
-        "COLLATION": "utf8mb4_unicode_520_ci",
-    }
-)
-
-print(f"Using test database: {DB_NAME} on {DATABASES['default']['HOST']}")
diff --git a/carer/manage.py b/carer/manage.py
deleted file mode 100644
index 091b39b..0000000
--- a/carer/manage.py
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-"""Django's command-line utility for administrative tasks."""
-import os
-import sys
-
-
-def main():
-    """Run administrative tasks."""
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "carer.settings")
-    try:
-        from django.core.management import execute_from_command_line
-    except ImportError as exc:
-        raise ImportError(
-            "Couldn't import Django. Are you sure it's installed and "
-            "available on your PYTHONPATH environment variable? Did you "
-            "forget to activate a virtual environment?"
-        ) from exc
-    execute_from_command_line(sys.argv)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/carer/carer/mtwerk/__init__.py b/carer/mtwerk/__init__.py
index e69de29..e69de29 100644
--- a/carer/carer/mtwerk/__init__.py
+++ b/carer/mtwerk/__init__.py
diff --git a/carer/carer/mtwerk/migrations/0001_initial.py b/carer/mtwerk/migrations/0001_initial.py
index d5c856a..d5c856a 100644
--- a/carer/carer/mtwerk/migrations/0001_initial.py
+++ b/carer/mtwerk/migrations/0001_initial.py
diff --git a/carer/carer/mtwerk/migrations/__init__.py b/carer/mtwerk/migrations/__init__.py
index e69de29..e69de29 100644
--- a/carer/carer/mtwerk/migrations/__init__.py
+++ b/carer/mtwerk/migrations/__init__.py
diff --git a/carer/carer/mtwerk/models.py b/carer/mtwerk/models.py
index 5cb026d..5cb026d 100644
--- a/carer/carer/mtwerk/models.py
+++ b/carer/mtwerk/models.py
diff --git a/tests/conftest.py b/tests/conftest.py
index 457f6a3..ca6661a 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -3,15 +3,17 @@ from __future__ import annotations
 import os
 import subprocess
 import sys
-from collections.abc import Callable
+from collections.abc import Callable, Generator
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import TYPE_CHECKING
 from uuid import uuid4
 
 import pytest
-from generalresearch.models.custom_types import PostgresDict
+from generalresearch.models.custom_types import InternalHostname, PostgresDict
 from generalresearch.pg_helper import PostgresConfig, PostgresDsn
 from mypy_boto3_mturk import MTurkClient
+from pydantic import TypeAdapter
 from pytest import TempPathFactory
 
 from jb.decorators import CLIENT_CONFIG
@@ -22,7 +24,6 @@ if TYPE_CHECKING:
 
 
 pytest_plugins = [
-    "test_utils.conftest",
     "tests.fixtures.amt",
     "tests.fixtures.flow",
     "tests.fixtures.http",
@@ -94,9 +95,139 @@ def settings() -> Settings:
 
 
 @pytest.fixture(scope="session")
+def postgres_instance(settings: Settings) -> Generator[PostgresDsn]:
+    """Create a ephemeral postgresql instance for us to use during pytest.
+
+    This does not create any tables, or schema definitions within the instance.
+    What this does is simply:
+
+        1. Create a database on a known, consistent, staging or unittest
+            defined Postgres server.
+
+        2. Return the PostgresDsn of that table
+
+        3. On shutdown, go ahead and delete that database after the
+            tests have finished.
+    """
+
+    msg = "Must define Postgres test settings"
+    assert settings.testing_postgres, msg
+    assert settings.testing_postgres_user, msg
+    assert settings.testing_postgres_pass, msg
+
+    db_uri, db_user, db_pass = (
+        settings.testing_postgres,
+        settings.testing_postgres_user,
+        settings.testing_postgres_pass,
+    )
+
+    # Connect to default DB to create the new one
+    from psycopg import connect
+    from psycopg.sql import SQL, Identifier
+
+    now = datetime.now(UTC)
+    ts: str = now.strftime("%Y-%m-%d")
+    db_name = f"unittest-{ts}-{uuid4().hex[:6]}"
+
+    db_path_connect = f"postgres://{db_user}:{db_pass}@{db_uri}"
+    db_path = f"{db_path_connect}/{db_name}"
+
+    # The DATABASE does NOT yet exist on the Postgres SERVER, thus
+    # we first must connect only to the SERVER (eg: default postgres path used)
+    conn = connect(f"{db_path_connect}/postgres")
+    conn.autocommit = True
+    cur = conn.cursor()
+    cur.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name)))
+    cur.close()
+    conn.close()
+
+    yield PostgresDsn(db_path)
+
+    # Teardown: drop the DB after the session
+    conn = connect(f"{db_path_connect}/postgres")
+    conn.autocommit = True
+    cur = conn.cursor()
+    cur.execute(SQL("DROP DATABASE {} WITH (FORCE)").format(Identifier(db_name)))
+    cur.close()
+    conn.close()
+
+
+@pytest.fixture(scope="session")
+def django_settings_file(
+    postgres_instance_dict: PostgresDict,
+    tmp_path_factory: TempPathFactory,
+) -> Callable[..., tuple[str, Path]]:
+
+    def _inner(extra_installed_apps: list[str] | None = None) -> tuple[str, Path]:
+        installed_apps = [
+            "django.contrib.postgres",
+            "django.contrib.contenttypes",
+        ] + (extra_installed_apps or [])
+
+        settings_dir = tmp_path_factory.mktemp("django-settings")
+        settings_module = "test_settings"
+
+        settings_content = f"""DATABASES = {{
+    "default": {{
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": {postgres_instance_dict["name"]!r},
+        "USER": {postgres_instance_dict["username"]!r},
+        "PASSWORD": {postgres_instance_dict["password"]!r},
+        "HOST": {postgres_instance_dict["host"]!r},
+        "PORT": {postgres_instance_dict["port"]!r},
+    }}
+}}
+INSTALLED_APPS = {installed_apps!r}
+DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
+LANGUAGE_CODE = "en-us"
+TIME_ZONE = "UTC"
+USE_I18N = True
+USE_L10N = True
+USE_TZ = True
+"""
+        settings_file_path = settings_dir / f"{settings_module}.py"
+        settings_file_path.write_text(settings_content, encoding="utf-8")
+
+        return settings_module, settings_dir
+
+    return _inner
+
+
+@pytest.fixture(scope="session")
+def postgres_instance_dict(
+    postgres_instance: PostgresDsn,
+) -> Generator[PostgresDict]:
+    host = postgres_instance.hosts()[0]
+    assert host is not None
+
+    msg = "Must have full Postgres details"
+    assert host["host"], msg
+    assert host["username"], msg
+    assert host["password"], msg
+
+    assert postgres_instance.path
+
+    yield PostgresDict(
+        username=host["username"],
+        password=host["password"],
+        host=host["host"],
+        name=postgres_instance.path.lstrip("/"),
+        port=5432,
+    )
+
+
+@pytest.fixture(scope="session")
+def postgres_instance_host(
+    postgres_instance_dict: PostgresDict,
+) -> Generator[InternalHostname]:
+    adapter = TypeAdapter(InternalHostname)
+    value = adapter.validate_python(postgres_instance_dict["host"])
+    yield value
+
+
+@pytest.fixture(scope="session")
 def django_db_factory(
     postgres_instance: PostgresDsn,
-    gr_repo: Callable[..., Path],
     django_settings_file: Callable[..., tuple[str, Path]],
     postgres_instance_dict: PostgresDict,
     tmp_path_factory: TempPathFactory,
@@ -105,7 +236,7 @@ def django_db_factory(
     _ran = {}
 
     def _inner(
-        django_project: str = "generalresearch.thl_django",
+        django_project: str = "carer.mtwerk",
     ) -> PostgresDsn | None:
 
         if _ran.get(django_project, False):
@@ -117,7 +248,7 @@ def django_db_factory(
         _manage_path = "generalresearch.thl_django.app.manage"
         _settings_module, _settings_dir = django_settings_file(
             extra_installed_apps=[
-                "generalresearch.thl_django",
+                "carer.mtwerk",
             ],
         )
 
@@ -177,11 +308,13 @@ def django_db_factory(
 
 
 @pytest.fixture(scope="session")
-def pg_config(settings: Settings) -> PostgresConfig:
+def pg_config(django_db_factory: Callable[..., PostgresDsn]) -> PostgresConfig:
+    _dsn = django_db_factory()
+
     return PostgresConfig(
-        dsn=settings.amt_jb_db,
+        dsn=_dsn,
         connect_timeout=1,
-        statement_timeout=1,
+        statement_timeout=5,
     )
 
 
@@ -193,7 +326,7 @@ def redis(settings: Settings):
     from generalresearch.redis_helper import RedisConfig
 
     redis_config = RedisConfig(
-        dsn=settings.redis,
+        dsn=settings.testing_redis,
         decode_responses=True,
         socket_timeout=settings.redis_timeout,
         socket_connect_timeout=settings.redis_timeout,
diff --git a/tests/test_postgres.py b/tests/test_postgres.py
index 6db4f82..244c132 100644
--- a/tests/test_postgres.py
+++ b/tests/test_postgres.py
@@ -51,3 +51,19 @@ class TestPostgresDjangoCreation:
 
     def test_ping(self, postgres_instance_dict: PostgresDict):
         assert can_ping(host=postgres_instance_dict["host"])
+
+    def test_django_creation(
+        self,
+        django_db_factory: Callable[..., None],
+    ):
+        dsn = django_db_factory()
+        assert isinstance(dsn, PostgresDsn)
+
+    def test_django_tables(self, pg_config: PostgresConfig):
+        res = pg_config.execute_sql_query(query="""
+            SELECT COUNT(*)
+            FROM information_schema.tables
+            WHERE table_schema = 'public';
+        """)
+        assert len(res) == 1
+        assert res[0]["count"] == 7