aboutsummaryrefslogtreecommitdiff
path: root/tests/incite
diff options
context:
space:
mode:
authorstuppie2026-09-07 11:47:43 -0600
committerstuppie2026-09-07 11:47:43 -0600
commit092960233652cce1f4dc7841856034a6635e9cd9 (patch)
tree46e5fcd4d1e1b7ed0b987980c6c67ffa6e6b45c7 /tests/incite
parent80fd8aab4c7271ddb619b0de18741d7ac77b490b (diff)
parent242579a44855873d5e054e375440e9d3492cd682 (diff)
downloadgeneralresearch-092960233652cce1f4dc7841856034a6635e9cd9.tar.gz
generalresearch-092960233652cce1f4dc7841856034a6635e9cd9.zip
Merge branch 'master' into dev-greg
Diffstat (limited to 'tests/incite')
-rw-r--r--tests/incite/collections/test_df_collection_base.py38
-rw-r--r--tests/incite/collections/test_df_collection_item_base.py53
-rw-r--r--tests/incite/collections/test_df_collection_item_thl_web.py454
-rw-r--r--tests/incite/collections/test_df_collection_thl_marketplaces.py30
-rw-r--r--tests/incite/collections/test_df_collection_thl_web.py131
-rw-r--r--tests/incite/mergers/foundations/test_enriched_session.py76
-rw-r--r--tests/incite/mergers/foundations/test_enriched_task_adjust.py49
-rw-r--r--tests/incite/mergers/foundations/test_enriched_wall.py119
-rw-r--r--tests/incite/mergers/foundations/test_user_id_product.py47
-rw-r--r--tests/incite/mergers/test_merge_collection.py73
-rw-r--r--tests/incite/mergers/test_merge_collection_item.py37
-rw-r--r--tests/incite/mergers/test_pop_ledger.py148
-rw-r--r--tests/incite/mergers/test_ym_survey_merge.py84
-rw-r--r--tests/incite/schemas/test_admin_responses.py62
-rw-r--r--tests/incite/schemas/test_thl_web.py8
-rw-r--r--tests/incite/test_collection_base.py91
-rw-r--r--tests/incite/test_collection_base_item.py82
-rw-r--r--tests/incite/test_grl_flow.py11
-rw-r--r--tests/incite/test_interval_idx.py9
19 files changed, 884 insertions, 718 deletions
diff --git a/tests/incite/collections/test_df_collection_base.py b/tests/incite/collections/test_df_collection_base.py
index 31d1720..6d715fa 100644
--- a/tests/incite/collections/test_df_collection_base.py
+++ b/tests/incite/collections/test_df_collection_base.py
@@ -1,18 +1,18 @@
-from datetime import datetime, timezone
+from datetime import UTC, datetime
from typing import TYPE_CHECKING
import pandas as pd
import pytest
from pandera.pandas import DataFrameSchema
-from generalresearch.incite.collections import (
+from generalresearch.incite.collections.base import (
DFCollection,
DFCollectionType,
)
-from test_utils.incite.conftest import mnt_filepath
if TYPE_CHECKING:
from generalresearch.incite.base import GRLDatasets
+ from generalresearch.pg_helper import PostgresConfig
df_collection_types = [e for e in DFCollectionType if e is not DFCollectionType.TEST]
@@ -24,7 +24,7 @@ class TestDFCollectionBase:
"""
- def test_init(self, mnt_filepath: "GRLDatasets", df_coll_type: DFCollectionType):
+ def test_init(self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType):
"""Try to initialize the DFCollection with various invalid parameters"""
with pytest.raises(expected_exception=ValueError) as cm:
DFCollection(archive_path=mnt_filepath.data_src)
@@ -46,24 +46,28 @@ class TestDFCollectionBase:
class TestDFCollectionBaseProperties:
@pytest.mark.skip
- def test_df_collection_items(self, mnt_filepath: "GRLDatasets", df_coll_type):
+ def test_df_collection_items(
+ self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType
+ ):
instance = DFCollection(
data_type=df_coll_type,
- start=datetime(year=1800, month=1, day=1, tzinfo=timezone.utc),
- finished=datetime(year=1900, month=1, day=1, tzinfo=timezone.utc),
- offset="100d",
+ start=datetime(year=1800, month=1, day=1, tzinfo=UTC),
+ finished=datetime(year=1900, month=1, day=1, tzinfo=UTC),
+ offset="100D",
archive_path=mnt_filepath.archive_path(enum_type=df_coll_type),
)
assert len(instance.interval_range) == len(instance.items)
assert len(instance.items) == 366
- def test_df_collection_progress(self, mnt_filepath: "GRLDatasets", df_coll_type):
+ def test_df_collection_progress(
+ self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType
+ ):
instance = DFCollection(
data_type=df_coll_type,
- start=datetime(year=1800, month=1, day=1, tzinfo=timezone.utc),
- finished=datetime(year=1900, month=1, day=1, tzinfo=timezone.utc),
- offset="100d",
+ start=datetime(year=1800, month=1, day=1, tzinfo=UTC),
+ finished=datetime(year=1900, month=1, day=1, tzinfo=UTC),
+ offset="100D",
archive_path=mnt_filepath.archive_path(enum_type=df_coll_type),
)
@@ -71,7 +75,9 @@ class TestDFCollectionBaseProperties:
assert isinstance(instance.progress, pd.DataFrame)
assert instance.progress.shape == (366, 6)
- def test_df_collection_schema(self, mnt_filepath: "GRLDatasets", df_coll_type):
+ def test_df_collection_schema(
+ self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType
+ ):
instance1 = DFCollection(
data_type=DFCollectionType.WALL, archive_path=mnt_filepath.data_src
)
@@ -88,12 +94,12 @@ class TestDFCollectionBaseProperties:
class TestDFCollectionBaseMethods:
@pytest.mark.skip
- def test_initial_load(self, mnt_filepath: "GRLDatasets", thl_web_rr):
+ def test_initial_load(self, mnt_filepath: GRLDatasets, thl_web_rr: PostgresConfig):
instance = DFCollection(
pg_config=thl_web_rr,
data_type=DFCollectionType.USER,
- start=datetime(year=2022, month=1, day=1, minute=0, tzinfo=timezone.utc),
- finished=datetime(year=2022, month=1, day=1, minute=5, tzinfo=timezone.utc),
+ start=datetime(year=2022, month=1, day=1, minute=0, tzinfo=UTC),
+ finished=datetime(year=2022, month=1, day=1, minute=5, tzinfo=UTC),
offset="2min",
archive_path=mnt_filepath.data_src,
)
diff --git a/tests/incite/collections/test_df_collection_item_base.py b/tests/incite/collections/test_df_collection_item_base.py
index 136d234..7a8793d 100644
--- a/tests/incite/collections/test_df_collection_item_base.py
+++ b/tests/incite/collections/test_df_collection_item_base.py
@@ -1,30 +1,30 @@
-from datetime import datetime, timezone
+from __future__ import annotations
+
+from datetime import UTC, datetime
from typing import TYPE_CHECKING
import pytest
-from generalresearch.incite.collections import (
+from generalresearch.incite.collections.base import (
+ MYSQL_ALLOWED_COLL_TYPES,
DFCollection,
DFCollectionItem,
DFCollectionType,
)
-from generalresearch.pg_helper import PostgresConfig
if TYPE_CHECKING:
from generalresearch.incite.base import GRLDatasets
-
-df_collection_types = [e for e in DFCollectionType if e is not DFCollectionType.TEST]
+ from generalresearch.pg_helper import PostgresConfig
-@pytest.mark.parametrize("df_coll_type", df_collection_types)
+@pytest.mark.parametrize("df_coll_type", MYSQL_ALLOWED_COLL_TYPES)
class TestDFCollectionItemBase:
-
- def test_init(self, mnt_filepath: "GRLDatasets", df_coll_type):
+ def test_init(self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType):
collection = DFCollection(
data_type=df_coll_type,
- offset="100d",
- start=datetime(year=1800, month=1, day=1, tzinfo=timezone.utc),
- finished=datetime(year=1900, month=1, day=1, tzinfo=timezone.utc),
+ offset="100D",
+ start=datetime(year=1800, month=1, day=1, tzinfo=UTC),
+ finished=datetime(year=1900, month=1, day=1, tzinfo=UTC),
archive_path=mnt_filepath.archive_path(enum_type=df_coll_type),
)
@@ -34,23 +34,23 @@ class TestDFCollectionItemBase:
assert isinstance(item, DFCollectionItem)
-@pytest.mark.parametrize("df_coll_type", df_collection_types)
+@pytest.mark.parametrize("df_coll_type", MYSQL_ALLOWED_COLL_TYPES)
class TestDFCollectionItemProperties:
-
@pytest.mark.skip
- def test_filename(self, df_coll_type):
+ def test_filename(self, df_coll_type: DFCollectionType):
pass
-@pytest.mark.parametrize("df_coll_type", df_collection_types)
+@pytest.mark.parametrize("df_coll_type", MYSQL_ALLOWED_COLL_TYPES)
class TestDFCollectionItemMethods:
-
- def test_has_mysql_false(self, mnt_filepath: "GRLDatasets", df_coll_type):
+ def test_has_mysql_false(
+ self, mnt_filepath: GRLDatasets, df_coll_type: DFCollectionType
+ ):
collection = DFCollection(
data_type=df_coll_type,
- offset="100d",
- start=datetime(year=1800, month=1, day=1, tzinfo=timezone.utc),
- finished=datetime(year=1900, month=1, day=1, tzinfo=timezone.utc),
+ offset="100D",
+ start=datetime(year=1800, month=1, day=1, tzinfo=UTC),
+ finished=datetime(year=1900, month=1, day=1, tzinfo=UTC),
archive_path=mnt_filepath.archive_path(enum_type=df_coll_type),
)
@@ -58,13 +58,16 @@ class TestDFCollectionItemMethods:
assert not instance1.has_mysql()
def test_has_mysql_true(
- self, thl_web_rr: PostgresConfig, mnt_filepath: "GRLDatasets", df_coll_type
+ self,
+ thl_web_rr: PostgresConfig,
+ mnt_filepath: GRLDatasets,
+ df_coll_type: DFCollectionType,
):
collection = DFCollection(
data_type=df_coll_type,
- offset="100d",
- start=datetime(year=1800, month=1, day=1, tzinfo=timezone.utc),
- finished=datetime(year=1900, month=1, day=1, tzinfo=timezone.utc),
+ offset="100D",
+ start=datetime(year=1800, month=1, day=1, tzinfo=UTC),
+ finished=datetime(year=1900, month=1, day=1, tzinfo=UTC),
archive_path=mnt_filepath.archive_path(enum_type=df_coll_type),
pg_config=thl_web_rr,
)
@@ -74,5 +77,5 @@ class TestDFCollectionItemMethods:
assert instance2.has_mysql()
@pytest.mark.skip
- def test_update_partial_archive(self, df_coll_type):
+ def test_update_partial_archive(self, df_coll_type: DFCollectionType):
pass
diff --git a/tests/incite/collections/test_df_collection_item_thl_web.py b/tests/incite/collections/test_df_collection_item_thl_web.py
index 8b8bcbe..eeabb41 100644
--- a/tests/incite/collections/test_df_collection_item_thl_web.py
+++ b/tests/incite/collections/test_df_collection_item_thl_web.py
@@ -1,17 +1,19 @@
from __future__ import annotations
-from collections.abc import Generator
-from datetime import datetime, timedelta, timezone
+from collections.abc import Callable, Generator
+from datetime import UTC, datetime, timedelta
from itertools import product as iter_product
from os.path import join as pjoin
from pathlib import Path, PurePath
-from typing import TYPE_CHECKING, Callable
+from typing import TYPE_CHECKING
from uuid import uuid4
import dask.dataframe as dd
import pandas as pd
import pytest
-from distributed import Client, Scheduler, Worker
+from dask.distributed import Client as DaskClient
+from dask.distributed import Scheduler as DaskScheduler
+from dask.distributed import Worker as DaskWorker
# noinspection PyUnresolvedReferences
from distributed.utils_test import (
@@ -22,18 +24,21 @@ from pandera.pandas import DataFrameSchema
from pydantic import FilePath
from generalresearch.incite.base import CollectionItemBase
-from generalresearch.incite.collections import (
- DFCollectionItem,
+from generalresearch.incite.collections.base import (
DFCollectionType,
)
from generalresearch.incite.schemas import ARCHIVE_AFTER
-from generalresearch.models.thl.product import Product
-from generalresearch.models.thl.user import User
from generalresearch.pg_helper import PostgresConfig
from generalresearch.sql_helper import PostgresDsn
if TYPE_CHECKING:
from generalresearch.incite.base import GRLDatasets
+ from generalresearch.incite.collections.base import (
+ DFCollection,
+ DFCollectionItem,
+ )
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.user import User
fake = Faker()
@@ -52,12 +57,11 @@ unsupported_mock_types = {
}
-def combo_object() -> Generator[str, None, None]:
- for x in iter_product(
+def combo_object() -> Generator[tuple[DFCollectionType, str]]:
+ yield from iter_product(
df_collections,
- ["15min", "45min", "1H"],
- ):
- yield from x
+ ["15min", "45min", "1h"],
+ )
class TestDFCollectionItemBase:
@@ -71,8 +75,12 @@ class TestDFCollectionItemBase:
argnames="df_collection_data_type, offset", argvalues=combo_object()
)
class TestDFCollectionItemProperties:
-
- def test_filename(self, df_collection_data_type, df_collection, offset: str):
+ def test_filename(
+ self,
+ df_collection_data_type: DFCollectionType,
+ df_collection: DFCollection,
+ offset: str,
+ ):
for i in df_collection.items:
assert isinstance(i.filename, str)
@@ -88,38 +96,59 @@ class TestDFCollectionItemProperties:
argnames="df_collection_data_type, offset", argvalues=combo_object()
)
class TestDFCollectionItemPropertiesBase:
-
- def test_name(self, df_collection_data_type, offset: str, df_collection):
+ def test_name(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.name, str)
- def test_finish(self, df_collection_data_type, offset: str, df_collection):
+ def test_finish(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.finish, datetime)
- def test_interval(self, df_collection_data_type, offset: str, df_collection):
+ def test_interval(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.interval, pd.Interval)
def test_partial_filename(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection: DFCollection,
):
for i in df_collection.items:
assert isinstance(i.partial_filename, str)
- def test_empty_filename(self, df_collection_data_type, offset: str, df_collection):
+ def test_empty_filename(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.empty_filename, str)
- def test_path(self, df_collection_data_type, offset: str, df_collection):
+ def test_path(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.path, FilePath)
- def test_partial_path(self, df_collection_data_type, offset: str, df_collection):
+ def test_partial_path(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.partial_path, FilePath)
- def test_empty_path(self, df_collection_data_type, offset: str, df_collection):
+ def test_empty_path(
+ self,
+ df_collection: DFCollection,
+ ):
for i in df_collection.items:
assert isinstance(i.empty_path, FilePath)
@@ -135,26 +164,25 @@ class TestDFCollectionItemPropertiesBase:
),
)
class TestDFCollectionItemMethod:
-
- def test_has_mysql(
+ def test_has_postgres(
self,
- df_collection,
- thl_web_rr: PostgresConfig,
+ df_collection_data_type: DFCollectionType,
offset: str,
duration: timedelta,
- df_collection_data_type,
- delete_df_collection,
+ delete_df_collection: Callable[..., None],
+ df_collection: DFCollection,
+ thl_web_rr: PostgresConfig,
):
delete_df_collection(coll=df_collection)
df_collection.pg_config = None
for i in df_collection.items:
- assert not i.has_mysql()
+ assert not i.has_postgres()
# Confirm that the regular connection should work as expected
df_collection.pg_config = thl_web_rr
for i in df_collection.items:
- assert i.has_mysql()
+ assert i.has_postgres()
# Make a fake connection and confirm it does NOT work
df_collection.pg_config = PostgresConfig(
@@ -163,17 +191,14 @@ class TestDFCollectionItemMethod:
statement_timeout=1,
)
for i in df_collection.items:
- assert not i.has_mysql()
+ assert not i.has_postgres()
@pytest.mark.skip
def test_update_partial_archive(
self,
- df_collection,
+ df_collection_data_type: DFCollectionType,
offset: str,
duration: timedelta,
- thl_web_rw: PostgresConfig,
- df_collection_data_type,
- delete_df_collection,
):
# for i in collection.items:
# assert i.update_partial_archive()
@@ -183,29 +208,16 @@ class TestDFCollectionItemMethod:
@pytest.mark.skip
def test_create_partial_archive(
self,
- df_collection,
+ df_collection_data_type: DFCollectionType,
offset: str,
- duration: str,
- create_main_accounts,
- thl_web_rw: PostgresConfig,
- thl_lm,
- df_collection_data_type,
- user_factory: Callable[..., User],
- product: Product,
- client_no_amm,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath: GRLDatasets,
+ duration: timedelta,
):
- assert 1 + 1 == 2
+ pass
def test_dict(
self,
- df_collection_data_type,
- offset: str,
- duration: timedelta,
- df_collection,
- delete_df_collection,
+ df_collection: DFCollection,
+ delete_df_collection: Callable[..., None],
):
delete_df_collection(coll=df_collection)
@@ -225,18 +237,17 @@ class TestDFCollectionItemMethod:
def test_from_mysql(
self,
- df_collection_data_type,
- df_collection,
+ df_collection_data_type: DFCollectionType,
+ df_collection: DFCollection,
offset: str,
duration: timedelta,
- create_main_accounts,
+ create_main_accounts: Callable[..., None],
thl_web_rw: PostgresConfig,
user_factory: Callable[..., User],
product: Product,
- incite_item_factory,
- delete_df_collection,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
if df_collection.data_type in unsupported_mock_types:
return
@@ -249,38 +260,32 @@ class TestDFCollectionItemMethod:
for item in df_collection.items:
# Unlike .from_mysql_ledger(), .from_mysql_standard() will return
# back and empty df with the correct columns in place
- delete_df_collection(coll=df_collection)
- df = item.from_mysql()
if df_collection.data_type == DFCollectionType.LEDGER:
- assert df is None
- else:
- assert df.empty
- assert set(df.columns) == set(df_collection._schema.columns.keys())
+ continue
+ delete_df_collection(coll=df_collection)
+ df = item.from_postgres_standard()
+ assert isinstance(df, pd.DataFrame)
+ assert df.empty
+ assert set(df.columns) == set(df_collection.type_schema.columns.keys())
incite_item_factory(user=u1, item=item)
- df = item.from_mysql()
+ df = item.from_postgres_standard()
+ assert isinstance(df, pd.DataFrame)
assert not df.empty
- assert set(df.columns) == set(df_collection._schema.columns.keys())
- if df_collection.data_type == DFCollectionType.LEDGER:
- # The number of rows in this dataframe will change depending
- # on the mocking of data. It's because if the account has
- # user wallet on, then there will be more transactions for
- # example.
- assert df.shape[0] > 0
+ assert set(df.columns) == set(df_collection.type_schema.columns.keys())
- def test_from_mysql_standard(
+ def test_from_postgres_standard(
self,
- df_collection_data_type,
- df_collection,
+ df_collection_data_type: DFCollectionType,
+ df_collection: DFCollection,
offset: str,
duration: timedelta,
user_factory: Callable[..., User],
product: Product,
- incite_item_factory,
- delete_df_collection,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
if df_collection.data_type in unsupported_mock_types:
return
@@ -292,48 +297,31 @@ class TestDFCollectionItemMethod:
item: DFCollectionItem
if df_collection.data_type == DFCollectionType.LEDGER:
- # We're using parametrize, so this If statement is just to
- # confirm other Item Types will always raise an assertion
- with pytest.raises(expected_exception=AssertionError) as cm:
- res = item.from_mysql_standard()
- assert (
- "Can't call from_mysql_standard for Ledger DFCollectionItem"
- in str(cm.value)
- )
-
continue
# Unlike .from_mysql_ledger(), .from_mysql_standard() will return
# back and empty df with the correct columns in place
- df = item.from_mysql_standard()
+ df = item.from_postgres_standard()
+ assert isinstance(df, pd.DataFrame)
assert df.empty
- assert set(df.columns) == set(df_collection._schema.columns.keys())
+ assert set(df.columns) == set(df_collection.type_schema.columns.keys())
incite_item_factory(user=u1, item=item)
- df = item.from_mysql_standard()
+ df = item.from_postgres_standard()
+ assert isinstance(df, pd.DataFrame)
assert not df.empty
- assert set(df.columns) == set(df_collection._schema.columns.keys())
+ assert set(df.columns) == set(df_collection.type_schema.columns.keys())
assert df.shape[0] > 0
- def test_from_mysql_ledger(
+ def test_from_postgres_ledger(
self,
- df_collection,
- user: User,
- create_main_accounts,
- offset: str,
- duration: timedelta,
- thl_web_rw: PostgresConfig,
- thl_lm,
- df_collection_data_type,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- client_no_amm,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
if df_collection.data_type != DFCollectionType.LEDGER:
return
@@ -348,14 +336,14 @@ class TestDFCollectionItemMethod:
# Okay, now continue with the actual Ledger Item tests... we need
# to ensure that this item.start - item.finish range hasn't had
# any prior transactions created within that range.
- assert item.from_mysql_ledger() is None
+ assert item.from_postgres_ledger() is None
# Create main accounts doesn't matter because it doesn't
# add any transactions to the db
- assert item.from_mysql_ledger() is None
+ assert item.from_postgres_ledger() is None
incite_item_factory(user=u1, item=item)
- df = item.from_mysql_ledger()
+ df = item.from_postgres_ledger()
assert isinstance(df, pd.DataFrame)
# Not only is this a np.int64 to int comparison, but I also know it
@@ -373,19 +361,12 @@ class TestDFCollectionItemMethod:
def test_to_archive(
self,
- df_collection,
- user: User,
- offset: str,
- duration: timedelta,
- df_collection_data_type,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- client_no_amm,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
if df_collection.data_type in unsupported_mock_types:
return
@@ -400,7 +381,7 @@ class TestDFCollectionItemMethod:
# Load up the data that we'll be using for various to_archive
# methods.
- df = item.from_mysql()
+ df = item.from_postgres_standard()
ddf = dd.from_pandas(df, npartitions=1)
# (1) Write the basic archive, the issue is that because it's
@@ -411,17 +392,12 @@ class TestDFCollectionItemMethod:
def test__to_archive(
self,
- df_collection_data_type,
- df_collection,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- offset: str,
- duration: timedelta,
- client_no_amm,
- user: User,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
+ mnt_filepath: GRLDatasets,
):
"""We already have a test for the "non-private" version of this,
which primarily just uses the respective Client to determine if
@@ -443,7 +419,7 @@ class TestDFCollectionItemMethod:
# Load up the data that we'll be using for various to_archive
# methods. Will always be empty pd.DataFrames for now...
- df = item.from_mysql()
+ df = item.from_db()
ddf = dd.from_pandas(df, npartitions=1)
# (1) Confirm a missing ddf (shouldn't bc of type hint) should
@@ -484,19 +460,28 @@ class TestDFCollectionItemMethod:
@pytest.mark.skip
def test_to_archive_numbered_partial(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_initial_load(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_clear_corrupt_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@@ -506,37 +491,36 @@ class TestDFCollectionItemMethod:
argvalues=list(iter_product(df_collections, ["12h", "10D"], [timedelta(days=15)])),
)
class TestDFCollectionItemMethodBase:
-
- @pytest.mark.skip
- def test_path_exists(
- self, df_collection_data_type, offset: str, duration: timedelta
- ):
- pass
-
- @pytest.mark.skip
- def test_next_numbered_path(
- self, df_collection_data_type, offset: str, duration: timedelta
- ):
- pass
-
@pytest.mark.skip
def test_search_highest_numbered_path(
- self, df_collection_data_type, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_tmp_filename(
- self, df_collection_data_type, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
- def test_tmp_path(self, df_collection_data_type, offset: str, duration: timedelta):
+ def test_tmp_path(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
+ ):
pass
def test_is_empty(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
"""
test_has_empty was merged into this because item.has_empty is
@@ -553,7 +537,8 @@ class TestDFCollectionItemMethodBase:
assert item.has_empty()
def test_has_partial_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
for item in df_collection.items:
assert not item.has_partial_archive()
@@ -561,7 +546,8 @@ class TestDFCollectionItemMethodBase:
assert item.has_partial_archive()
def test_has_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
for item in df_collection.items:
# (1) Originally, nothing exists... so let's just make a file and
@@ -598,7 +584,8 @@ class TestDFCollectionItemMethodBase:
assert item.has_archive(include_empty=True)
def test_delete_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
for item in df_collection.items:
item: DFCollectionItem
@@ -621,9 +608,11 @@ class TestDFCollectionItemMethodBase:
assert not item.partial_path.exists()
def test_should_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
- schema: DataFrameSchema = df_collection._schema
+ schema: DataFrameSchema = df_collection.type_schema
+ assert schema.metadata
aa = schema.metadata[ARCHIVE_AFTER]
# It shouldn't be None, it can be timedelta(seconds=0)
@@ -632,19 +621,23 @@ class TestDFCollectionItemMethodBase:
for item in df_collection.items:
item: DFCollectionItem
- if datetime.now(tz=timezone.utc) > item.finish + aa:
+ if datetime.now(tz=UTC) > item.finish + aa:
assert item.should_archive()
else:
assert not item.should_archive()
@pytest.mark.skip
def test_set_empty(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
def test_valid_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
# Originally, nothing has been saved or anything.. so confirm it
# always comes back as None
@@ -668,18 +661,28 @@ class TestDFCollectionItemMethodBase:
@pytest.mark.skip
def test_validate_df(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_from_archive(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
def test__to_dict(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
+ df_collection: DFCollection,
):
for item in df_collection.items:
@@ -698,30 +701,39 @@ class TestDFCollectionItemMethodBase:
@pytest.mark.skip
def test_delete_partial(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_cleanup_partials(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@pytest.mark.skip
def test_delete_dangling_partials(
- self, df_collection_data_type, df_collection, offset: str, duration: timedelta
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ duration: timedelta,
):
pass
@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
-async def test_client(client, s, worker):
+async def test_client(client: DaskClient, s: DaskScheduler, worker: DaskWorker):
"""c,s,a are all required - the secondary Worker (b) is not required"""
- assert isinstance(client, Client)
- assert isinstance(s, Scheduler)
- assert isinstance(worker, Worker)
+ assert isinstance(client, DaskClient)
+ assert isinstance(s, DaskScheduler)
+ assert isinstance(worker, DaskWorker)
@pytest.mark.parametrize(
@@ -730,12 +742,18 @@ async def test_client(client, s, worker):
)
@gen_cluster(client=True, nthreads=[("127.0.0.1", 1)])
@pytest.mark.anyio
-async def test_client_parametrize(c, s, w, df_collection_data_type, offset: str):
+async def test_client_parametrize(
+ c: DaskClient,
+ s: DaskScheduler,
+ w: DaskWorker,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+):
"""c,s,a are all required - the secondary Worker (b) is not required"""
- assert isinstance(c, Client), f"c is not Client, it's {type(c)}"
- assert isinstance(s, Scheduler), f"s is not Scheduler, it's {type(s)}"
- assert isinstance(w, Worker), f"w is not Worker, it's {type(w)}"
+ assert isinstance(c, DaskClient), f"c is not Client, it's {type(c)}"
+ assert isinstance(s, DaskScheduler), f"s is not Scheduler, it's {type(s)}"
+ assert isinstance(w, DaskWorker), f"w is not Worker, it's {type(w)}"
assert df_collection_data_type is not None
assert isinstance(offset, str)
@@ -751,22 +769,15 @@ async def test_client_parametrize(c, s, w, df_collection_data_type, offset: str)
argvalues=list(iter_product(df_collections, ["12h", "10D"], [timedelta(days=15)])),
)
class TestDFCollectionItemFunctionalTest:
-
def test_to_archive_and_ddf(
self,
- df_collection_data_type,
- offset: str,
- duration: timedelta,
- client_no_amm,
- df_collection,
- user: User,
+ client_no_amm: DaskClient,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath: GRLDatasets,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
if df_collection.data_type in unsupported_mock_types:
return
@@ -804,17 +815,11 @@ class TestDFCollectionItemFunctionalTest:
def test_filesize_estimate(
self,
- df_collection,
- user: User,
- offset: str,
- duration: timedelta,
- client_no_amm,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- df_collection_data_type,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath: GRLDatasets,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
"""A functional test to write some Parquet files for the
DFCollection and then confirm that the files get written
@@ -828,8 +833,6 @@ class TestDFCollectionItemFunctionalTest:
import pyarrow.parquet as pq
- from generalresearch.models.thl.user import User
-
if df_collection.data_type in unsupported_mock_types:
return
delete_df_collection(coll=df_collection)
@@ -853,18 +856,13 @@ class TestDFCollectionItemFunctionalTest:
def test_to_archive_client(
self,
- client_no_amm,
- df_collection,
+ client_no_amm: DaskClient,
+ df_collection: DFCollection,
user_factory: Callable[..., User],
product: Product,
- offset: str,
- duration: timedelta,
- df_collection_data_type,
- incite_item_factory,
- delete_df_collection,
- mnt_filepath: GRLDatasets,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
):
- from generalresearch.models.thl.user import User
delete_df_collection(coll=df_collection)
df_collection._client = client_no_amm
@@ -880,7 +878,7 @@ class TestDFCollectionItemFunctionalTest:
# Load up the data that we'll be using for various to_archive
# methods. Will always be empty pd.DataFrames for now...
- df = item.from_mysql()
+ df = item.from_db()
ddf = dd.from_pandas(df, npartitions=1)
assert isinstance(ddf, dd.DataFrame)
@@ -893,7 +891,8 @@ class TestDFCollectionItemFunctionalTest:
@pytest.mark.skip
def test_get_items(
- self, df_collection, product: Product, offset: str, duration: timedelta
+ self,
+ df_collection: DFCollection,
):
with pytest.warns(expected_warning=ResourceWarning) as cm:
df_collection.get_items_last365()
@@ -906,27 +905,22 @@ class TestDFCollectionItemFunctionalTest:
def test_saving_protections(
self,
- client_no_amm,
- df_collection_data_type,
- df_collection,
- incite_item_factory,
- delete_df_collection,
+ df_collection: DFCollection,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
user_factory: Callable[..., User],
product: Product,
- offset: str,
- duration: timedelta,
- mnt_filepath: GRLDatasets,
):
"""Don't allow creating an archive for data that will likely be
overwritten or updated
"""
- from generalresearch.models.thl.user import User
if df_collection.data_type in unsupported_mock_types:
return
u1: User = user_factory(product=product)
- schema: DataFrameSchema = df_collection._schema
+ schema: DataFrameSchema = df_collection.type_schema
+ assert schema.metadata
aa = schema.metadata[ARCHIVE_AFTER]
assert isinstance(aa, timedelta)
@@ -948,21 +942,14 @@ class TestDFCollectionItemFunctionalTest:
def test_empty_item(
self,
- client_no_amm,
- df_collection_data_type,
- df_collection,
- incite_item_factory,
- delete_df_collection,
- user: User,
- offset: str,
- duration: timedelta,
- mnt_filepath: GRLDatasets,
+ df_collection: DFCollection,
+ delete_df_collection: Callable[..., None],
):
delete_df_collection(coll=df_collection)
for item in df_collection.items:
assert not item.has_empty()
- df: pd.DataFrame = item.from_mysql()
+ df: pd.DataFrame = item.from_db()
# We do this check b/c the Ledger returns back None and
# I don't want it to fail when we go to make a ddf
@@ -976,18 +963,13 @@ class TestDFCollectionItemFunctionalTest:
def test_file_touching(
self,
- client_no_amm,
- df_collection_data_type,
- df_collection,
- incite_item_factory,
- delete_df_collection,
+ client_no_amm: DaskClient,
+ df_collection: DFCollection,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
user_factory: Callable[..., User],
product: Product,
- offset: str,
- duration: timedelta,
- mnt_filepath,
):
- from generalresearch.models.thl.user import User
delete_df_collection(coll=df_collection)
df_collection._client = client_no_amm
diff --git a/tests/incite/collections/test_df_collection_thl_marketplaces.py b/tests/incite/collections/test_df_collection_thl_marketplaces.py
index 981f62e..0f79b81 100644
--- a/tests/incite/collections/test_df_collection_thl_marketplaces.py
+++ b/tests/incite/collections/test_df_collection_thl_marketplaces.py
@@ -1,25 +1,26 @@
-from datetime import datetime, timezone
+from collections.abc import Generator
+from datetime import UTC, datetime
from itertools import product
from typing import TYPE_CHECKING
import pytest
from pandera.pandas import Column, DataFrameSchema, Index
-from generalresearch.incite.collections import DFCollection, DFCollectionType
+from generalresearch.incite.collections.base import DFCollection, DFCollectionType
from generalresearch.incite.collections.thl_marketplaces import (
InnovateSurveyHistoryCollection,
MorningSurveyTimeseriesCollection,
SagoSurveyHistoryCollection,
SpectrumSurveyTimeseriesCollection,
)
-from test_utils.incite.conftest import mnt_filepath
if TYPE_CHECKING:
from generalresearch.incite.base import GRLDatasets
+ from generalresearch.pg_helper import PostgresConfig
-def combo_object():
- for x in product(
+def combo_object() -> Generator[tuple[type, str]]:
+ yield from product(
[
InnovateSurveyHistoryCollection,
MorningSurveyTimeseriesCollection,
@@ -27,14 +28,19 @@ def combo_object():
SpectrumSurveyTimeseriesCollection,
],
["5min", "6H", "30D"],
- ):
- yield from x
+ )
@pytest.mark.parametrize("df_coll, offset", combo_object())
class TestDFCollection_thl_marketplaces:
- def test_init(self, mnt_filepath, df_coll, offset, spectrum_rw):
+ def test_init(
+ self,
+ mnt_filepath: GRLDatasets,
+ df_coll: DFCollection,
+ offset: str,
+ spectrum_rw: PostgresConfig,
+ ):
assert issubclass(df_coll, DFCollection)
# This is stupid, but we need to pull the default from the
@@ -43,7 +49,7 @@ class TestDFCollection_thl_marketplaces:
assert isinstance(data_type, DFCollectionType)
# (1) Can't be totally empty, needs a path...
- with pytest.raises(expected_exception=Exception) as cm:
+ with pytest.raises(expected_exception=ValueError):
instance = df_coll()
# (2) Confirm it only needs the archive_path
@@ -57,8 +63,8 @@ class TestDFCollection_thl_marketplaces:
archive_path=mnt_filepath.archive_path(enum_type=data_type),
sql_helper=spectrum_rw,
offset=offset,
- start=datetime(year=2023, month=6, day=1, minute=0, tzinfo=timezone.utc),
- finished=datetime(year=2023, month=6, day=1, minute=5, tzinfo=timezone.utc),
+ start=datetime(year=2023, month=6, day=1, minute=0, tzinfo=UTC),
+ finished=datetime(year=2023, month=6, day=1, minute=5, tzinfo=UTC),
)
assert isinstance(instance, DFCollection)
@@ -66,7 +72,7 @@ class TestDFCollection_thl_marketplaces:
assert isinstance(instance._schema, DataFrameSchema)
assert isinstance(instance._schema.index, Index)
- for c in instance._schema.columns.keys():
+ for c in instance._schema.columns:
assert isinstance(c, str)
col = instance._schema.columns[c]
assert isinstance(col, Column)
diff --git a/tests/incite/collections/test_df_collection_thl_web.py b/tests/incite/collections/test_df_collection_thl_web.py
index b09d44c..7253dd0 100644
--- a/tests/incite/collections/test_df_collection_thl_web.py
+++ b/tests/incite/collections/test_df_collection_thl_web.py
@@ -3,25 +3,20 @@ from __future__ import annotations
from collections.abc import Generator
from datetime import datetime
from itertools import product
-from typing import TYPE_CHECKING
import dask.dataframe as dd
import pandas as pd
import pytest
from pandera.pandas import DataFrameSchema
-from generalresearch.incite.collections import DFCollection, DFCollectionType
-
-if TYPE_CHECKING:
- from generalresearch.incite.base import GRLDatasets
- from generalresearch.incite.collections import (
- DFCollectionItem,
- DFCollectionType,
- )
+from generalresearch.incite.collections.base import (
+ DFCollection,
+ DFCollectionType,
+)
-def combo_object() -> Generator[tuple, None, None]:
- for x in product(
+def combo_object() -> Generator[tuple[DFCollectionType, str]]:
+ yield from product(
[
DFCollectionType.USER,
DFCollectionType.WALL,
@@ -30,9 +25,8 @@ def combo_object() -> Generator[tuple, None, None]:
DFCollectionType.AUDIT_LOG,
DFCollectionType.LEDGER,
],
- ["30min", "1H"],
- ):
- yield from x
+ ["30min", "1h"],
+ )
@pytest.mark.parametrize(
@@ -41,7 +35,10 @@ def combo_object() -> Generator[tuple, None, None]:
class TestDFCollection_thl_web:
def test_init(
- self, df_collection_data_type: DFCollectionType, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
assert isinstance(df_collection_data_type, DFCollectionType)
assert isinstance(df_collection, DFCollection)
@@ -52,12 +49,12 @@ class TestDFCollection_thl_web:
)
class TestDFCollection_thl_web_Properties:
- def test_items(self, df_collection):
+ def test_items(self, df_collection: DFCollection):
assert isinstance(df_collection.items, list)
for i in df_collection.items:
assert i._collection == df_collection
- def test__schema(self, df_collection):
+ def test__schema(self, df_collection: DFCollection):
assert isinstance(df_collection._schema, DataFrameSchema)
@@ -67,16 +64,16 @@ class TestDFCollection_thl_web_Properties:
class TestDFCollection_thl_web_BaseProperties:
@pytest.mark.skip
- def test__interval_range(self, df_collection):
+ def test__interval_range(self, df_collection: DFCollection):
pass
- def test_interval_start(self, df_collection):
+ def test_interval_start(self, df_collection: DFCollection):
assert isinstance(df_collection.interval_start, datetime)
- def test_interval_range(self, df_collection):
+ def test_interval_range(self, df_collection: DFCollection):
assert isinstance(df_collection.interval_range, list)
- def test_progress(self, df_collection):
+ def test_progress(self, df_collection: DFCollection):
assert isinstance(df_collection.progress, pd.DataFrame)
@@ -86,17 +83,21 @@ class TestDFCollection_thl_web_BaseProperties:
class TestDFCollection_thl_web_Methods:
@pytest.mark.skip
- def test_initial_loads(self, df_collection_data_type, df_collection, offset):
+ def test_initial_loads(
+ self, df_collection_data_type, df_collection: DFCollection, offset: str
+ ):
pass
@pytest.mark.skip
def test_fetch_force_rr_latest(
- self, df_collection_data_type, df_collection, offset: str
+ self, df_collection_data_type, df_collection: DFCollection, offset: str
):
pass
@pytest.mark.skip
- def test_force_rr_latest(self, df_collection_data_type, df_collection, offset):
+ def test_force_rr_latest(
+ self, df_collection_data_type, df_collection: DFCollection, offset: str
+ ):
pass
@@ -105,63 +106,108 @@ class TestDFCollection_thl_web_Methods:
)
class TestDFCollection_thl_web_BaseMethods:
- def test_fetch_all_paths(self, df_collection_data_type, offset: str, df_collection):
+ def test_fetch_all_paths(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
res = df_collection.fetch_all_paths(
items=None, force_rr_latest=False, include_partial=False
)
assert isinstance(res, list)
@pytest.mark.skip
- def test_ddf(self, df_collection_data_type, offset: str, df_collection):
+ def test_ddf(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
res = df_collection.ddf()
assert isinstance(res, dd.DataFrame)
# -- cleanup --
@pytest.mark.skip
def test_schedule_cleanup(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
- def test_cleanup(self, df_collection_data_type, offset: str, df_collection):
+ def test_cleanup(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
pass
@pytest.mark.skip
def test_cleanup_partials(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
def test_clear_tmp_archives(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
def test_clear_corrupt_archives(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
def test_rebuild_symlinks(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
# -- Source timing --
@pytest.mark.skip
- def test_get_item(self, df_collection_data_type, offset: str, df_collection):
+ def test_get_item(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
pass
@pytest.mark.skip
- def test_get_item_start(self, df_collection_data_type, offset: str, df_collection):
+ def test_get_item_start(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
pass
@pytest.mark.skip
- def test_get_items(self, df_collection_data_type, offset: str, df_collection):
+ def test_get_items(
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
+ ):
# If we get all the items from the start of the collection, it
# should include all the items!
res1 = df_collection.items
@@ -170,18 +216,27 @@ class TestDFCollection_thl_web_BaseMethods:
@pytest.mark.skip
def test_get_items_from_year(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
def test_get_items_last90(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
@pytest.mark.skip
def test_get_items_last365(
- self, df_collection_data_type, offset: str, df_collection
+ self,
+ df_collection_data_type: DFCollectionType,
+ offset: str,
+ df_collection: DFCollection,
):
pass
diff --git a/tests/incite/mergers/foundations/test_enriched_session.py b/tests/incite/mergers/foundations/test_enriched_session.py
index 47f243e..71b2442 100644
--- a/tests/incite/mergers/foundations/test_enriched_session.py
+++ b/tests/incite/mergers/foundations/test_enriched_session.py
@@ -1,20 +1,35 @@
-from datetime import datetime, timedelta, timezone
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
from decimal import Decimal
from itertools import product
-from typing import Optional
+from typing import TYPE_CHECKING
import dask.dataframe as dd
import pandas as pd
import pytest
+from dask.distributed import Client as DaskClient
from generalresearch.incite.schemas.admin_responses import (
AdminPOPSessionSchema,
)
-from generalresearch.pg_helper import PostgresConfig
-from test_utils.incite.collections.conftest import (
- session_collection,
- wall_collection,
-)
+
+if TYPE_CHECKING:
+ from generalresearch.incite.collections.thl_web import (
+ SessionDFCollection,
+ WallDFCollection,
+ )
+ from generalresearch.incite.mergers.foundations.enriched_session import (
+ EnrichedSessionMerge,
+ )
+ from generalresearch.models.admin.request import (
+ ReportRequest,
+ )
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.session import Session
+ from generalresearch.models.thl.user import User
+ from generalresearch.pg_helper import PostgresConfig
@pytest.mark.parametrize(
@@ -30,17 +45,16 @@ class TestEnrichedSession:
def test_base(
self,
- client_no_amm,
- product,
- user_factory,
- wall_collection,
- session_collection,
- enriched_session_merge,
+ client_no_amm: DaskClient,
+ product: Product,
+ user_factory: Callable[..., User],
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
+ enriched_session_merge: EnrichedSessionMerge,
thl_web_rr: PostgresConfig,
- delete_df_collection,
- incite_item_factory,
+ delete_df_collection: Callable[..., None],
+ incite_item_factory: Callable[..., None],
):
- from generalresearch.models.thl.user import User
delete_df_collection(coll=session_collection)
@@ -77,31 +91,31 @@ class TestEnrichedSession:
class TestEnrichedSessionAdmin:
@pytest.fixture
- def start(self) -> "datetime":
- return datetime(year=2020, month=3, day=14, tzinfo=timezone.utc)
+ def start(self) -> datetime:
+ return datetime(year=2020, month=3, day=14, tzinfo=UTC)
@pytest.fixture
def offset(self) -> str:
return "1d"
@pytest.fixture
- def duration(self) -> Optional["timedelta"]:
+ def duration(self) -> timedelta | None:
return timedelta(days=5)
def test_to_admin_response(
self,
- event_report_request,
- enriched_session_merge,
- client_no_amm,
- wall_collection,
- session_collection,
+ event_report_request: ReportRequest,
+ enriched_session_merge: EnrichedSessionMerge,
+ client_no_amm: DaskClient,
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
thl_web_rr: PostgresConfig,
- session_report_request,
- user_factory,
- start,
- session_factory,
- product_factory,
- delete_df_collection,
+ session_report_request: ReportRequest,
+ user_factory: Callable[..., User],
+ start: datetime,
+ session_factory: Callable[..., Session],
+ product_factory: Callable[..., Product],
+ delete_df_collection: Callable[..., None],
):
delete_df_collection(coll=wall_collection)
delete_df_collection(coll=session_collection)
@@ -112,7 +126,7 @@ class TestEnrichedSessionAdmin:
for p in [p1, p2]:
u = user_factory(product=p)
for i in range(50):
- s = session_factory(
+ _ = session_factory(
user=u,
wall_count=1,
wall_req_cpi=Decimal("1.00"),
diff --git a/tests/incite/mergers/foundations/test_enriched_task_adjust.py b/tests/incite/mergers/foundations/test_enriched_task_adjust.py
index 96c214f..877d22f 100644
--- a/tests/incite/mergers/foundations/test_enriched_task_adjust.py
+++ b/tests/incite/mergers/foundations/test_enriched_task_adjust.py
@@ -1,16 +1,30 @@
+from __future__ import annotations
+
+from collections.abc import Callable
from datetime import timedelta
from itertools import product as iter_product
+from typing import TYPE_CHECKING
import dask.dataframe as dd
import pandas as pd
import pytest
+from dask.distributed import Client as DaskClient
-from test_utils.incite.collections.conftest import (
- wall_collection,
- task_adj_collection,
- session_collection,
-)
-from test_utils.incite.mergers.conftest import enriched_wall_merge
+if TYPE_CHECKING:
+ from generalresearch.incite.collections.thl_web import (
+ SessionDFCollection,
+ TaskAdjustmentDFCollection,
+ WallDFCollection,
+ )
+ from generalresearch.incite.mergers.foundations.enriched_task_adjust import (
+ EnrichedTaskAdjustMerge,
+ )
+ from generalresearch.incite.mergers.foundations.enriched_wall import (
+ EnrichedWallMerge,
+ )
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.user import User
+ from generalresearch.pg_helper import PostgresConfig
@pytest.mark.parametrize(
@@ -27,19 +41,18 @@ class TestEnrichedTaskAdjust:
@pytest.mark.skip
def test_base(
self,
- client_no_amm,
- user_factory,
- product,
- task_adj_collection,
- wall_collection,
- session_collection,
- enriched_wall_merge,
- enriched_task_adjust_merge,
- incite_item_factory,
- delete_df_collection,
- thl_web_rr,
+ client_no_amm: DaskClient,
+ user_factory: Callable[..., User],
+ product: Product,
+ task_adj_collection: TaskAdjustmentDFCollection,
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
+ enriched_wall_merge: EnrichedWallMerge,
+ enriched_task_adjust_merge: EnrichedTaskAdjustMerge,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
+ thl_web_rr: PostgresConfig,
):
- from generalresearch.models.thl.user import User
# -- Build & Setup
delete_df_collection(coll=session_collection)
diff --git a/tests/incite/mergers/foundations/test_enriched_wall.py b/tests/incite/mergers/foundations/test_enriched_wall.py
index 8f4995b..2b9afb8 100644
--- a/tests/incite/mergers/foundations/test_enriched_wall.py
+++ b/tests/incite/mergers/foundations/test_enriched_wall.py
@@ -1,34 +1,33 @@
-from datetime import timedelta, timezone, datetime
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
from decimal import Decimal
from itertools import product as iter_product
-from typing import Optional
+from typing import TYPE_CHECKING
import dask.dataframe as dd
import pandas as pd
import pytest
-
-# noinspection PyUnresolvedReferences
-from distributed.utils_test import (
- gen_cluster,
- client_no_amm,
- loop,
- loop_in_thread,
- cleanup,
- cluster_fixture,
- client,
-)
+from dask.distributed import Client as DaskClient
from generalresearch.incite.mergers.foundations.enriched_wall import (
EnrichedWallMergeItem,
)
-from test_utils.incite.collections.conftest import (
- session_collection,
- wall_collection,
-)
-from test_utils.incite.conftest import incite_item_factory
-from test_utils.incite.mergers.conftest import (
- enriched_wall_merge,
-)
+
+if TYPE_CHECKING:
+ from generalresearch.incite.collections.thl_web import (
+ SessionDFCollection,
+ WallDFCollection,
+ )
+
+ # noinspection PyUnresolvedReferences
+ from generalresearch.incite.mergers.foundations.enriched_wall import (
+ EnrichedWallMerge,
+ )
+ from generalresearch.models.admin.request import ReportRequest
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.session import Session
+ from generalresearch.models.thl.user import User
+ from generalresearch.pg_helper import PostgresConfig
@pytest.mark.parametrize(
@@ -39,17 +38,16 @@ class TestEnrichedWall:
def test_base(
self,
- client_no_amm,
- product,
- user_factory,
- wall_collection,
- thl_web_rr,
- session_collection,
- enriched_wall_merge,
- delete_df_collection,
- incite_item_factory,
+ client_no_amm: DaskClient,
+ product: Product,
+ user_factory: Callable[..., User],
+ wall_collection: WallDFCollection,
+ thl_web_rr: PostgresConfig,
+ session_collection: SessionDFCollection,
+ enriched_wall_merge: EnrichedWallMerge,
+ delete_df_collection: Callable[..., None],
+ incite_item_factory: Callable[..., None],
):
- from generalresearch.models.thl.user import User
# -- Build & Setup
delete_df_collection(coll=session_collection)
@@ -82,15 +80,15 @@ class TestEnrichedWall:
def test_base_item(
self,
- client_no_amm,
- product,
- user_factory,
- wall_collection,
- session_collection,
- enriched_wall_merge,
- delete_df_collection,
- thl_web_rr,
- incite_item_factory,
+ client_no_amm: DaskClient,
+ product: Product,
+ user_factory: Callable[..., User],
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
+ enriched_wall_merge: EnrichedWallMerge,
+ delete_df_collection: Callable[..., None],
+ thl_web_rr: PostgresConfig,
+ incite_item_factory: Callable[..., None],
):
# -- Build & Setup
delete_df_collection(coll=session_collection)
@@ -118,7 +116,7 @@ class TestEnrichedWall:
try:
modified_time1 = path.stat().st_mtime
- except (Exception,):
+ except OSError:
modified_time1 = 0
item.build(
@@ -158,18 +156,23 @@ class TestEnrichedWall:
class TestEnrichedWallToAdmin:
@pytest.fixture
- def start(self) -> "datetime":
- return datetime(year=2020, month=3, day=14, tzinfo=timezone.utc)
+ def start(self) -> datetime:
+ return datetime(year=2020, month=3, day=14, tzinfo=UTC)
@pytest.fixture
def offset(self) -> str:
return "1d"
@pytest.fixture
- def duration(self) -> Optional["timedelta"]:
+ def duration(self) -> timedelta | None:
return timedelta(days=5)
- def test_empty(self, enriched_wall_merge, client_no_amm, start):
+ def test_empty(
+ self,
+ enriched_wall_merge: EnrichedWallMerge,
+ client_no_amm: DaskClient,
+ start: datetime,
+ ):
from generalresearch.models.admin.request import ReportRequest
rr = ReportRequest.model_validate({"interval": "5min", "start": start})
@@ -186,18 +189,18 @@ class TestEnrichedWallToAdmin:
def test_to_admin_response(
self,
- event_report_request,
- enriched_wall_merge,
- client_no_amm,
- wall_collection,
- session_collection,
- thl_web_rr,
- user,
- session_factory,
- delete_df_collection,
- product_factory,
- user_factory,
- start,
+ event_report_request: ReportRequest,
+ enriched_wall_merge: EnrichedWallMerge,
+ client_no_amm: DaskClient,
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
+ thl_web_rr: PostgresConfig,
+ user: User,
+ session_factory: Callable[..., Session],
+ delete_df_collection: Callable[..., None],
+ product_factory: Callable[..., Product],
+ user_factory: Callable[..., User],
+ start: datetime,
):
delete_df_collection(coll=wall_collection)
delete_df_collection(coll=session_collection)
@@ -208,7 +211,7 @@ class TestEnrichedWallToAdmin:
for p in [p1, p2]:
u = user_factory(product=p)
for i in range(50):
- s = session_factory(
+ _ = session_factory(
user=u,
wall_count=2,
wall_req_cpi=Decimal("1.00"),
diff --git a/tests/incite/mergers/foundations/test_user_id_product.py b/tests/incite/mergers/foundations/test_user_id_product.py
index f96bfb4..8c4b2f7 100644
--- a/tests/incite/mergers/foundations/test_user_id_product.py
+++ b/tests/incite/mergers/foundations/test_user_id_product.py
@@ -1,24 +1,22 @@
-from datetime import timedelta, datetime, timezone
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
from itertools import product
+from typing import TYPE_CHECKING
import pandas as pd
import pytest
-
-# noinspection PyUnresolvedReferences
-from distributed.utils_test import (
- gen_cluster,
- client_no_amm,
- loop,
- loop_in_thread,
- cleanup,
- cluster_fixture,
- client,
-)
+from dask.distributed import Client as DaskClient
from generalresearch.incite.mergers.foundations.user_id_product import (
UserIdProductMergeItem,
)
-from test_utils.incite.mergers.conftest import user_id_product_merge
+
+if TYPE_CHECKING:
+ # noinspection PyUnresolvedReferences
+ from generalresearch.incite.mergers.foundations.user_id_product import (
+ UserIdProductMerge,
+ )
@pytest.mark.parametrize(
@@ -27,25 +25,28 @@ from test_utils.incite.mergers.conftest import user_id_product_merge
product(
["12h", "3D"],
[timedelta(days=5)],
- [
- (datetime.now(tz=timezone.utc) - timedelta(days=35)).replace(
- microsecond=0
- )
- ],
+ [(datetime.now(tz=UTC) - timedelta(days=35)).replace(microsecond=0)],
)
),
)
class TestUserIDProduct:
@pytest.mark.skip
- def test_base(self, client_no_amm, user_id_product_merge):
+ def test_base(
+ self, client_no_amm: DaskClient, user_id_product_merge: UserIdProductMerge
+ ):
ddf = user_id_product_merge.ddf()
df = client_no_amm.compute(collections=ddf, sync=True)
assert isinstance(df, pd.DataFrame)
assert not df.empty
@pytest.mark.skip
- def test_base_item(self, client_no_amm, user_id_product_merge, user_collection):
+ def test_base_item(
+ self,
+ client_no_amm: DaskClient,
+ user_id_product_merge: UserIdProductMerge,
+ user_collection,
+ ):
assert len(user_id_product_merge.items) == 1
for item in user_id_product_merge.items:
@@ -55,7 +56,7 @@ class TestUserIDProduct:
try:
modified_time1 = path.stat().st_mtime
- except (Exception,):
+ except OSError:
modified_time1 = 0
user_id_product_merge.build(client=client_no_amm, user_coll=user_collection)
@@ -64,7 +65,9 @@ class TestUserIDProduct:
assert modified_time2 > modified_time1
@pytest.mark.skip
- def test_read(self, client_no_amm, user_id_product_merge):
+ def test_read(
+ self, client_no_amm: DaskClient, user_id_product_merge: UserIdProductMerge
+ ):
users_ddf = user_id_product_merge.ddf()
df = client_no_amm.compute(collections=users_ddf, sync=True)
diff --git a/tests/incite/mergers/test_merge_collection.py b/tests/incite/mergers/test_merge_collection.py
index ec507bc..7ed3996 100644
--- a/tests/incite/mergers/test_merge_collection.py
+++ b/tests/incite/mergers/test_merge_collection.py
@@ -1,17 +1,22 @@
-from datetime import datetime, timezone, timedelta
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
from itertools import product
+from typing import TYPE_CHECKING
import pandas as pd
import pytest
from pandera.pandas import DataFrameSchema
-from generalresearch.incite.mergers import (
+from generalresearch.incite.mergers.base import (
MergeCollection,
MergeType,
)
-from test_utils.incite.conftest import mnt_filepath
-merge_types = list(e for e in MergeType if e != MergeType.TEST)
+if TYPE_CHECKING:
+ from generalresearch.incite.base import GRLDatasets
+
+merge_types = [e for e in MergeType if e != MergeType.TEST]
@pytest.mark.parametrize(
@@ -21,17 +26,20 @@ merge_types = list(e for e in MergeType if e != MergeType.TEST)
merge_types,
["5min", "6h", "14D"],
[timedelta(days=30)],
- [
- (datetime.now(tz=timezone.utc) - timedelta(days=35)).replace(
- microsecond=0
- )
- ],
+ [(datetime.now(tz=UTC) - timedelta(days=35)).replace(microsecond=0)],
)
),
)
class TestMergeCollection:
- def test_init(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_init(
+ self,
+ merge_type: MergeType,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ mnt_filepath: GRLDatasets,
+ ):
with pytest.raises(expected_exception=ValueError) as cm:
MergeCollection(archive_path=mnt_filepath.data_src)
assert "Must explicitly provide a merge_type" in str(cm.value)
@@ -42,7 +50,14 @@ class TestMergeCollection:
)
assert instance.merge_type == merge_type
- def test_items(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_items(
+ self,
+ mnt_filepath: GRLDatasets,
+ merge_type: MergeType,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ ):
instance = MergeCollection(
merge_type=merge_type,
offset=offset,
@@ -53,7 +68,14 @@ class TestMergeCollection:
assert len(instance.interval_range) == len(instance.items)
- def test_progress(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_progress(
+ self,
+ mnt_filepath: GRLDatasets,
+ merge_type: MergeType,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ ):
instance = MergeCollection(
merge_type=merge_type,
offset=offset,
@@ -67,7 +89,14 @@ class TestMergeCollection:
assert instance.progress.shape[1] == 7
assert instance.progress["group_by"].isnull().all()
- def test_schema(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_schema(
+ self,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ mnt_filepath: GRLDatasets,
+ merge_type: MergeType,
+ ):
instance = MergeCollection(
merge_type=merge_type,
archive_path=mnt_filepath.archive_path(enum_type=merge_type),
@@ -75,7 +104,14 @@ class TestMergeCollection:
assert isinstance(instance._schema, DataFrameSchema)
- def test_load(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_load(
+ self,
+ mnt_filepath: GRLDatasets,
+ merge_type: MergeType,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ ):
instance = MergeCollection(
merge_type=merge_type,
start=start,
@@ -87,7 +123,14 @@ class TestMergeCollection:
# Confirm that there are no archives available yet
assert instance.progress.has_archive.eq(False).all()
- def test_get_items(self, mnt_filepath, merge_type, offset, duration, start):
+ def test_get_items(
+ self,
+ mnt_filepath: GRLDatasets,
+ merge_type: MergeType,
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ ):
instance = MergeCollection(
start=start,
finished=start + duration,
diff --git a/tests/incite/mergers/test_merge_collection_item.py b/tests/incite/mergers/test_merge_collection_item.py
index 96f8789..baf1bc4 100644
--- a/tests/incite/mergers/test_merge_collection_item.py
+++ b/tests/incite/mergers/test_merge_collection_item.py
@@ -1,17 +1,19 @@
-from datetime import datetime, timezone, timedelta
+from __future__ import annotations
+
+from datetime import timedelta
from itertools import product
from pathlib import PurePath
+from typing import TYPE_CHECKING
import pytest
-from generalresearch.incite.mergers import MergeCollectionItem, MergeType
-from generalresearch.incite.mergers.foundations.enriched_session import (
- EnrichedSessionMerge,
-)
-from generalresearch.incite.mergers.foundations.enriched_wall import (
- EnrichedWallMerge,
-)
-from test_utils.incite.mergers.conftest import merge_collection
+from generalresearch.incite.mergers.base import MergeType
+
+if TYPE_CHECKING:
+ from generalresearch.incite.mergers.base import (
+ MergeCollection,
+ MergeCollectionItem,
+ )
@pytest.mark.parametrize(
@@ -26,7 +28,10 @@ from test_utils.incite.mergers.conftest import merge_collection
)
class TestMergeCollectionItem:
- def test_file_naming(self, merge_collection, offset, duration, start):
+ def test_file_naming(
+ self,
+ merge_collection: MergeCollection,
+ ):
assert len(merge_collection.items) == 25
items: list[MergeCollectionItem] = merge_collection.items
@@ -41,7 +46,10 @@ class TestMergeCollectionItem:
assert i._collection.offset in i.filename
assert i.start.strftime("%Y-%m-%d-%H-%M-%S") in i.filename
- def test_archives(self, merge_collection, offset, duration, start):
+ def test_archives(
+ self,
+ merge_collection: MergeCollection,
+ ):
assert len(merge_collection.items) == 25
for i in merge_collection.items:
@@ -51,10 +59,13 @@ class TestMergeCollectionItem:
assert not i.has_partial_archive()
assert i.has_archive() == i.path_exists(generic_path=i.path)
- res = set([i.should_archive() for i in merge_collection.items])
+ res = {i.should_archive() for i in merge_collection.items}
assert len(res) == 1
- def test_item_to_archive(self, merge_collection, offset, duration, start):
+ def test_item_to_archive(
+ self,
+ merge_collection: MergeCollection,
+ ):
for item in merge_collection.items:
item: MergeCollectionItem
assert not item.has_archive()
diff --git a/tests/incite/mergers/test_pop_ledger.py b/tests/incite/mergers/test_pop_ledger.py
index 6f96108..9ec188b 100644
--- a/tests/incite/mergers/test_pop_ledger.py
+++ b/tests/incite/mergers/test_pop_ledger.py
@@ -1,18 +1,28 @@
-from datetime import timedelta, datetime, timezone
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
from itertools import product as iter_product
-from typing import Optional
+from typing import TYPE_CHECKING
import pandas as pd
import pytest
-from distributed.utils_test import client_no_amm
+from dask.distributed import Client as DaskClient
from generalresearch.incite.schemas.mergers.pop_ledger import (
numerical_col_names,
)
-from test_utils.incite.collections.conftest import ledger_collection
-from test_utils.incite.conftest import mnt_filepath, incite_item_factory
-from test_utils.incite.mergers.conftest import pop_ledger_merge
-from test_utils.managers.ledger.conftest import create_main_accounts
+
+if TYPE_CHECKING:
+ from generalresearch.incite.base import GRLDatasets
+ from generalresearch.incite.collections.thl_web import (
+ LedgerDFCollection,
+ SessionDFCollection,
+ )
+ from generalresearch.incite.mergers.pop_ledger import PopLedgerMerge
+ from generalresearch.managers.thl.ledger_manager.thl_ledger import ThlLedgerManager
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.user import User
@pytest.mark.parametrize(
@@ -27,25 +37,25 @@ from test_utils.managers.ledger.conftest import create_main_accounts
class TestMergePOPLedger:
@pytest.fixture
- def start(self) -> "datetime":
- return datetime(year=2020, month=3, day=14, tzinfo=timezone.utc)
+ def start(self) -> datetime:
+ return datetime(year=2020, month=3, day=14, tzinfo=UTC)
@pytest.fixture
- def duration(self) -> Optional["timedelta"]:
+ def duration(self) -> timedelta | None:
return timedelta(days=5)
def test_base(
self,
- client_no_amm,
- ledger_collection,
- pop_ledger_merge,
- product,
- user_factory,
- create_main_accounts,
- thl_lm,
- delete_df_collection,
- incite_item_factory,
- delete_ledger_db,
+ client_no_amm: DaskClient,
+ ledger_collection: LedgerDFCollection,
+ pop_ledger_merge: PopLedgerMerge,
+ product: Product,
+ user_factory: Callable[..., User],
+ create_main_accounts: Callable[..., None],
+ thl_ledger_manager: ThlLedgerManager,
+ delete_df_collection: Callable[..., None],
+ incite_item_factory: Callable[..., None],
+ delete_ledger_db: Callable[..., None],
):
from generalresearch.models.thl.ledger import LedgerAccount
@@ -79,19 +89,19 @@ class TestMergePOPLedger:
# --
- user_wallet_account: LedgerAccount = thl_lm.get_account_or_create_user_wallet(
- user=u
+ thl_ledger_manager.get_account_or_create_user_wallet(user=u)
+ cash_account: LedgerAccount = thl_ledger_manager.get_account_cash()
+ rev_account: LedgerAccount = (
+ thl_ledger_manager.get_account_task_complete_revenue()
)
- cash_account: LedgerAccount = thl_lm.get_account_cash()
- rev_account: LedgerAccount = thl_lm.get_account_task_complete_revenue()
item_finishes = [i.finish for i in ledger_collection.items]
item_finishes.sort(reverse=True)
last_item_finish = item_finishes[0]
# Pure SQL based lookups
- cash_balance: int = thl_lm.get_account_balance(account=cash_account)
- rev_balance: int = thl_lm.get_account_balance(account=rev_account)
+ cash_balance: int = thl_ledger_manager.get_account_balance(account=cash_account)
+ rev_balance: int = thl_ledger_manager.get_account_balance(account=rev_account)
assert cash_balance > rev_balance
# (1) Test Cash Account
@@ -129,39 +139,42 @@ class TestMergePOPLedger:
def test_pydantic_init(
self,
- client_no_amm,
- ledger_collection,
- pop_ledger_merge,
- mnt_filepath,
- product,
- user_factory,
- create_main_accounts,
- offset,
- duration,
- start,
- thl_lm,
- incite_item_factory,
- delete_df_collection,
- delete_ledger_db,
- session_collection,
+ client_no_amm: DaskClient,
+ ledger_collection: LedgerDFCollection,
+ pop_ledger_merge: PopLedgerMerge,
+ mnt_filepath: GRLDatasets,
+ product: Product,
+ user_factory: Callable[..., User],
+ create_main_accounts: Callable[..., None],
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ thl_ledger_manager: ThlLedgerManager,
+ incite_item_factory: Callable[..., None],
+ delete_df_collection: Callable[..., None],
+ delete_ledger_db: Callable[..., None],
+ session_collection: SessionDFCollection,
):
+ from generalresearch.models.thl.finance import ProductBalances
from generalresearch.models.thl.ledger import LedgerAccount
from generalresearch.models.thl.product import Product
- from generalresearch.models.thl.finance import ProductBalances
u = user_factory(product=product, created=session_collection.start)
assert ledger_collection.finished is not None
assert isinstance(u.product, Product)
delete_ledger_db()
- create_main_accounts(),
+ create_main_accounts()
+
delete_df_collection(coll=ledger_collection)
- bp_account: LedgerAccount = thl_lm.get_account_or_create_bp_wallet(
+ bp_account: LedgerAccount = thl_ledger_manager.get_account_or_create_bp_wallet(
product=u.product
)
- cash_account: LedgerAccount = thl_lm.get_account_cash()
- rev_account: LedgerAccount = thl_lm.get_account_task_complete_revenue()
+ cash_account: LedgerAccount = thl_ledger_manager.get_account_cash()
+ rev_account: LedgerAccount = (
+ thl_ledger_manager.get_account_task_complete_revenue()
+ )
for item in ledger_collection.items:
incite_item_factory(item=item, user=u)
@@ -191,8 +204,10 @@ class TestMergePOPLedger:
assert instance.payout == instance.net == instance.bp_payment_credit
assert instance.available_balance < instance.net
assert instance.available_balance + instance.retainer == instance.net
- assert instance.balance == thl_lm.get_account_balance(bp_account)
- assert df["bp_payment.CREDIT"].sum() == thl_lm.get_account_balance(bp_account)
+ assert instance.balance == thl_ledger_manager.get_account_balance(bp_account)
+ assert df["bp_payment.CREDIT"].sum() == thl_ledger_manager.get_account_balance(
+ bp_account
+ )
# (2) Filter by the Cash Account
ddf = pop_ledger_merge.ddf(
@@ -205,7 +220,7 @@ class TestMergePOPLedger:
)
df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True)
- cash_balance: int = thl_lm.get_account_balance(account=cash_account)
+ cash_balance: int = thl_ledger_manager.get_account_balance(account=cash_account)
assert df["bp_payment.CREDIT"].sum() == 0
assert cash_balance > 0
assert df["mp_payment.CREDIT"].sum() == 0
@@ -222,7 +237,7 @@ class TestMergePOPLedger:
)
df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True)
- rev_balance: int = thl_lm.get_account_balance(account=rev_account)
+ rev_balance: int = thl_ledger_manager.get_account_balance(account=rev_account)
assert rev_balance == 0
assert df["bp_payment.CREDIT"].sum() == 0
assert df["mp_payment.DEBIT"].sum() == 0
@@ -230,27 +245,28 @@ class TestMergePOPLedger:
def test_resample(
self,
- client_no_amm,
- ledger_collection,
- pop_ledger_merge,
- mnt_filepath,
- user_factory,
- product,
- create_main_accounts,
- offset,
- duration,
- start,
- thl_lm,
- delete_df_collection,
- incite_item_factory,
+ client_no_amm: DaskClient,
+ ledger_collection: LedgerDFCollection,
+ pop_ledger_merge: PopLedgerMerge,
+ mnt_filepath: GRLDatasets,
+ user_factory: Callable[..., User],
+ product: Product,
+ create_main_accounts: Callable[..., None],
+ offset: str,
+ duration: timedelta,
+ start: datetime,
+ thl_ledger_manager: ThlLedgerManager,
+ delete_df_collection: Callable[..., None],
+ incite_item_factory: Callable[..., None],
):
- from generalresearch.models.thl.user import User
assert ledger_collection.finished is not None
delete_df_collection(coll=ledger_collection)
u1: User = user_factory(product=product)
- bp_account = thl_lm.get_account_or_create_bp_wallet(product=u1.product)
+ bp_account = thl_ledger_manager.get_account_or_create_bp_wallet(
+ product=u1.product
+ )
for item in ledger_collection.items:
incite_item_factory(user=u1, item=item)
@@ -280,7 +296,7 @@ class TestMergePOPLedger:
assert isinstance(df.index, pd.Index)
assert isinstance(df.index, pd.DatetimeIndex)
- bp_account_balance = thl_lm.get_account_balance(account=bp_account)
+ thl_ledger_manager.get_account_balance(account=bp_account)
# Initial sum
initial_sum = df.sum().sum()
diff --git a/tests/incite/mergers/test_ym_survey_merge.py b/tests/incite/mergers/test_ym_survey_merge.py
index 4c2df6b..d83a98c 100644
--- a/tests/incite/mergers/test_ym_survey_merge.py
+++ b/tests/incite/mergers/test_ym_survey_merge.py
@@ -1,25 +1,28 @@
-from datetime import timedelta, timezone, datetime
+from __future__ import annotations
+
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
from itertools import product
+from typing import TYPE_CHECKING
import pandas as pd
import pytest
+from dask.distributed import Client as DaskClient
+
+if TYPE_CHECKING:
+ from generalresearch.incite.collections.thl_web import (
+ SessionDFCollection,
+ WallDFCollection,
+ )
+ from generalresearch.incite.mergers.foundations.enriched_session import (
+ EnrichedSessionMerge,
+ )
+ from generalresearch.incite.mergers.ym_survey_wall import YMSurveyWallMerge
+ from generalresearch.models.thl.product import Product
+ from generalresearch.models.thl.user import User
+ from generalresearch.pg_helper import PostgresConfig
# noinspection PyUnresolvedReferences
-from distributed.utils_test import (
- gen_cluster,
- client_no_amm,
- loop,
- loop_in_thread,
- cleanup,
- cluster_fixture,
- client,
-)
-
-from test_utils.incite.collections.conftest import wall_collection, session_collection
-from test_utils.incite.mergers.conftest import (
- enriched_session_merge,
- ym_survey_wall_merge,
-)
@pytest.mark.parametrize(
@@ -28,11 +31,7 @@ from test_utils.incite.mergers.conftest import (
product(
["12h", "3D"],
[timedelta(days=30)],
- [
- (datetime.now(tz=timezone.utc) - timedelta(days=35)).replace(
- microsecond=0
- )
- ],
+ [(datetime.now(tz=UTC) - timedelta(days=35)).replace(microsecond=0)],
)
),
)
@@ -46,18 +45,17 @@ class TestYMSurveyMerge:
def test_base(
self,
- client_no_amm,
- user_factory,
- product,
- ym_survey_wall_merge,
- wall_collection,
- session_collection,
- enriched_session_merge,
- delete_df_collection,
- incite_item_factory,
- thl_web_rr,
+ client_no_amm: DaskClient,
+ user_factory: Callable[..., User],
+ product: Product,
+ ym_survey_wall_merge: YMSurveyWallMerge,
+ wall_collection: WallDFCollection,
+ session_collection: SessionDFCollection,
+ enriched_session_merge: EnrichedSessionMerge,
+ delete_df_collection: Callable[..., None],
+ incite_item_factory: Callable[..., None],
+ thl_web_rr: PostgresConfig,
):
- from generalresearch.models.thl.user import User
delete_df_collection(coll=session_collection)
user: User = user_factory(product=product, created=session_collection.start)
@@ -85,10 +83,10 @@ class TestYMSurveyMerge:
assert enriched_session_merge.progress.has_archive.eq(True).all()
ddf = enriched_session_merge.ddf()
- df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True)
+ df1: pd.DataFrame | None = client_no_amm.compute(collections=ddf, sync=True)
- assert isinstance(df, pd.DataFrame)
- assert not df.empty
+ assert isinstance(df1, pd.DataFrame)
+ assert not df1.empty
# --
@@ -102,18 +100,18 @@ class TestYMSurveyMerge:
# --
ddf = ym_survey_wall_merge.ddf()
- df: pd.DataFrame = client_no_amm.compute(collections=ddf, sync=True)
+ df2: pd.DataFrame | None = client_no_amm.compute(collections=ddf, sync=True)
- assert isinstance(df, pd.DataFrame)
- assert not df.empty
+ assert isinstance(df2, pd.DataFrame)
+ assert not df2.empty
# --
- assert df.product_id.nunique() == 1
- assert df.team_id.nunique() == 1
- assert df.source.nunique() > 1
+ assert df2.product_id.nunique() == 1
+ assert df2.team_id.nunique() == 1
+ assert df2.source.nunique() > 1
- started_min_ts = df.started.min()
- started_max_ts = df.started.max()
+ started_min_ts = df2.started.min()
+ started_max_ts = df2.started.max()
assert type(started_min_ts) is pd.Timestamp
assert type(started_max_ts) is pd.Timestamp
diff --git a/tests/incite/schemas/test_admin_responses.py b/tests/incite/schemas/test_admin_responses.py
index 43aa399..d2658ea 100644
--- a/tests/incite/schemas/test_admin_responses.py
+++ b/tests/incite/schemas/test_admin_responses.py
@@ -1,15 +1,17 @@
-from datetime import datetime, timezone, timedelta
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta
from random import sample
-from typing import List
import numpy as np
import pandas as pd
+import pandera as pa
import pytest
from generalresearch.incite.schemas import empty_dataframe_from_schema
from generalresearch.incite.schemas.admin_responses import (
- AdminPOPSchema,
SIX_HOUR_SECONDS,
+ AdminPOPSchema,
)
from generalresearch.locales import Localelator
@@ -17,12 +19,14 @@ from generalresearch.locales import Localelator
class TestAdminPOPSchema:
schema_df = empty_dataframe_from_schema(AdminPOPSchema)
countries = list(Localelator().get_all_countries())[:5]
- dates = [datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10)]
+ dates = [
+ datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10) # noqa
+ ]
@classmethod
def assign_valid_vals(cls, df: pd.DataFrame) -> pd.DataFrame:
for c in df.columns:
- check_attrs: dict = AdminPOPSchema.columns[c].checks[0].statistics
+ check_attrs = AdminPOPSchema.columns[c].checks[0].statistics
df[c] = np.random.randint(
check_attrs["min_value"], check_attrs["max_value"], df.shape[0]
)
@@ -30,7 +34,7 @@ class TestAdminPOPSchema:
return df
def test_empty(self):
- with pytest.raises(Exception):
+ with pytest.raises(pa.errors.SchemaError):
AdminPOPSchema.validate(pd.DataFrame())
def test_new_empty_df(self):
@@ -43,7 +47,7 @@ class TestAdminPOPSchema:
def test_valid(self):
# (1) Works with raw naive datetime
dates = [
- datetime(year=2024, month=1, day=i, tzinfo=None).isoformat()
+ datetime(year=2024, month=1, day=i, tzinfo=None).isoformat() # noqa
for i in range(1, 10)
]
df = pd.DataFrame(
@@ -58,7 +62,10 @@ class TestAdminPOPSchema:
assert isinstance(df, pd.DataFrame)
# (2) Works with isoformat naive datetime
- dates = [datetime(year=2024, month=1, day=i, tzinfo=None) for i in range(1, 10)]
+ dates = [
+ datetime(year=2024, month=1, day=i, tzinfo=None) # noqa
+ for i in range(1, 10)
+ ]
df = pd.DataFrame(
index=pd.MultiIndex.from_product(
iterables=[dates, self.countries], names=["index0", "index1"]
@@ -72,8 +79,7 @@ class TestAdminPOPSchema:
def test_index_tz_parser(self):
tz_dates = [
- datetime(year=2024, month=1, day=i, tzinfo=timezone.utc)
- for i in range(1, 10)
+ datetime(year=2024, month=1, day=i, tzinfo=UTC) for i in range(1, 10)
]
df = pd.DataFrame(
@@ -85,16 +91,16 @@ class TestAdminPOPSchema:
df = self.assign_valid_vals(df)
# Initially, they're all set with a timezone
- timestmaps: List[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
- assert all([ts.tz == timezone.utc for ts in timestmaps])
+ timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
+ assert all(ts.tz == UTC for ts in timestmaps)
# After validation, the timezone is removed
df = AdminPOPSchema.validate(df)
- timestmaps: List[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
- assert all([ts.tz is None for ts in timestmaps])
+ timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
+ assert all(ts.tz is None for ts in timestmaps)
def test_index_tz_no_future_beyond_one_year(self):
- now = datetime.now(tz=timezone.utc)
+ now = datetime.now(tz=UTC)
tz_dates = [now + timedelta(days=i * 365) for i in range(1, 10)]
df = pd.DataFrame(
@@ -125,12 +131,12 @@ class TestAdminPOPSchema:
df = self.assign_valid_vals(df)
vals = [i for i in df.index.get_level_values(1)]
- assert all([isinstance(v, float) for v in vals])
+ assert all(isinstance(v, float) for v in vals)
df = AdminPOPSchema.validate(df, lazy=True)
vals = [i for i in df.index.get_level_values(1)]
- assert all([isinstance(v, str) for v in vals])
+ assert all(isinstance(v, str) for v in vals)
# --- int to str ---
@@ -144,12 +150,12 @@ class TestAdminPOPSchema:
df = self.assign_valid_vals(df)
vals = [i for i in df.index.get_level_values(1)]
- assert all([isinstance(v, int) for v in vals])
+ assert all(isinstance(v, int) for v in vals)
df = AdminPOPSchema.validate(df, lazy=True)
vals = [i for i in df.index.get_level_values(1)]
- assert all([isinstance(v, str) for v in vals])
+ assert all(isinstance(v, str) for v in vals)
# a = 1
assert isinstance(df, pd.DataFrame)
@@ -157,9 +163,7 @@ class TestAdminPOPSchema:
def test_invalid_parsing(self):
# (1) Timezones AND as strings will still parse correctly
tz_str_dates = [
- datetime(
- year=2024, month=1, day=1, minute=i, tzinfo=timezone.utc
- ).isoformat()
+ datetime(year=2024, month=1, day=1, minute=i, tzinfo=UTC).isoformat()
for i in range(1, 10)
]
df = pd.DataFrame(
@@ -173,12 +177,12 @@ class TestAdminPOPSchema:
df = AdminPOPSchema.validate(df, lazy=True)
assert isinstance(df, pd.DataFrame)
- timestmaps: List[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
- assert all([ts.tz is None for ts in timestmaps])
+ timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
+ assert all(ts.tz is None for ts in timestmaps)
# (2) Timezones are removed
dates = [
- datetime(year=2024, month=1, day=1, minute=i, tzinfo=timezone.utc)
+ datetime(year=2024, month=1, day=1, minute=i, tzinfo=UTC)
for i in range(1, 10)
]
df = pd.DataFrame(
@@ -190,13 +194,13 @@ class TestAdminPOPSchema:
df = self.assign_valid_vals(df)
# Has tz before validation, and none after
- timestmaps: List[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
- assert all([ts.tz is timezone.utc for ts in timestmaps])
+ timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
+ assert all(ts.tz is UTC for ts in timestmaps)
df = AdminPOPSchema.validate(df, lazy=True)
- timestmaps: List[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
- assert all([ts.tz is None for ts in timestmaps])
+ timestmaps: list[pd.Timestamp] = [i for i in df.index.get_level_values(0)]
+ assert all(ts.tz is None for ts in timestmaps)
def test_clipping(self):
df = pd.DataFrame(
diff --git a/tests/incite/schemas/test_thl_web.py b/tests/incite/schemas/test_thl_web.py
index 7f4434b..9b34ce0 100644
--- a/tests/incite/schemas/test_thl_web.py
+++ b/tests/incite/schemas/test_thl_web.py
@@ -16,7 +16,7 @@ class TestWallSchema:
df = pd.DataFrame(columns=THLWallSchema.columns.keys())
- with pytest.raises(SchemaError) as cm:
+ with pytest.raises(SchemaError):
THLWallSchema.validate(df)
def test_no_rows(self):
@@ -24,7 +24,7 @@ class TestWallSchema:
df = pd.DataFrame(index=["uuid"], columns=THLWallSchema.columns.keys())
- with pytest.raises(SchemaError) as cm:
+ with pytest.raises(SchemaError):
THLWallSchema.validate(df)
def test_new_empty_df(self):
@@ -50,7 +50,7 @@ class TestSessionSchema:
df = pd.DataFrame(columns=THLSessionSchema.columns.keys())
df.set_index("uuid", inplace=True)
- with pytest.raises(SchemaError) as cm:
+ with pytest.raises(SchemaError):
THLSessionSchema.validate(df)
def test_no_rows(self):
@@ -58,7 +58,7 @@ class TestSessionSchema:
df = pd.DataFrame(index=["id"], columns=THLSessionSchema.columns.keys())
- with pytest.raises(SchemaError) as cm:
+ with pytest.raises(SchemaError):
THLSessionSchema.validate(df)
def test_new_empty_df(self):
diff --git a/tests/incite/test_collection_base.py b/tests/incite/test_collection_base.py
index 7e6605f..1a664a2 100644
--- a/tests/incite/test_collection_base.py
+++ b/tests/incite/test_collection_base.py
@@ -1,7 +1,10 @@
-from datetime import datetime, timedelta, timezone
+from __future__ import annotations
+
+from datetime import UTC, datetime, timedelta, timezone
from os.path import exists as pexists
from os.path import join as pjoin
from pathlib import Path
+from typing import TYPE_CHECKING
from uuid import uuid4
import numpy as np
@@ -10,21 +13,21 @@ import pytest
from _pytest._code.code import ExceptionInfo
from generalresearch.incite.base import CollectionBase
-from test_utils.incite.conftest import mnt_filepath
-AGO_15min = (datetime.now(tz=timezone.utc) - timedelta(minutes=15)).replace(
- microsecond=0
-)
-AGO_1HR = (datetime.now(tz=timezone.utc) - timedelta(hours=1)).replace(microsecond=0)
-AGO_2HR = (datetime.now(tz=timezone.utc) - timedelta(hours=2)).replace(microsecond=0)
+if TYPE_CHECKING:
+ from generalresearch.incite.base import GRLDatasets
+
+AGO_15min = (datetime.now(tz=UTC) - timedelta(minutes=15)).replace(microsecond=0)
+AGO_1HR = (datetime.now(tz=UTC) - timedelta(hours=1)).replace(microsecond=0)
+AGO_2HR = (datetime.now(tz=UTC) - timedelta(hours=2)).replace(microsecond=0)
class TestCollectionBase:
- def test_init(self, mnt_filepath):
+ def test_init(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
assert instance.df.empty is True
- def test_init_df(self, mnt_filepath):
+ def test_init_df(self, mnt_filepath: GRLDatasets):
# Only an empty pd.DataFrame can ever be provided
instance = CollectionBase(
df=pd.DataFrame({}), archive_path=mnt_filepath.data_src
@@ -46,11 +49,11 @@ class TestCollectionBase:
)
assert "Do not provide a pd.DataFrame" in str(cm.value)
- def test_init_start(self, mnt_filepath):
+ def test_init_start(self, mnt_filepath: GRLDatasets):
with pytest.raises(expected_exception=ValueError) as cm:
cm: ExceptionInfo
CollectionBase(
- start=datetime.now(tz=timezone.utc) - timedelta(days=10),
+ start=datetime.now(tz=UTC) - timedelta(days=10),
archive_path=mnt_filepath.data_src,
)
assert "Collection.start must not have microseconds" in str(cm.value)
@@ -66,9 +69,7 @@ class TestCollectionBase:
assert "Timezone is not UTC" in str(cm.value)
instance = CollectionBase(archive_path=mnt_filepath.data_src)
- assert instance.start == datetime(
- year=2018, month=1, day=1, tzinfo=timezone.utc
- )
+ assert instance.start == datetime(year=2018, month=1, day=1, tzinfo=UTC)
with pytest.raises(expected_exception=ValueError) as cm:
cm: ExceptionInfo
@@ -79,7 +80,7 @@ class TestCollectionBase:
cm.value
)
- def test_init_archive_path(self, mnt_filepath):
+ def test_init_archive_path(self, mnt_filepath: GRLDatasets):
"""DirectoryPath is apparently smart enough to confirm that the
directory path exists.
"""
@@ -104,7 +105,7 @@ class TestCollectionBase:
CollectionBase(archive_path=new_path)
assert "Path does not point to a directory" in str(cm.value)
- def test_init_offset(self, mnt_filepath):
+ def test_init_offset(self, mnt_filepath: GRLDatasets):
with pytest.raises(expected_exception=ValueError) as cm:
cm: ExceptionInfo
CollectionBase(offset="1:X", archive_path=mnt_filepath.data_src)
@@ -112,7 +113,7 @@ class TestCollectionBase:
with pytest.raises(expected_exception=ValueError) as cm:
cm: ExceptionInfo
- CollectionBase(offset=f"59sec", archive_path=mnt_filepath.data_src)
+ CollectionBase(offset="59sec", archive_path=mnt_filepath.data_src)
assert "Must be equal to, or longer than 1 min" in str(cm.value)
with pytest.raises(expected_exception=ValueError) as cm:
@@ -123,14 +124,14 @@ class TestCollectionBase:
class TestCollectionBaseProperties:
- def test_items(self, mnt_filepath):
+ def test_items(self, mnt_filepath: GRLDatasets):
with pytest.raises(expected_exception=NotImplementedError) as cm:
cm: ExceptionInfo
instance = CollectionBase(archive_path=mnt_filepath.data_src)
- x = instance.items
+ _ = instance.items
assert "Must override" in str(cm.value)
- def test_interval_range(self, mnt_filepath):
+ def test_interval_range(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
# Private method requires the end parameter
with pytest.raises(expected_exception=AssertionError) as cm:
@@ -145,14 +146,14 @@ class TestCollectionBaseProperties:
instance._interval_range(end=datetime.now(tz=tz))
assert "Timezones must match" in str(cm.value)
- res = instance._interval_range(end=datetime.now(tz=timezone.utc))
+ res = instance._interval_range(end=datetime.now(tz=UTC))
assert isinstance(res, pd.IntervalIndex)
assert res.closed_left
assert res.is_non_overlapping_monotonic
assert res.is_monotonic_increasing
assert res.is_unique
- def test_interval_range2(self, mnt_filepath):
+ def test_interval_range2(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
assert isinstance(instance.interval_range, list)
@@ -171,16 +172,16 @@ class TestCollectionBaseProperties:
)
assert len(instance.interval_range) == 2
- def test_progress(self, mnt_filepath):
+ def test_progress(self, mnt_filepath: GRLDatasets):
with pytest.raises(expected_exception=NotImplementedError) as cm:
cm: ExceptionInfo
instance = CollectionBase(
start=AGO_15min, offset="3min", archive_path=mnt_filepath.data_src
)
- x = instance.progress
+ _ = instance.progress
assert "Must override" in str(cm.value)
- def test_progress2(self, mnt_filepath):
+ def test_progress2(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(
start=AGO_2HR,
offset="15min",
@@ -189,10 +190,10 @@ class TestCollectionBaseProperties:
assert instance.df.empty
with pytest.raises(expected_exception=NotImplementedError) as cm:
- df = instance.progress
+ _ = instance.progress
assert "Must override" in str(cm.value)
- def test_items2(self, mnt_filepath):
+ def test_items2(self, mnt_filepath: GRLDatasets):
"""There can't be a test for this because the Items need a path whic
isn't possible in the generic form
"""
@@ -202,7 +203,7 @@ class TestCollectionBaseProperties:
with pytest.raises(expected_exception=NotImplementedError) as cm:
cm: ExceptionInfo
- items = instance.items
+ _ = instance.items
assert "Must override" in str(cm.value)
# item = items[-3]
@@ -213,19 +214,19 @@ class TestCollectionBaseProperties:
# assert str(df.product_id.dtype) == "object"
# assert str(ddf.product_id.dtype) == "string"
- def test_items3(self, mnt_filepath):
+ def test_items3(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(
start=AGO_2HR,
offset="15min",
archive_path=mnt_filepath.data_src,
)
with pytest.raises(expected_exception=NotImplementedError) as cm:
- item = instance.items[0]
+ _ = instance.items[0]
assert "Must override" in str(cm.value)
class TestCollectionBaseMethodsCleanup:
- def test_fetch_force_rr_latest(self, mnt_filepath):
+ def test_fetch_force_rr_latest(self, mnt_filepath: GRLDatasets):
coll = CollectionBase(archive_path=mnt_filepath.data_src)
with pytest.raises(expected_exception=Exception) as cm:
@@ -233,7 +234,7 @@ class TestCollectionBaseMethodsCleanup:
coll.fetch_force_rr_latest(sources=[])
assert "Must override" in str(cm.value)
- def test_fetch_all_paths(self, mnt_filepath):
+ def test_fetch_all_paths(self, mnt_filepath: GRLDatasets):
coll = CollectionBase(archive_path=mnt_filepath.data_src)
with pytest.raises(expected_exception=NotImplementedError) as cm:
@@ -244,19 +245,19 @@ class TestCollectionBaseMethodsCleanup:
assert "Must override" in str(cm.value)
-class TestCollectionBaseMethodsCleanup:
+class TestCollectionBaseMethodsCleanup2:
@pytest.mark.skip
- def test_cleanup_partials(self, mnt_filepath):
+ def test_cleanup_partials(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
assert instance.cleanup_partials() is None # it doesn't return anything
- def test_clear_tmp_archives(self, mnt_filepath):
+ def test_clear_tmp_archives(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
assert instance.clear_tmp_archives() is None # it doesn't return anything
@pytest.mark.skip
- def test_clear_corrupt_archives(self, mnt_filepath):
+ def test_clear_corrupt_archives(self, mnt_filepath: GRLDatasets):
"""TODO: expand this so it actually has corrupt archives that we
check to see if they're removed
"""
@@ -264,14 +265,14 @@ class TestCollectionBaseMethodsCleanup:
assert instance.clear_corrupt_archives() is None # it doesn't return anything
@pytest.mark.skip
- def test_rebuild_symlinks(self, mnt_filepath):
+ def test_rebuild_symlinks(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
assert instance.rebuild_symlinks() is None
class TestCollectionBaseMethodsSourceTiming:
- def test_get_item(self, mnt_filepath):
+ def test_get_item(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
i = pd.Interval(left=1, right=2, closed="left")
@@ -279,40 +280,40 @@ class TestCollectionBaseMethodsSourceTiming:
instance.get_item(interval=i)
assert "Must override" in str(cm.value)
- def test_get_item_start(self, mnt_filepath):
+ def test_get_item_start(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
- dt = datetime.now(tz=timezone.utc)
+ dt = datetime.now(tz=UTC)
start = pd.Timestamp(dt)
with pytest.raises(expected_exception=NotImplementedError) as cm:
instance.get_item_start(start=start)
assert "Must override" in str(cm.value)
- def test_get_items(self, mnt_filepath):
+ def test_get_items(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
- dt = datetime.now(tz=timezone.utc)
+ dt = datetime.now(tz=UTC)
with pytest.raises(expected_exception=NotImplementedError) as cm:
instance.get_items(since=dt)
assert "Must override" in str(cm.value)
- def test_get_items_from_year(self, mnt_filepath):
+ def test_get_items_from_year(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
with pytest.raises(expected_exception=NotImplementedError) as cm:
instance.get_items_from_year(year=2020)
assert "Must override" in str(cm.value)
- def test_get_items_last90(self, mnt_filepath):
+ def test_get_items_last90(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
with pytest.raises(expected_exception=NotImplementedError) as cm:
instance.get_items_last90()
assert "Must override" in str(cm.value)
- def test_get_items_last365(self, mnt_filepath):
+ def test_get_items_last365(self, mnt_filepath: GRLDatasets):
instance = CollectionBase(archive_path=mnt_filepath.data_src)
with pytest.raises(expected_exception=NotImplementedError) as cm:
diff --git a/tests/incite/test_collection_base_item.py b/tests/incite/test_collection_base_item.py
index e5d1d02..b9f1c26 100644
--- a/tests/incite/test_collection_base_item.py
+++ b/tests/incite/test_collection_base_item.py
@@ -1,6 +1,9 @@
-from datetime import datetime, timezone
+from __future__ import annotations
+
+from datetime import UTC, datetime
from os.path import join as pjoin
from pathlib import Path
+from typing import TYPE_CHECKING
from uuid import uuid4
import dask.dataframe as dd
@@ -10,10 +13,13 @@ from pydantic import ValidationError
from generalresearch.incite.base import CollectionItemBase
+if TYPE_CHECKING:
+ from generalresearch.incite.base import GRLDatasets
+
class TestCollectionItemBase:
def test_init(self):
- dt = datetime.now(tz=timezone.utc).replace(microsecond=0)
+ dt = datetime.now(tz=UTC).replace(microsecond=0)
instance = CollectionItemBase()
instance2 = CollectionItemBase(start=dt)
@@ -25,7 +31,7 @@ class TestCollectionItemBase:
assert 0 == instance.start.microsecond == instance2.start.microsecond
def test_init_start(self):
- dt = datetime.now(tz=timezone.utc)
+ dt = datetime.now(tz=UTC)
with pytest.raises(expected_exception=ValidationError) as cm:
CollectionItemBase(start=dt)
@@ -40,20 +46,20 @@ class TestCollectionItemBaseProperties:
def test_finish(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.finish
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.finish
def test_interval(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.interval
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.interval
def test_filename(self):
instance = CollectionItemBase()
with pytest.raises(expected_exception=NotImplementedError) as cm:
- res = instance.filename
+ _ = instance.filename
assert "Do not use CollectionItemBase directly" in str(cm.value)
@@ -61,7 +67,7 @@ class TestCollectionItemBaseProperties:
instance = CollectionItemBase()
with pytest.raises(expected_exception=NotImplementedError) as cm:
- res = instance.filename
+ _ = instance.filename
assert "Do not use CollectionItemBase directly" in str(cm.value)
@@ -69,27 +75,27 @@ class TestCollectionItemBaseProperties:
instance = CollectionItemBase()
with pytest.raises(expected_exception=NotImplementedError) as cm:
- res = instance.filename
+ _ = instance.filename
assert "Do not use CollectionItemBase directly" in str(cm.value)
def test_path(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.path
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.path
def test_partial_path(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.partial_path
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.partial_path
def test_empty_path(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.empty_path
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.empty_path
class TestCollectionItemBaseMethods:
@@ -106,41 +112,41 @@ class TestCollectionItemBaseMethods:
instance = CollectionItemBase()
with pytest.raises(expected_exception=NotImplementedError) as cm:
- res = instance.tmp_filename()
+ _ = instance.tmp_filename()
assert "Do not use CollectionItemBase directly" in str(cm.value)
def test_tmp_path(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.tmp_path()
+ with pytest.raises(expected_exception=AttributeError):
+ instance.tmp_path()
def test_is_empty(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.is_empty()
+ with pytest.raises(expected_exception=AttributeError):
+ instance.is_empty()
def test_has_empty(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.has_empty()
+ with pytest.raises(expected_exception=AttributeError):
+ instance.has_empty()
def test_has_partial_archive(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.has_partial_archive()
+ with pytest.raises(expected_exception=AttributeError):
+ instance.has_partial_archive()
@pytest.mark.parametrize("include_empty", [True, False])
- def test_has_archive(self, include_empty):
+ def test_has_archive(self, include_empty: bool):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.has_archive(include_empty=include_empty)
+ with pytest.raises(expected_exception=AttributeError):
+ instance.has_archive(include_empty=include_empty)
- def test_delete_archive_file(self, mnt_filepath):
+ def test_delete_archive_file(self, mnt_filepath: GRLDatasets):
path1 = Path(pjoin(mnt_filepath.data_src, f"{uuid4().hex}.zip"))
# Confirm it doesn't exist, and that delete_archive() doesn't throw
@@ -155,7 +161,7 @@ class TestCollectionItemBaseMethods:
CollectionItemBase.delete_archive(generic_path=path1)
assert not path1.exists()
- def test_delete_archive_dir(self, mnt_filepath):
+ def test_delete_archive_dir(self, mnt_filepath: GRLDatasets):
path1 = Path(pjoin(mnt_filepath.data_src, f"{uuid4().hex}"))
# Confirm it doesn't exist, and that delete_archive() doesn't throw
@@ -174,20 +180,20 @@ class TestCollectionItemBaseMethods:
def test_should_archive(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.should_archive()
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.should_archive()
def test_set_empty(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.set_empty()
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.set_empty()
def test_valid_archive(self):
instance = CollectionItemBase()
- with pytest.raises(expected_exception=AttributeError) as cm:
- res = instance.valid_archive(generic_path=None, sample=None)
+ with pytest.raises(expected_exception=AttributeError):
+ _ = instance.valid_archive(generic_path=None, sample=None)
class TestCollectionItemBaseMethodsORM:
@@ -197,11 +203,11 @@ class TestCollectionItemBaseMethodsORM:
pass
@pytest.mark.parametrize("is_partial", [True, False])
- def test_to_archive(self, is_partial):
+ def test_to_archive(self, is_partial: bool):
instance = CollectionItemBase()
with pytest.raises(expected_exception=NotImplementedError) as cm:
- res = instance.to_archive(
+ _ = instance.to_archive(
ddf=dd.from_pandas(data=pd.DataFrame()), is_partial=is_partial
)
assert "Must override" in str(cm.value)
diff --git a/tests/incite/test_grl_flow.py b/tests/incite/test_grl_flow.py
index c632f9a..6aea182 100644
--- a/tests/incite/test_grl_flow.py
+++ b/tests/incite/test_grl_flow.py
@@ -1,15 +1,16 @@
class TestGRLFlow:
def test_init(self, mnt_filepath, thl_web_rr):
+ from generalresearch.incite.collections.thl_web import (
+ LedgerDFCollection,
+ TaskAdjustmentDFCollection,
+ )
from generalresearch.incite.defaults import (
ledger_df_collection,
task_df_collection,
- pop_ledger as plm,
)
-
- from generalresearch.incite.collections.thl_web import (
- LedgerDFCollection,
- TaskAdjustmentDFCollection,
+ from generalresearch.incite.defaults import (
+ pop_ledger as plm,
)
from generalresearch.incite.mergers.pop_ledger import PopLedgerMerge
diff --git a/tests/incite/test_interval_idx.py b/tests/incite/test_interval_idx.py
index ea2bced..04d0bb2 100644
--- a/tests/incite/test_interval_idx.py
+++ b/tests/incite/test_interval_idx.py
@@ -1,12 +1,13 @@
+from datetime import UTC, datetime
+
import pandas as pd
-from datetime import datetime, timezone, timedelta
class TestIntervalIndex:
def test_init(self):
- start = datetime(year=2000, month=1, day=1)
- end = datetime(year=2000, month=1, day=10)
+ start = datetime(year=2000, month=1, day=1, tzinfo=UTC)
+ end = datetime(year=2000, month=1, day=10, tzinfo=UTC)
iv_r: pd.IntervalIndex = pd.interval_range(
start=start, end=end, freq="1d", closed="left"
@@ -17,7 +18,7 @@ class TestIntervalIndex:
# If the offset is longer than the end - start it will not
# error. It will simply have 0 rows.
iv_r: pd.IntervalIndex = pd.interval_range(
- start=start, end=end, freq="30d", closed="left"
+ start=start, end=end, freq="30D", closed="left"
)
assert isinstance(iv_r, pd.IntervalIndex)
assert len(iv_r.to_list()) == 0