wehub-resource-sync
593b94c120
pytest / Unit Tests (push) Has been cancelled
pytest / Integration (integration_tests_a) (push) Has been cancelled
pytest / Integration (integration_tests_b) (push) Has been cancelled
pytest / Integration (integration_tests_c) (push) Has been cancelled
pytest / Integration (integration_tests_d) (push) Has been cancelled
pytest / Integration (integration_tests_e) (push) Has been cancelled
pytest / Integration (integration_tests_f) (push) Has been cancelled
pytest / Integration (integration_tests_g) (push) Has been cancelled
pytest / Integration (integration_tests_h) (push) Has been cancelled
pytest / Integration (integration_tests_i) (push) Has been cancelled
pytest / Integration (integration_tests_j) (push) Has been cancelled
pytest / Distributed (distributed_a) (push) Has been cancelled
pytest / Distributed (distributed_b) (push) Has been cancelled
pytest / Distributed (distributed_c) (push) Has been cancelled
pytest / Distributed (distributed_d) (push) Has been cancelled
pytest / Distributed (distributed_e) (push) Has been cancelled
pytest / Distributed (distributed_f) (push) Has been cancelled
pytest / Minimal Install (push) Has been cancelled
pytest / Event File (push) Has been cancelled
pytest (slow) / py-slow (push) Has been cancelled
Publish JSON Schema / publish-schema (push) Has been cancelled
106 lines
2.8 KiB
Python
106 lines
2.8 KiB
Python
import datetime
|
|
import time
|
|
|
|
import pandas as pd
|
|
import pytest
|
|
from dateutil.parser import parse
|
|
|
|
from ludwig.api import LudwigModel
|
|
from ludwig.constants import (
|
|
BACKEND,
|
|
BINARY,
|
|
DATE,
|
|
EPOCHS,
|
|
FILL_WITH_CONST,
|
|
INPUT_FEATURES,
|
|
MISSING_VALUE_STRATEGY,
|
|
NAME,
|
|
OUTPUT_FEATURES,
|
|
PREPROCESSING,
|
|
RAY,
|
|
TRAINER,
|
|
TYPE,
|
|
)
|
|
from ludwig.utils.date_utils import create_vector_from_datetime_obj
|
|
|
|
ray = pytest.importorskip("ray")
|
|
|
|
pytestmark = [
|
|
pytest.mark.distributed,
|
|
pytest.mark.distributed_f,
|
|
]
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def string_date_df() -> "pd.DataFrame":
|
|
df = pd.DataFrame.from_dict(
|
|
{
|
|
"date_feature": [str(datetime.datetime.now()) for i in range(100)],
|
|
"binary_feature": [i % 2 for i in range(100)],
|
|
}
|
|
)
|
|
return df
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def int_date_df() -> "pd.DataFrame":
|
|
df = pd.DataFrame.from_dict(
|
|
{
|
|
"date_feature": [time.time_ns() for i in range(100)],
|
|
"binary_feature": [i % 2 for i in range(100)],
|
|
}
|
|
)
|
|
return df
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def float_date_df() -> "pd.DataFrame":
|
|
df = pd.DataFrame.from_dict(
|
|
{
|
|
"date_feature": [time.time() for i in range(100)],
|
|
"binary_feature": [i % 2 for i in range(100)],
|
|
}
|
|
)
|
|
return df
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"date_df",
|
|
[
|
|
pytest.param("string_date_df", id="string_date"),
|
|
pytest.param("int_date_df", id="int_date"),
|
|
pytest.param("float_date_df", id="float_date"),
|
|
],
|
|
)
|
|
def test_date_feature_formats(date_df, request, ray_cluster_2cpu):
|
|
df = request.getfixturevalue(date_df)
|
|
|
|
config = {
|
|
INPUT_FEATURES: [
|
|
{
|
|
NAME: "date_feature",
|
|
TYPE: DATE,
|
|
PREPROCESSING: {MISSING_VALUE_STRATEGY: FILL_WITH_CONST, "fill_value": "1970-01-01 00:00:00"},
|
|
}
|
|
],
|
|
OUTPUT_FEATURES: [{NAME: "binary_feature", TYPE: BINARY}],
|
|
TRAINER: {EPOCHS: 2},
|
|
BACKEND: {TYPE: RAY, "processor": {TYPE: "dask"}},
|
|
}
|
|
|
|
fill_value = create_vector_from_datetime_obj(parse("1970-01-01 00:00:00"))
|
|
|
|
model = LudwigModel(config)
|
|
preprocessed = model.preprocess(df)
|
|
|
|
# Because parsing errors are suppressed, we want to ensure that the data was preprocessed correctly. Sample data is
|
|
# drawn from the current time, so the recorded years should not match the fill value's year.
|
|
for date in preprocessed.training_set.to_df().compute().iloc[:, 0].values:
|
|
assert date[0] != fill_value[0]
|
|
|
|
for date in preprocessed.validation_set.to_df().compute().iloc[:, 0].values:
|
|
assert date[0] != fill_value[0]
|
|
|
|
for date in preprocessed.test_set.to_df().compute().iloc[:, 0].values:
|
|
assert date[0] != fill_value[0]
|