wehub-resource-sync
593b94c120
pytest / Unit Tests (push) Has been cancelled
pytest / Integration (integration_tests_a) (push) Has been cancelled
pytest / Integration (integration_tests_b) (push) Has been cancelled
pytest / Integration (integration_tests_c) (push) Has been cancelled
pytest / Integration (integration_tests_d) (push) Has been cancelled
pytest / Integration (integration_tests_e) (push) Has been cancelled
pytest / Integration (integration_tests_f) (push) Has been cancelled
pytest / Integration (integration_tests_g) (push) Has been cancelled
pytest / Integration (integration_tests_h) (push) Has been cancelled
pytest / Integration (integration_tests_i) (push) Has been cancelled
pytest / Integration (integration_tests_j) (push) Has been cancelled
pytest / Distributed (distributed_a) (push) Has been cancelled
pytest / Distributed (distributed_b) (push) Has been cancelled
pytest / Distributed (distributed_c) (push) Has been cancelled
pytest / Distributed (distributed_d) (push) Has been cancelled
pytest / Distributed (distributed_e) (push) Has been cancelled
pytest / Distributed (distributed_f) (push) Has been cancelled
pytest / Minimal Install (push) Has been cancelled
pytest / Event File (push) Has been cancelled
pytest (slow) / py-slow (push) Has been cancelled
Publish JSON Schema / publish-schema (push) Has been cancelled
72 lines
2.2 KiB
Python
72 lines
2.2 KiB
Python
"""Utilities for producing synthetic test data that is convergence-friendly."""
|
|
|
|
from collections import namedtuple
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
from sklearn.model_selection import train_test_split
|
|
|
|
RANDOM_SEED = 42
|
|
NUMBER_OBSERVATIONS = 200
|
|
|
|
GeneratedData = namedtuple("GeneratedData", "train_df validation_df test_df")
|
|
|
|
|
|
def get_feature_configs():
|
|
input_features = [
|
|
{"name": "x", "type": "number"},
|
|
]
|
|
output_features = [
|
|
{
|
|
"name": "y",
|
|
"type": "number",
|
|
"loss": {"type": "mean_squared_error"},
|
|
"decoder": {
|
|
"num_fc_layers": 2,
|
|
"fc_output_size": 64,
|
|
},
|
|
}
|
|
]
|
|
|
|
return input_features, output_features
|
|
|
|
|
|
def get_generated_data():
|
|
# function generates simple training data that guarantee convergence
|
|
# within 30 epochs for suitable config
|
|
|
|
# generate data
|
|
np.random.seed(RANDOM_SEED)
|
|
x = np.array(range(NUMBER_OBSERVATIONS)).reshape(-1, 1)
|
|
y = 2 * x + 1 + np.random.normal(size=x.shape[0]).reshape(-1, 1)
|
|
raw_df = pd.DataFrame(np.concatenate((x, y), axis=1), columns=["x", "y"])
|
|
|
|
# create training data
|
|
train, valid_test = train_test_split(raw_df, train_size=0.7)
|
|
|
|
# create validation and test data
|
|
validation, test = train_test_split(valid_test, train_size=0.5)
|
|
|
|
return GeneratedData(train, validation, test)
|
|
|
|
|
|
def get_generated_data_for_optimizer():
|
|
# function generates simple training data that guarantee convergence
|
|
# within 30 epochs for suitable config
|
|
|
|
# generate data
|
|
np.random.seed(RANDOM_SEED)
|
|
x = np.array(range(NUMBER_OBSERVATIONS)).reshape(-1, 1)
|
|
y = 2 * x + 1 + np.random.normal(size=x.shape[0]).reshape(-1, 1)
|
|
raw_df = pd.DataFrame(np.concatenate((x, y), axis=1), columns=["x", "y"])
|
|
raw_df["x"] = (raw_df["x"] - raw_df["x"].min()) / (raw_df["x"].max() - raw_df["x"].min())
|
|
raw_df["y"] = (raw_df["y"] - raw_df["y"].min()) / (raw_df["y"].max() - raw_df["y"].min())
|
|
|
|
# create training data
|
|
train, valid_test = train_test_split(raw_df, train_size=0.7)
|
|
|
|
# create validation and test data
|
|
validation, test = train_test_split(valid_test, train_size=0.5)
|
|
|
|
return GeneratedData(train, validation, test)
|