# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ # ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ # ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ # ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ # ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ # ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ # ┃ Copyright (c) 2017, the Perspective Authors. ┃ # ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ # ┃ This file is part of the Perspective library, distributed under the terms ┃ # ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ # ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ from datetime import date, datetime from io import StringIO import numpy as np import pandas as pd from pytest import mark import perspective as psp client = psp.Server().new_local_client() Table = client.table def arrow_bytes_to_pandas(view): import pyarrow with pyarrow.ipc.open_stream(pyarrow.BufferReader(view.to_arrow())) as reader: return reader.read_pandas() class TestTablePandas(object): def test_empty_table(self): tbl = Table([]) assert tbl.size() == 0 assert tbl.schema() == {} def test_table_dataframe(self): d = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] data = pd.DataFrame(d) tbl = Table(data) assert tbl.size() == 2 assert tbl.schema() == {"index": "integer", "a": "integer", "b": "integer"} assert tbl.view().to_records() == [ {"a": 1, "b": 2, "index": 0}, {"a": 3, "b": 4, "index": 1}, ] def test_table_dataframe_column_order(self): d = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}] data = pd.DataFrame(d, columns=["b", "c", "a", "d"]) tbl = Table(data) assert tbl.size() == 2 assert tbl.columns() == ["index", "b", "c", "a", "d"] def test_table_dataframe_selective_column_order(self): d = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}] data = pd.DataFrame(d, columns=["b", "c", "a"]) tbl = Table(data) assert tbl.size() == 2 assert tbl.columns() == ["index", "b", "c", "a"] def test_table_dataframe_does_not_mutate(self): # make sure we don't mutate the dataframe that a user passes in data = pd.DataFrame( { "a": np.array([None, 1, None, 2], dtype=object), "b": np.array([1.5, None, 2.5, None], dtype=object), } ) assert data["a"].tolist() == [None, 1, None, 2] assert data["b"].tolist() == [1.5, None, 2.5, None] tbl = Table(data) assert tbl.size() == 4 assert tbl.schema() == {"index": "integer", "a": "integer", "b": "float"} assert data["a"].tolist() == [None, 1, None, 2] assert data["b"].tolist() == [1.5, None, 2.5, None] @mark.skip(reason="Deprecated support for Series") def test_table_date_series(self, util): data = util.make_series(freq="D") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == {"index": "date", "0": "float"} assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1)), util.to_timestamp(datetime(2000, 1, 2)), util.to_timestamp(datetime(2000, 1, 3)), util.to_timestamp(datetime(2000, 1, 4)), util.to_timestamp(datetime(2000, 1, 5)), util.to_timestamp(datetime(2000, 1, 6)), util.to_timestamp(datetime(2000, 1, 7)), util.to_timestamp(datetime(2000, 1, 8)), util.to_timestamp(datetime(2000, 1, 9)), util.to_timestamp(datetime(2000, 1, 10)), ] @mark.skip(reason="Deprecated support for Series") def test_table_time_series(self, util): data = util.make_series(freq="H") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == {"index": "datetime", "0": "float"} assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 1, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 2, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 3, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 4, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 5, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 6, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 7, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 8, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 9, 0, 0)), ] @mark.skip(reason="pyarrow dataframe does not support date inference") def test_table_dataframe_infer_date(self, util): data = util.make_dataframe(freq="ME") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == { "index": "date", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 31)), util.to_timestamp(datetime(2000, 2, 29)), util.to_timestamp(datetime(2000, 3, 31)), util.to_timestamp(datetime(2000, 4, 30)), util.to_timestamp(datetime(2000, 5, 31)), util.to_timestamp(datetime(2000, 6, 30)), util.to_timestamp(datetime(2000, 7, 31)), util.to_timestamp(datetime(2000, 8, 31)), util.to_timestamp(datetime(2000, 9, 30)), util.to_timestamp(datetime(2000, 10, 31)), ] def test_table_dataframe_infer_date_fixed(self, util): data = util.make_dataframe(freq="ME") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == { "index": "datetime", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 31)), util.to_timestamp(datetime(2000, 2, 29)), util.to_timestamp(datetime(2000, 3, 31)), util.to_timestamp(datetime(2000, 4, 30)), util.to_timestamp(datetime(2000, 5, 31)), util.to_timestamp(datetime(2000, 6, 30)), util.to_timestamp(datetime(2000, 7, 31)), util.to_timestamp(datetime(2000, 8, 31)), util.to_timestamp(datetime(2000, 9, 30)), util.to_timestamp(datetime(2000, 10, 31)), ] def test_table_dataframe_infer_time(self, util): data = util.make_dataframe(freq="h") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == { "index": "datetime", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 1, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 2, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 3, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 4, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 5, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 6, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 7, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 8, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 9, 0, 0)), ] @mark.skip(reason="pyarrow dataframe does not support date inference") def test_table_dataframe_year_start_index(self, util): data = util.make_dataframe(freq="YS") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == { "index": "date", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2001, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2002, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2003, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2004, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2005, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2006, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2007, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2008, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2009, 1, 1, 0, 0, 0)), ] def test_table_dataframe_year_start_index_fixed(self, util): data = util.make_dataframe(freq="YS") tbl = Table(data) assert tbl.size() == 10 assert tbl.schema() == { "index": "datetime", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2001, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2002, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2003, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2004, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2005, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2006, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2007, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2008, 1, 1, 0, 0, 0)), util.to_timestamp(datetime(2009, 1, 1, 0, 0, 0)), ] @mark.skip(reason="pyarrow dataframe does not support date inference") def test_table_dataframe_quarter_index(self, util): data = util.make_dataframe(size=4, freq="QE") tbl = Table(data) assert tbl.size() == 4 assert tbl.schema() == { "index": "date", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 3, 31, 0, 0, 0)), util.to_timestamp(datetime(2000, 6, 30, 0, 0, 0)), util.to_timestamp(datetime(2000, 9, 30, 0, 0, 0)), util.to_timestamp(datetime(2000, 12, 31, 0, 0, 0)), ] def test_table_dataframe_quarter_index_fixed(self, util): data = util.make_dataframe(size=4, freq="QE") tbl = Table(data) assert tbl.size() == 4 assert tbl.schema() == { "index": "datetime", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 3, 31, 0, 0, 0)), util.to_timestamp(datetime(2000, 6, 30, 0, 0, 0)), util.to_timestamp(datetime(2000, 9, 30, 0, 0, 0)), util.to_timestamp(datetime(2000, 12, 31, 0, 0, 0)), ] def test_table_dataframe_minute_index(self, util): data = util.make_dataframe(size=5, freq="min") tbl = Table(data) assert tbl.size() == 5 assert tbl.schema() == { "index": "datetime", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"] == [ util.to_timestamp(datetime(2000, 1, 1, 0, 0)), util.to_timestamp(datetime(2000, 1, 1, 0, 1)), util.to_timestamp(datetime(2000, 1, 1, 0, 2)), util.to_timestamp(datetime(2000, 1, 1, 0, 3)), util.to_timestamp(datetime(2000, 1, 1, 0, 4)), ] def test_table_pandas_periodindex(self, util): df = util.make_period_dataframe(30) tbl = Table(df) assert tbl.size() == 30 assert tbl.schema() == { "index": "integer", "a": "float", "b": "float", "c": "float", "d": "float", } assert tbl.view().to_columns()["index"][:5] == [360, 361, 362, 363, 364] @mark.skip(reason="pyarrow does not support this") def test_table_pandas_period(self, util): df = pd.DataFrame( { "a": [ pd.Period("1Q2019"), pd.Period("2Q2019"), pd.Period("3Q2019"), pd.Period("4Q2019"), ] } ) tbl = Table(df) assert tbl.size() == 4 assert tbl.schema() == {"index": "integer", "a": "datetime"} assert tbl.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 1, 1)), util.to_timestamp(datetime(2019, 4, 1)), util.to_timestamp(datetime(2019, 7, 1)), util.to_timestamp(datetime(2019, 10, 1)), ] def test_table_pandas_from_schema_int(self): data = [None, 1, None, 2, None, 3, 4] df = pd.DataFrame({"a": data}) table = Table({"a": "integer"}) table.update(df) assert table.view().to_columns()["a"] == data def test_table_pandas_from_schema_bool(self): data = [True, False, True, False] df = pd.DataFrame({"a": data}) table = Table({"a": "boolean"}) table.update(df) assert table.view().to_columns()["a"] == data @mark.skip(reason="pyarrow does not support this") def test_table_pandas_from_schema_bool_str(self): data = ["True", "False", "True", "False"] df = pd.DataFrame({"a": data}) table = Table({"a": "boolean"}) table.update(df) assert table.view().to_columns()["a"] == [True, False, True, False] def test_table_pandas_from_schema_float(self): data = [None, 1.5, None, 2.5, None, 3.5, 4.5] df = pd.DataFrame({"a": data}) table = Table({"a": "float"}) table.update(df) assert table.view().to_columns()["a"] == data def test_table_pandas_from_schema_float_all_nan(self): data = [np.nan, np.nan, np.nan, np.nan] df = pd.DataFrame({"a": data}) table = Table({"a": "float"}) table.update(df) assert table.view().to_columns()["a"] == [None, None, None, None] def test_table_pandas_from_schema_float_to_int(self): data = [None, 1.5, None, 2.5, None, 3.5, 4.5] df = pd.DataFrame({"a": data}) table = Table({"a": "integer"}) table.update(df) # truncates decimal assert table.view().to_columns()["a"] == [None, 1, None, 2, None, 3, 4] def test_table_pandas_from_schema_int_to_float(self): data = [None, 1, None, 2, None, 3, 4] df = pd.DataFrame({"a": data}) table = Table({"a": "float"}) table.update(df) assert table.view().to_columns()["a"] == [None, 1.0, None, 2.0, None, 3.0, 4.0] def test_table_pandas_from_schema_date(self, util): data = [date(2019, 8, 15), None, date(2019, 8, 16)] df = pd.DataFrame({"a": data}) table = Table({"a": "date"}) table.update(df) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 8, 15)), None, util.to_timestamp(datetime(2019, 8, 16)), ] def test_table_pandas_from_schema_datetime(self, util): data = [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] df = pd.DataFrame({"a": pd.to_datetime(data, unit="ms")}) table = Table({"a": "datetime"}) table.update(df) assert table.view().to_columns()["a"] == data def test_table_pandas_from_schema_datetime_timestamp_s(self, util): data = [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), np.nan, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), np.nan, ] df = pd.DataFrame({"a": pd.to_datetime(data, unit="ms")}) table = Table({"a": "datetime"}) table.update(df) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] @mark.skip(reason="This is no longer relevant") def test_table_pandas_from_schema_datetime_timestamp_ms(self, util): data = [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)) * 1000, np.nan, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), *1000, np.nan, ] df = pd.DataFrame({"a": pd.to_datetime(data, unit="ms")}) table = Table({"a": "datetime"}) table.update(df) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] def test_table_pandas_from_schema_str(self): data = ["a", None, "b", None, "c"] df = pd.DataFrame({"a": data}) table = Table({"a": "string"}) table.update(df) assert table.view().to_columns()["a"] == data def test_table_pandas_none(self): data = [None, None, None] df = pd.DataFrame({"a": data}) table = Table(df) assert table.view().to_columns()["a"] == data def test_table_pandas_symmetric_table(self): # make sure that updates are symmetric to table creation df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5]}) t1 = Table(df) t2 = Table({"a": "integer", "b": "float"}) t2.update(df) assert t1.view().to_columns() == { "index": [0, 1, 2, 3], "a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5], } def test_table_pandas_symmetric_stacked_updates(self): # make sure that updates are symmetric to table creation df = pd.DataFrame({"a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5]}) t1 = Table(df) t1.update(df) t2 = Table({"a": "integer", "b": "float"}) t2.update(df) t2.update(df) assert t1.view().to_columns() == { "index": [0, 1, 2, 3, 0, 1, 2, 3], "a": [1, 2, 3, 4, 1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5, 1.5, 2.5, 3.5, 4.5], } def test_table_pandas_transitive(self): # serialized output -> table -> serialized output records = { "a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5], "c": [np.nan, np.nan, "abc", np.nan], "d": [None, True, None, False], "e": [ float("nan"), datetime(2019, 7, 11, 12, 30), float("nan"), datetime(2019, 7, 11, 12, 30), ], } df = pd.DataFrame(records) t1 = Table(df) out1 = arrow_bytes_to_pandas(t1.view(columns=["a", "b", "c", "d", "e"])) t2 = Table(out1) assert t1.schema() == t2.schema() out2 = t2.view().to_columns() assert t1.view().to_columns() == out2 # dtype=object should have correct inferred types def test_table_pandas_object_to_int(self): df = pd.DataFrame({"a": np.array([1, 2, None, 2, None, 3, 4], dtype=object)}) table = Table(df) assert table.schema() == {"index": "integer", "a": "integer"} assert table.view().to_columns()["a"] == [1, 2, None, 2, None, 3, 4] def test_table_pandas_object_to_float(self): df = pd.DataFrame({"a": np.array([None, 1, None, 2, None, 3, 4], dtype=object)}) table = Table(df) assert table.schema() == {"index": "integer", "a": "integer"} assert table.view().to_columns()["a"] == [None, 1.0, None, 2.0, None, 3.0, 4.0] def test_table_pandas_object_to_bool(self): df = pd.DataFrame( {"a": np.array([True, False, True, False, True, False], dtype=object)} ) table = Table(df) assert table.schema() == {"index": "integer", "a": "boolean"} assert table.view().to_columns()["a"] == [True, False, True, False, True, False] def test_table_pandas_object_to_date(self, util): df = pd.DataFrame( {"a": np.array([date(2019, 7, 11), date(2019, 7, 12), None], dtype=object)} ) table = Table(df) assert table.schema() == {"index": "integer", "a": "date"} assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11)), util.to_timestamp(datetime(2019, 7, 12)), None, ] def test_table_pandas_object_to_datetime(self, util): df = pd.DataFrame( { "a": np.array( [ datetime(2019, 7, 11, 1, 2, 3), datetime(2019, 7, 12, 1, 2, 3), None, ], dtype=object, ) } ) table = Table(df) assert table.schema() == {"index": "integer", "a": "datetime"} assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 1, 2, 3)), util.to_timestamp(datetime(2019, 7, 12, 1, 2, 3)), None, ] def test_table_pandas_object_to_str(self): df = pd.DataFrame({"a": np.array(["abc", "def", None, "ghi"], dtype=object)}) table = Table(df) assert table.schema() == {"index": "integer", "a": "string"} assert table.view().to_columns()["a"] == ["abc", "def", None, "ghi"] # Type matching def test_table_pandas_update_float_schema_with_int(self): df = pd.DataFrame({"a": [1.5, 2.5, 3.5, 4.5], "b": [1, 2, 3, 4]}) table = Table({"a": "float", "b": "float"}) table.update(df) assert table.view().to_columns() == { "a": [1.5, 2.5, 3.5, 4.5], "b": [1.0, 2.0, 3.0, 4.0], } def test_table_pandas_update_int32_with_int64(self): df = pd.DataFrame({"a": [1, 2, 3, 4]}) table = Table({"a": [1, 2, 3, 4]}) table.update(df) assert table.view().to_columns() == {"a": [1, 2, 3, 4, 1, 2, 3, 4]} def test_table_pandas_update_int64_with_float(self): df = pd.DataFrame({"a": [1.5, 2.5, 3.5, 4.5]}) table = Table(pd.DataFrame({"a": [1, 2, 3, 4]})) table.update(df) assert table.view().to_columns()["a"] == [1, 2, 3, 4, 1, 2, 3, 4] def test_table_pandas_update_date_schema_with_datetime(self, util): df = pd.DataFrame({"a": np.array([date(2019, 7, 11)])}) table = Table({"a": "date"}) table.update(df) assert table.schema() == {"a": "date"} assert table.view().to_columns() == { "a": [util.to_timestamp(datetime(2019, 7, 11))] } @mark.skip(reason="Not supported by pyarrow (?)") def test_table_pandas_update_datetime_schema_with_date(self, util): df = pd.DataFrame({"a": np.array([date(2019, 7, 11)])}) table = Table({"a": "datetime"}) table.update(df) assert table.schema() == {"a": "datetime"} assert table.view().to_columns() == { "a": [util.to_timestamp(datetime(2019, 7, 11, 0, 0))] } # Timestamps def test_table_pandas_timestamp_to_datetime(self, util): data = [ pd.Timestamp("2019-07-11 12:30:05"), None, pd.Timestamp("2019-07-11 13:30:05"), None, ] df = pd.DataFrame({"a": data}) table = Table(df) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] def test_table_pandas_timestamp_explicit_dtype(self, util): data = [ pd.Timestamp("2019-07-11 12:30:05"), None, pd.Timestamp("2019-07-11 13:30:05"), None, ] df = pd.DataFrame({"a": np.array(data, dtype="datetime64[ns]")}) table = Table(df) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] def test_table_pandas_update_datetime_with_timestamp(self, util): data = [ pd.Timestamp("2019-07-11 12:30:05"), None, pd.Timestamp("2019-07-11 13:30:05"), None, ] df = pd.DataFrame({"a": data}) df2 = pd.DataFrame({"a": data}) table = Table(df) table.update(df2) assert table.view().to_columns()["a"] == [ util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 12, 30, 5)), None, util.to_timestamp(datetime(2019, 7, 11, 13, 30, 5)), None, ] # NaN/NaT reading def test_table_pandas_nan(self): data = [np.nan, np.nan, np.nan, np.nan] df = pd.DataFrame({"a": data}) table = Table(df) assert table.view().to_columns()["a"] == [None, None, None, None] def test_table_pandas_int_nan(self): data = [np.nan, 1, np.nan, 2] df = pd.DataFrame({"a": data}) table = Table(df) assert table.view().to_columns()["a"] == [None, 1, None, 2] def test_table_pandas_float_nan(self): data = [np.nan, 1.5, np.nan, 2.5] df = pd.DataFrame({"a": data}) table = Table(df) assert table.view().to_columns()["a"] == [None, 1.5, None, 2.5] def test_table_read_nan_int_col(self): data = pd.DataFrame( {"str": ["abc", float("nan"), "def"], "int": [np.nan, 1, 2]} ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "int": "float", } # np.nan is float type - ints convert to floats when filled in assert tbl.size() == 3 assert tbl.view().to_columns() == { "index": [0, 1, 2], "str": ["abc", None, "def"], "int": [None, 1.0, 2.0], } def test_table_read_nan_float_col(self): data = pd.DataFrame( {"str": [float("nan"), "abc", float("nan")], "float": [np.nan, 1.5, 2.5]} ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "float": "float", } # can only promote to string or float assert tbl.size() == 3 assert tbl.view().to_columns() == { "index": [0, 1, 2], "str": [None, "abc", None], "float": [None, 1.5, 2.5], } def test_table_read_nan_bool_col(self): data = pd.DataFrame( {"bool": [np.nan, True, np.nan], "bool2": [False, np.nan, True]} ) tbl = Table(data) # if np.nan begins a column, it is inferred as float and then can be promoted. if np.nan is in the values (but not at start), the column type is whatever is inferred. assert tbl.schema() == { "index": "integer", "bool": "boolean", "bool2": "boolean", } assert tbl.size() == 3 # np.nans are always serialized as None assert tbl.view().to_columns() == { "index": [0, 1, 2], "bool": [None, True, None], "bool2": [False, None, True], } def test_table_read_nan_date_col(self): data = pd.DataFrame( {"str": ["abc", "def"], "date": [float("nan"), date(2019, 7, 11)]} ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "date": "date", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "date": [None, 1562803200000], } def test_table_read_nan_datetime_col(self, util): data = pd.DataFrame( { "str": ["abc", "def"], "datetime": [float("nan"), datetime(2019, 7, 11, 11, 0)], } ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "datetime": "datetime", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "datetime": [None, util.to_timestamp(datetime(2019, 7, 11, 11, 0))], } def test_table_read_nat_datetime_col(self, util): data = pd.DataFrame( {"str": ["abc", "def"], "datetime": ["NaT", datetime(2019, 7, 11, 11, 0)]} ) # datetime col is `datetime` in pandas<2, `object` in pandas>=2, so convert data.datetime = pd.to_datetime(data.datetime) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "datetime": "datetime", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "datetime": [None, util.to_timestamp(datetime(2019, 7, 11, 11, 0))], } def test_table_read_nan_datetime_as_date_col(self, util): data = pd.DataFrame( {"str": ["abc", "def"], "datetime": [float("nan"), datetime(2019, 7, 11)]} ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "datetime": "datetime", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "datetime": [None, util.to_timestamp(datetime(2019, 7, 11))], } def test_table_read_nan_datetime_no_seconds(self, util): data = pd.DataFrame( { "str": ["abc", "def"], "datetime": [float("nan"), datetime(2019, 7, 11, 11, 0)], } ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "datetime": "datetime", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "datetime": [None, util.to_timestamp(datetime(2019, 7, 11, 11, 0))], } def test_table_read_nan_datetime_milliseconds(self, util): data = pd.DataFrame( { "str": ["abc", "def"], "datetime": [np.nan, datetime(2019, 7, 11, 10, 30, 55)], } ) tbl = Table(data) assert tbl.schema() == { "index": "integer", "str": "string", "datetime": "datetime", } # can only promote to string or float assert tbl.size() == 2 assert tbl.view().to_columns() == { "index": [0, 1], "str": ["abc", "def"], "datetime": [None, util.to_timestamp(datetime(2019, 7, 11, 10, 30, 55))], } @mark.skip(reason="lol wtf") def test_table_pandas_correct_csv_nan_end(self): s = "string,\nint\n,1\n,2\nabc,3" csv = StringIO(s) data = pd.read_csv(csv) tbl = Table(data) assert tbl.schema() == {"index": "integer", "str": "string", "int": "integer"} assert tbl.size() == 3 assert tbl.view().to_columns() == { "index": [0, 1, 2], "str": [None, None, "abc"], "int": [1, 2, 3], } @mark.skip(reason="lol wtf") def test_table_pandas_correct_csv_nan_intermittent(self): s = "string,\nfloat\nabc,\n,2\nghi," csv = StringIO(s) data = pd.read_csv(csv) tbl = Table(data) assert tbl.schema() == {"index": "integer", "str": "string", "float": "float"} assert tbl.size() == 3 assert tbl.view().to_columns() == { "index": [0, 1, 2], "str": ["abc", None, "ghi"], "float": [None, 2, None], } @mark.skip(reason="pyarrow does not support series") def test_table_series(self): import pandas as pd data = pd.Series([1, 2, 3], name="a") tbl = Table(data) assert tbl.size() == 3 @mark.skip(reason="pyarrow does not support series") def test_table_indexed_series(self): import pandas as pd data = pd.Series([1, 2, 3], index=["a", "b", "c"], name="a") tbl = Table(data) assert tbl.schema() == {"index": "string", "a": "integer"} assert tbl.size() == 3 def test_groupbys(self, superstore): df_pivoted = superstore.set_index(["Country", "Region"]) table = Table(df_pivoted) columns = table.columns() assert table.size() == 100 assert "Country" in columns assert "Region" in columns def test_pivottable(self, superstore): pt = pd.pivot_table( superstore, values="Discount", index=["Country", "Region"], columns="Category", ) table = Table(pt) columns = table.columns() assert "Country" in columns assert "Region" in columns @mark.skip(reason="TODO move this to Python") def test_splitbys(self): arrays = [ np.array( [ "bar", "bar", "bar", "bar", "baz", "baz", "baz", "baz", "foo", "foo", "foo", "foo", "qux", "qux", "qux", "qux", ] ), np.array( [ "one", "one", "two", "two", "one", "one", "two", "two", "one", "one", "two", "two", "one", "one", "two", "two", ] ), np.array( [ "X", "Y", "X", "Y", "X", "Y", "X", "Y", "X", "Y", "X", "Y", "X", "Y", "X", "Y", ] ), ] tuples = list(zip(*arrays)) index = pd.MultiIndex.from_tuples(tuples, names=["first", "second", "third"]) df_both = pd.DataFrame( np.random.randn(3, 16), index=["A", "B", "C"], columns=index ) table = Table(df_both) assert table.size() == 48