chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,403 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
__version__ = "4.5.2"
|
||||
__all__ = [
|
||||
"_jupyter_labextension_paths",
|
||||
"Server",
|
||||
"Client",
|
||||
"Table",
|
||||
"View",
|
||||
"PerspectiveError",
|
||||
"ProxySession",
|
||||
"AsyncClient",
|
||||
"AsyncServer",
|
||||
"GenericSQLVirtualServerModel",
|
||||
"VirtualDataSlice",
|
||||
"VirtualServer",
|
||||
"num_cpus",
|
||||
"set_num_cpus",
|
||||
"system_info",
|
||||
]
|
||||
|
||||
__doc__ = """
|
||||
The Python language bindings for [Perspective](https://perspective-dev.github.io), a
|
||||
high performance data-visualization and analytics component for the web browser.
|
||||
|
||||
A simple example which loads an [Apache Arrow](https://arrow.apache.org/) and
|
||||
computes a "Group By" operation, returning a new Arrow.
|
||||
|
||||
```python
|
||||
from perspective import Server
|
||||
|
||||
client = Server().new_local_client()
|
||||
table = client.table(arrow_bytes_data)
|
||||
view = table.view(group_by = ["CounterParty", "Security"])
|
||||
arrow = view.to_arrow()
|
||||
```
|
||||
|
||||
Perspective for Python uses the exact same C++ data engine used by the
|
||||
[WebAssembly version](https://docs.rs/perspective-js/latest/perspective_js/) and
|
||||
[Rust version](https://docs.rs/crate/perspective/latest). The library consists
|
||||
of many of the same abstractions and API as in JavaScript, as well as
|
||||
Python-specific data loading support for [NumPy](https://numpy.org/),
|
||||
[Pandas](https://pandas.pydata.org/) (and
|
||||
[Apache Arrow](https://arrow.apache.org/), as in JavaScript).
|
||||
|
||||
Additionally, `perspective-python` provides a session manager suitable for
|
||||
integration into server systems such as
|
||||
[Tornado websockets](https://www.tornadoweb.org/en/stable/websocket.html),
|
||||
[AIOHTTP](https://docs.aiohttp.org/en/stable/web_quickstart.html#websockets), or
|
||||
[Starlette](https://www.starlette.io/websockets/)/[FastAPI](https://fastapi.tiangolo.com/advanced/websockets/),
|
||||
which allows fully _virtual_ Perspective tables to be interacted with by
|
||||
multiple `<perspective-viewer>` in a web browser. You can also interact with a
|
||||
Perspective table from python clients, and to that end client libraries are
|
||||
implemented for both Tornado and AIOHTTP.
|
||||
|
||||
As `<perspective-viewer>` will only consume the data necessary to render the
|
||||
current screen, this runtime mode allows _ludicrously-sized_ datasets with
|
||||
instant-load after they've been manifest on the server (at the expense of
|
||||
network latency on UI interaction).
|
||||
|
||||
The included `PerspectiveWidget` allows running such a viewer in
|
||||
[JupyterLab](https://jupyterlab.readthedocs.io/en/stable/) in either server or
|
||||
client (via WebAssembly) mode, and the included `PerspectiveTornadoHandler`
|
||||
makes it simple to extend a Tornado server with virtual Perspective support.
|
||||
|
||||
The `perspective` module exports several tools:
|
||||
|
||||
- `Server` the constructor for a new isntance of the Perspective data engine.
|
||||
- The `perspective.widget` module exports `PerspectiveWidget`, the JupyterLab
|
||||
widget for interactive visualization in a notebook cell.
|
||||
- The `perspective.handlers` modules exports web frameworks handlers that
|
||||
interface with a `perspective-client` in JavaScript.
|
||||
- `perspective.handlers.tornado.PerspectiveTornadoHandler` for
|
||||
[Tornado](https://www.tornadoweb.org/)
|
||||
- `perspective.handlers.starlette.PerspectiveStarletteHandler` for
|
||||
[Starlette](https://www.starlette.io/) and
|
||||
[FastAPI](https://fastapi.tiangolo.com)
|
||||
- `perspective.handlers.aiohttp.PerspectiveAIOHTTPHandler` for
|
||||
[AIOHTTP](https://docs.aiohttp.org),
|
||||
|
||||
This user's guide provides an overview of the most common ways to use
|
||||
Perspective in Python: the `Table` API, the JupyterLab widget, and the Tornado
|
||||
handler.
|
||||
|
||||
[More Examples](https://github.com/perspective-dev/perspective/tree/master/examples) are
|
||||
available on GitHub.
|
||||
|
||||
## Installation
|
||||
|
||||
`perspective-python` contains full bindings to the Perspective API, a JupyterLab
|
||||
widget, and a WebSocket handlers for several webserver libraries that allow you
|
||||
to host Perspective using server-side Python.
|
||||
|
||||
`perspective-python` can be installed from [PyPI](https://pypi.org) via `pip`:
|
||||
|
||||
```bash
|
||||
pip install perspective-python
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
A `Table` can be created from a dataset or a schema, the specifics of which are
|
||||
[discussed](#loading-data-with-table) in the JavaScript section of the user's
|
||||
guide. In Python, however, Perspective supports additional data types that are
|
||||
commonly used when processing data:
|
||||
|
||||
- `pandas.DataFrame`
|
||||
- `polars.DataFrame`
|
||||
- `bytes` (encoding an Apache Arrow)
|
||||
- `objects` (either extracting a repr or via reference)
|
||||
- `str` (encoding as a CSV)
|
||||
|
||||
A `Table` is created in a similar fashion to its JavaScript equivalent:
|
||||
|
||||
```python
|
||||
from datetime import date, datetime
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import perspective
|
||||
|
||||
data = pd.DataFrame({
|
||||
"int": np.arange(100),
|
||||
"float": [i * 1.5 for i in range(100)],
|
||||
"bool": [True for i in range(100)],
|
||||
"date": [date.today() for i in range(100)],
|
||||
"datetime": [datetime.now() for i in range(100)],
|
||||
"string": [str(i) for i in range(100)]
|
||||
})
|
||||
|
||||
table = perspective.table(data, index="float")
|
||||
```
|
||||
|
||||
Likewise, a `View` can be created via the `view()` method:
|
||||
|
||||
```python
|
||||
view = table.view(group_by=["float"], filter=[["bool", "==", True]])
|
||||
column_data = view.to_columns()
|
||||
row_data = view.to_json()
|
||||
```
|
||||
|
||||
#### Pandas and Polars Support
|
||||
|
||||
Perspective's `Table` can be constructed from `pandas.DataFrame` and
|
||||
`polars.DataFrame` objects. Internally, this just uses
|
||||
[`pyarrow::from_pandas`](https://arrow.apache.org/docs/python/pandas.html),
|
||||
which dictates behavior of this feature including type support.
|
||||
|
||||
If the dataframe does not have an index set, an integer-typed column named
|
||||
`"index"` is created. If you want to preserve the indexing behavior of the
|
||||
dataframe passed into Perspective, simply create the `Table` with
|
||||
`index="index"` as a keyword argument. This tells Perspective to once again
|
||||
treat the index as a primary key:
|
||||
|
||||
```python
|
||||
data.set_index("datetime")
|
||||
table = perspective.table(data, index="index")
|
||||
```
|
||||
|
||||
#### Time Zone Handling
|
||||
|
||||
When parsing `"datetime"` strings, times are assumed _local time_ unless an
|
||||
explicit timezone offset is parsed. All `"datetime"` columns (regardless of
|
||||
input time zone) are _output_ to the user as `datetime.datetime` objects in
|
||||
_local time_ according to the Python runtime.
|
||||
|
||||
This behavior is consistent with Perspective's behavior in JavaScript. For more
|
||||
details, see this in-depth
|
||||
[explanation](https://github.com/perspective-dev/perspective/pull/867) of
|
||||
`perspective-python` semantics around time zone handling.
|
||||
|
||||
#### Callbacks and Events
|
||||
|
||||
`perspective.Table` allows for `on_update` and `on_delete` callbacks to be
|
||||
set—simply call `on_update` or `on_delete` with a reference to a function or a
|
||||
lambda without any parameters:
|
||||
|
||||
```python
|
||||
def update_callback():
|
||||
print("Updated!")
|
||||
|
||||
# set the update callback
|
||||
on_update_id = view.on_update(update_callback)
|
||||
|
||||
|
||||
def delete_callback():
|
||||
print("Deleted!")
|
||||
|
||||
# set the delete callback
|
||||
on_delete_id = view.on_delete(delete_callback)
|
||||
|
||||
# set a lambda as a callback
|
||||
view.on_delete(lambda: print("Deleted x2!"))
|
||||
```
|
||||
|
||||
If the callback is a named reference to a function, it can be removed with
|
||||
`remove_update` or `remove_delete`:
|
||||
|
||||
```python
|
||||
view.remove_update(on_update_id)
|
||||
view.remove_delete(on_delete_id)
|
||||
```
|
||||
|
||||
Callbacks defined with a lambda function cannot be removed, as lambda functions
|
||||
have no identifier.
|
||||
|
||||
### Hosting `Table` and `View` instances
|
||||
|
||||
`Server` "hosts" all `perspective.Table` and `perspective.View` instances
|
||||
created by its connected `Client`s. Hosted tables/views can have their methods
|
||||
called from other sources than the Python server, i.e. by a `perspective-viewer`
|
||||
running in a JavaScript client over the network, interfacing with
|
||||
`perspective-python` through the websocket API.
|
||||
|
||||
The server has full control of all hosted `Table` and `View` instances, and can
|
||||
call any public API method on hosted instances. This makes it extremely easy to
|
||||
stream data to a hosted `Table` using `.update()`:
|
||||
|
||||
```python
|
||||
server = perspective.Server()
|
||||
client = server.new_local_client()
|
||||
table = client.table(data, name="data_source")
|
||||
|
||||
for i in range(10):
|
||||
# updates continue to propagate automatically
|
||||
table.update(new_data)
|
||||
```
|
||||
|
||||
The `name` provided is important, as it enables Perspective in JavaScript to
|
||||
look up a `Table` and get a handle to it over the network. Otherwise, `name`
|
||||
will be assigned randomlu and the `Client` must look this up with
|
||||
`CLient.get_hosted_table_names()`
|
||||
|
||||
### Client/Server Replicated Mode
|
||||
|
||||
Using Tornado and
|
||||
[`PerspectiveTornadoHandler`](python.md#perspectivetornadohandler), as well as
|
||||
`Perspective`'s JavaScript library, we can set up "distributed" Perspective
|
||||
instances that allows multiple browser `perspective-viewer` clients to read from
|
||||
a common `perspective-python` server, as in the
|
||||
[Tornado Example Project](https://github.com/perspective-dev/perspective/tree/master/examples/python-tornado).
|
||||
|
||||
This architecture works by maintaining two `Tables`—one on the server, and one
|
||||
on the client that mirrors the server's `Table` automatically using `on_update`.
|
||||
All updates to the table on the server are automatically applied to each client,
|
||||
which makes this architecture a natural fit for streaming dashboards and other
|
||||
distributed use-cases. In conjunction with [multithreading](#multi-threading),
|
||||
distributed Perspective offers consistently high performance over large numbers
|
||||
of clients and large datasets.
|
||||
|
||||
_*server.py*_
|
||||
|
||||
```python
|
||||
from perspective import Server
|
||||
from perspective.hadnlers.tornado import PerspectiveTornadoHandler
|
||||
|
||||
# Create an instance of Server, and host a Table
|
||||
SERVER = Server()
|
||||
CLIENT = SERVER.new_local_client()
|
||||
|
||||
# The Table is exposed at `localhost:8888/websocket` with the name `data_source`
|
||||
client.table(data, name = "data_source")
|
||||
|
||||
app = tornado.web.Application([
|
||||
# create a websocket endpoint that the client JavaScript can access
|
||||
(r"/websocket", PerspectiveTornadoHandler, {"perspective_server": SERVER})
|
||||
])
|
||||
|
||||
# Start the Tornado server
|
||||
app.listen(8888)
|
||||
loop = tornado.ioloop.IOLoop.current()
|
||||
loop.start()
|
||||
```
|
||||
|
||||
Instead of calling `load(server_table)`, create a `View` using `server_table`
|
||||
and pass that into `viewer.load()`. This will automatically register an
|
||||
`on_update` callback that synchronizes state between the server and the client.
|
||||
|
||||
_*index.html*_
|
||||
|
||||
```html
|
||||
<perspective-viewer id="viewer" editable></perspective-viewer>
|
||||
|
||||
<script type="module">
|
||||
// Create a client that expects a Perspective server
|
||||
// to accept connections at the specified URL.
|
||||
const websocket = await perspective.websocket(
|
||||
"ws://localhost:8888/websocket"
|
||||
);
|
||||
|
||||
// Get a handle to the Table on the server
|
||||
const server_table = await websocket.open_table("data_source_one");
|
||||
|
||||
// Create a new view
|
||||
const server_view = await table.view();
|
||||
|
||||
// Create a Table on the client using `perspective.worker()`
|
||||
const worker = await perspective.worker();
|
||||
const client_table = await worker.table(view);
|
||||
|
||||
// Load the client table in the `<perspective-viewer>`.
|
||||
document.getElementById("viewer").load(client_table);
|
||||
</script>
|
||||
```
|
||||
|
||||
For a more complex example that offers distributed editing of the server
|
||||
dataset, see
|
||||
[client_server_editing.html](https://github.com/perspective-dev/perspective/blob/master/examples/python-tornado/client_server_editing.html).
|
||||
|
||||
We also provide examples for Starlette/FastAPI and AIOHTTP:
|
||||
|
||||
- [Starlette Example Project](https://github.com/perspective-dev/perspective/tree/master/examples/python-starlette).
|
||||
- [AIOHTTP Example Project](https://github.com/perspective-dev/perspective/tree/master/examples/python-aiohttp).
|
||||
|
||||
### Server-only Mode
|
||||
|
||||
The server setup is identical to [Distributed Mode](#distributed-mode) above,
|
||||
but instead of creating a view, the client calls `load(server_table)`: In
|
||||
Python, use `Server` and `PerspectiveTornadoHandler` to create a websocket
|
||||
server that exposes a `Table`. In this example, `table` is a proxy for the
|
||||
`Table` we created on the server. All API methods are available on _proxies_,
|
||||
the.g.us calling `view()`, `schema()`, `update()` on `table` will pass those
|
||||
operations to the Python `Table`, execute the commands, and return the result
|
||||
back to Javascript.
|
||||
|
||||
```html
|
||||
<perspective-viewer id="viewer" editable></perspective-viewer>
|
||||
```
|
||||
|
||||
```javascript
|
||||
const websocket = perspective.websocket("ws://localhost:8888/websocket");
|
||||
const table = websocket.open_table("data_source");
|
||||
document.getElementById("viewer").load(table);
|
||||
```
|
||||
|
||||
"""
|
||||
|
||||
|
||||
import functools
|
||||
|
||||
from .perspective import (
|
||||
Client,
|
||||
PerspectiveError,
|
||||
ProxySession,
|
||||
Server,
|
||||
AsyncServer,
|
||||
AsyncClient,
|
||||
VirtualServer,
|
||||
VirtualDataSlice,
|
||||
GenericSQLVirtualServerModel,
|
||||
# NOTE: these are classes without constructors,
|
||||
# so we import them just for type hinting
|
||||
Table, # noqa: F401
|
||||
View, # noqa: F401
|
||||
num_cpus,
|
||||
set_num_cpus,
|
||||
)
|
||||
|
||||
|
||||
GLOBAL_SERVER = Server()
|
||||
GLOBAL_CLIENT = GLOBAL_SERVER.new_local_client()
|
||||
|
||||
|
||||
@functools.wraps(Client.table)
|
||||
def table(*args, **kwargs):
|
||||
return GLOBAL_CLIENT.table(*args, **kwargs)
|
||||
|
||||
|
||||
@functools.wraps(Client.open_table)
|
||||
def open_table(*args, **kwargs):
|
||||
return GLOBAL_CLIENT.table(*args, **kwargs)
|
||||
|
||||
|
||||
@functools.wraps(Client.get_hosted_table_names)
|
||||
def get_hosted_table_names(*args, **kwargs):
|
||||
return GLOBAL_CLIENT.get_hosted_table_names(*args, **kwargs)
|
||||
|
||||
|
||||
@functools.wraps(Client.join)
|
||||
def join(*args, **kwargs):
|
||||
return GLOBAL_CLIENT.join(*args, **kwargs)
|
||||
|
||||
|
||||
@functools.wraps(Client.system_info)
|
||||
def system_info(*args, **kwargs):
|
||||
return GLOBAL_CLIENT.system_info(*args, **kwargs)
|
||||
|
||||
|
||||
def _jupyter_labextension_paths():
|
||||
"""
|
||||
Read by `jupyter labextension develop`
|
||||
@private
|
||||
"""
|
||||
return [{"src": "labextension", "dest": "@perspective-dev/jupyterlab"}]
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"load_extensions": {
|
||||
"@perspective-dev/jupyterlab/extension": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,69 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from aiohttp import web, WSMsgType
|
||||
import perspective
|
||||
import asyncio
|
||||
|
||||
|
||||
class PerspectiveAIOHTTPHandler(object):
|
||||
"""`PerspectiveAIOHTTPHandler` is a drop-in implementation of Perspective.
|
||||
|
||||
Use it inside AIOHTTP routing to create a server-side Perspective that is
|
||||
ready to receive websocket messages from the front-end `perspective-viewer`.
|
||||
|
||||
The Perspective client and server will automatically keep the Websocket
|
||||
alive without timing out.
|
||||
|
||||
# Security
|
||||
|
||||
`PerspectiveAIOHTTPHandler` is a reference integration with no
|
||||
authentication, authorization, origin enforcement, or rate limiting,
|
||||
and is not safe to expose to untrusted networks — see
|
||||
[`SECURITY.md`](https://github.com/perspective-dev/perspective/blob/master/SECURITY.md)
|
||||
for the full threat model.
|
||||
|
||||
# Examples
|
||||
|
||||
>>> server = Server()
|
||||
>>> async def websocket_handler(request):
|
||||
... handler = PerspectiveAIOHTTPHandler(perspective_server=server, request=request)
|
||||
... await handler.run()
|
||||
|
||||
>>> app = web.Application()
|
||||
>>> app.router.add_get("/websocket", websocket_handler)
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self.server = kwargs.pop("perspective_server", perspective.GLOBAL_SERVER)
|
||||
self._request = kwargs.pop("request")
|
||||
self._executor = kwargs.pop("executor", None)
|
||||
self._loop = kwargs.pop("loop", asyncio.get_event_loop())
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def run(self) -> web.WebSocketResponse:
|
||||
def inner(msg):
|
||||
self._loop.create_task(self._ws.send_bytes(msg))
|
||||
|
||||
self.session = self.server.new_session(inner)
|
||||
try:
|
||||
self._ws = web.WebSocketResponse()
|
||||
await self._ws.prepare(self._request)
|
||||
async for msg in self._ws:
|
||||
if msg.type == WSMsgType.BINARY:
|
||||
if self._executor is not None:
|
||||
self._executor.submit(self.session.handle_request, msg.data)
|
||||
else:
|
||||
self.session.handle_request(msg.data)
|
||||
finally:
|
||||
self.session.close()
|
||||
return self._ws
|
||||
@@ -0,0 +1,63 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import asyncio
|
||||
import perspective
|
||||
|
||||
|
||||
class PerspectiveStarletteHandler(object):
|
||||
"""`PerspectiveStarletteHandler` is a drop-in implementation of Perspective.
|
||||
|
||||
# Security
|
||||
|
||||
`PerspectiveStarletteHandler` is a reference integration with no
|
||||
authentication, authorization, origin enforcement, or rate limiting,
|
||||
and is not safe to expose to untrusted networks — see
|
||||
[`SECURITY.md`](https://github.com/perspective-dev/perspective/blob/master/SECURITY.md)
|
||||
for the full threat model.
|
||||
|
||||
# Examples
|
||||
|
||||
>>> server = Server()
|
||||
>>> client = server.client()
|
||||
>>> client.table(pd.read_csv("superstore.csv"), name="data_source_one")
|
||||
>>> app = FastAPI()
|
||||
>>> async def endpoint(websocket: Websocket):
|
||||
... handler = PerspectiveStarletteHandler(server, websocket)
|
||||
... await handler.run()
|
||||
... app.add_api_websocket_route('/websocket', endpoint)
|
||||
"""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
self._server = kwargs.pop("perspective_server", perspective.GLOBAL_SERVER)
|
||||
self._websocket = kwargs.pop("websocket")
|
||||
self._executor = kwargs.pop("executor", None)
|
||||
self._loop = kwargs.pop("loop", asyncio.get_event_loop())
|
||||
super().__init__(**kwargs)
|
||||
|
||||
async def run(self) -> None:
|
||||
def inner(msg):
|
||||
self._loop.create_task(self._websocket.send_bytes(msg))
|
||||
|
||||
self.session = self._server.new_session(inner)
|
||||
|
||||
try:
|
||||
await self._websocket.accept()
|
||||
while True:
|
||||
message = await self._websocket.receive()
|
||||
self._websocket._raise_on_disconnect(message)
|
||||
if self._executor is not None:
|
||||
self._executor.submit(self.session.handle_request, message["bytes"])
|
||||
else:
|
||||
self.session.handle_request(message["bytes"])
|
||||
finally:
|
||||
self.session.close()
|
||||
@@ -0,0 +1,192 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from tornado.websocket import WebSocketHandler, WebSocketClosedError
|
||||
from tornado.ioloop import IOLoop
|
||||
import perspective
|
||||
|
||||
__doc__ = """
|
||||
Perspective ships with a pre-built Tornado handler that makes integration with
|
||||
`tornado.websockets` extremely easy. This allows you to run an instance of
|
||||
`Perspective` on a server using Python, open a websocket to a `Table`, and
|
||||
access the `Table` in JavaScript and through `<perspective-viewer>`. All
|
||||
instructions sent to the `Table` are processed in Python, which executes the
|
||||
commands, and returns its output through the websocket back to Javascript.
|
||||
|
||||
### Python setup
|
||||
|
||||
To use the handler, we need to first have a `Server`, a `Client` and an instance
|
||||
of a `Table`:
|
||||
|
||||
```python
|
||||
SERVER = Server()
|
||||
CLIENT = SERVER.new_local_client()
|
||||
```
|
||||
|
||||
Once the server has been created, create a `Table` instance with a name. The
|
||||
name that you host the table under is important — it acts as a unique accessor
|
||||
on the JavaScript side, which will look for a Table hosted at the websocket with
|
||||
the name you specify.
|
||||
|
||||
```python
|
||||
TABLE = client.table(data, name="data_source_one")
|
||||
```
|
||||
|
||||
After the server and table setup is complete, create a websocket endpoint and
|
||||
provide it a reference to `PerspectiveTornadoHandler`. You must provide the
|
||||
configuration object in the route tuple, and it must contain
|
||||
`"perspective_server"`, which is a reference to the `Server` you just created.
|
||||
|
||||
```python
|
||||
from perspective.handlers.tornado import PerspectiveTornadoHandler
|
||||
|
||||
app = tornado.web.Application([
|
||||
|
||||
# ... other handlers ...
|
||||
|
||||
# Create a websocket endpoint that the client JavaScript can access
|
||||
(r"/websocket", PerspectiveTornadoHandler, {"perspective_server": SERVER, "check_origin": True})
|
||||
])
|
||||
```
|
||||
|
||||
Optionally, the configuration object can also include `check_origin`, a boolean
|
||||
that determines whether the websocket accepts requests from origins other than
|
||||
where the server is hosted. See
|
||||
[Tornado docs](https://www.tornadoweb.org/en/stable/websocket.html#tornado.websocket.WebSocketHandler.check_origin)
|
||||
for more details.
|
||||
|
||||
### JavaScript setup
|
||||
|
||||
Once the server is up and running, you can access the Table you just hosted
|
||||
using `perspective.websocket` and `open_table()`. First, create a client that
|
||||
expects a Perspective server to accept connections at the specified URL:
|
||||
|
||||
```javascript
|
||||
const websocket = await perspective.websocket("ws://localhost:8888/websocket");
|
||||
```
|
||||
|
||||
Next open the `Table` we created on the server by name:
|
||||
|
||||
```javascript
|
||||
const table = await websocket.open_table("data_source_one");
|
||||
```
|
||||
|
||||
`table` is a proxy for the `Table` we created on the server. All operations that
|
||||
are possible through the JavaScript API are possible on the Python API as well,
|
||||
thus calling `view()`, `schema()`, `update()` etc. on `const table` will pass
|
||||
those operations to the Python `Table`, execute the commands, and return the
|
||||
result back to JavaScript. Similarly, providing this `table` to a
|
||||
`<perspective-viewer>` instance will allow virtual rendering:
|
||||
|
||||
```javascript
|
||||
await viewer.load(table);
|
||||
```
|
||||
|
||||
`perspective.websocket` expects a Websocket URL where it will send instructions.
|
||||
When `open_table` is called, the name to a hosted Table is passed through, and a
|
||||
request is sent through the socket to fetch the Table. No actual `Table`
|
||||
instance is passed inbetween the runtimes; all instructions are proxied through
|
||||
websockets.
|
||||
|
||||
This provides for great flexibility — while `Perspective.js` is full of
|
||||
features, browser WebAssembly runtimes currently have some performance
|
||||
restrictions on memory and CPU feature utilization, and the architecture in
|
||||
general suffers when the dataset itself is too large to download to the client
|
||||
in full.
|
||||
|
||||
The Python runtime does not suffer from memory limitations, utilizes Apache
|
||||
Arrow internal threadpools for threading and parallel processing, and generates
|
||||
architecture optimized code, which currently makes it more suitable as a
|
||||
server-side runtime than `node.js`.
|
||||
"""
|
||||
|
||||
|
||||
class PerspectiveTornadoHandler(WebSocketHandler):
|
||||
"""`PerspectiveTornadoHandler` is a `perspective.Server` API as a `tornado`
|
||||
websocket handler.
|
||||
|
||||
Use it inside `tornado` routing to create a `perspective.Server` that can
|
||||
connect to a JavaScript (Wasm) `Client`, providing a virtual interface to
|
||||
the `Server`'s resources for e.g. `<perspective-viewer>`.
|
||||
|
||||
You may need to increase the `websocket_max_message_size` kwarg
|
||||
to the `tornado.web.Application` constructor, as well as provide the
|
||||
`max_buffer_size` optional arg, for large datasets.
|
||||
|
||||
# Security
|
||||
|
||||
`PerspectiveTornadoHandler` is a reference integration with no
|
||||
authentication, authorization, origin enforcement, or rate limiting,
|
||||
and is not safe to expose to untrusted networks — see
|
||||
[`SECURITY.md`](https://github.com/perspective-dev/perspective/blob/master/SECURITY.md)
|
||||
for the full threat model.
|
||||
|
||||
# Arguments
|
||||
|
||||
- `loop`: An optional `IOLoop` instance to use for scheduling IO calls,
|
||||
defaults to `IOLoop.current()`.
|
||||
- `executor`: An optional executor for scheduling `perspective.Server`
|
||||
message processing calls from websocket `Client`s.
|
||||
|
||||
# Examples
|
||||
|
||||
>>> server = psp.Server()
|
||||
>>> client = server.new_local_client()
|
||||
>>> client.table(pd.read_csv("superstore.csv"), name="data_source_one")
|
||||
>>> app = tornado.web.Application([
|
||||
... (r"/", MainHandler),
|
||||
... (r"/websocket", PerspectiveTornadoHandler, {
|
||||
... "perspective_server": server,
|
||||
... })
|
||||
... ])
|
||||
"""
|
||||
|
||||
def check_origin(self, origin):
|
||||
return True
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
perspective_server=perspective.GLOBAL_SERVER,
|
||||
loop=None,
|
||||
executor=None,
|
||||
max_buffer_size=None,
|
||||
):
|
||||
self.server = perspective_server
|
||||
self.loop = loop or IOLoop.current()
|
||||
self.executor = executor
|
||||
if max_buffer_size is not None:
|
||||
self.request.connection.stream.max_buffer_size = max_buffer_size
|
||||
|
||||
def open(self):
|
||||
def write(msg):
|
||||
try:
|
||||
self.write_message(msg, binary=True)
|
||||
except WebSocketClosedError:
|
||||
self.close()
|
||||
|
||||
def send_response(msg):
|
||||
self.loop.add_callback(write, msg)
|
||||
|
||||
self.session = self.server.new_session(send_response)
|
||||
|
||||
def on_close(self) -> None:
|
||||
self.session.close()
|
||||
del self.session
|
||||
|
||||
def on_message(self, msg: bytes):
|
||||
if not isinstance(msg, bytes):
|
||||
return
|
||||
|
||||
if self.executor is None:
|
||||
self.session.handle_request(msg)
|
||||
else:
|
||||
self.executor.submit(self.session.handle_request, msg)
|
||||
@@ -0,0 +1,35 @@
|
||||
<script type="module" src="$psp_cdn_perspective"></script>
|
||||
<script type="module" src="$psp_cdn_perspective_viewer"></script>
|
||||
<script type="module" src="$psp_cdn_perspective_viewer_datagrid"></script>
|
||||
<script type="module" src="$psp_cdn_perspective_viewer_charts"></script>
|
||||
<link rel="stylesheet" crossorigin="anonymous" href="$psp_cdn_perspective_viewer_themes" />
|
||||
|
||||
<div class="perspective-envelope" id="perspective-envelope-$viewer_id">
|
||||
<script type="application/vnd.apache.arrow.file">
|
||||
$b64_data
|
||||
</script>
|
||||
<perspective-viewer style="height: 690px;"></perspective-viewer>
|
||||
<script type="module">
|
||||
// from MDN
|
||||
function base64ToBytes(base64) {
|
||||
const binString = atob(base64);
|
||||
return Uint8Array.from(binString, (m) => m.codePointAt(0));
|
||||
}
|
||||
|
||||
import * as perspective from "$psp_cdn_perspective";
|
||||
const viewerId = $viewer_id;
|
||||
const currentScript = document.scripts[document.scripts.length - 1];
|
||||
const envelope = document.getElementById(`perspective-envelope-$${viewerId}`);
|
||||
const dataScript = envelope.querySelector('script[type="application/vnd.apache.arrow.file"]');;
|
||||
if (!dataScript)
|
||||
throw new Error('data script missing for viewer', viewerId);
|
||||
const data = base64ToBytes(dataScript.textContent);
|
||||
const viewerAttrs = $viewer_attrs;
|
||||
|
||||
// Create a new worker, then a new table promise on that worker.
|
||||
const table = await perspective.worker().table(data.buffer);
|
||||
const viewer = envelope.querySelector('perspective-viewer');
|
||||
viewer.load(table);
|
||||
viewer.restore(viewerAttrs);
|
||||
</script>
|
||||
</div>
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,83 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from perspective import (
|
||||
AsyncClient,
|
||||
AsyncServer,
|
||||
PerspectiveError,
|
||||
)
|
||||
|
||||
|
||||
import pytest
|
||||
from unittest.mock import Mock
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
return AsyncServer()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(server):
|
||||
async def send_request(msg):
|
||||
await sess.handle_request(msg)
|
||||
|
||||
async def send_response(msg):
|
||||
await client.handle_response(msg)
|
||||
|
||||
sess = server.new_session(send_response)
|
||||
client = AsyncClient(send_request)
|
||||
return client
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_request_callback_awaited(client):
|
||||
table_names = await client.get_hosted_table_names()
|
||||
assert table_names == []
|
||||
|
||||
|
||||
# This test also exists in the pyodide-tests/ pytest suite, with the same name.
|
||||
# Unfortunately code sharing between the two test suites is currently not easy.
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_client_kitchen_sink(client):
|
||||
"""run various things on the async client"""
|
||||
table = await client.table({"a": [0]}, name="my-cool-data", limit=100)
|
||||
view = await table.view()
|
||||
# synchronous methods
|
||||
assert table.get_index() is None
|
||||
assert table.get_name() == "my-cool-data"
|
||||
limit = table.get_limit()
|
||||
assert limit == 100
|
||||
|
||||
# view update callbacks
|
||||
view_updated = Mock()
|
||||
await view.on_update(view_updated)
|
||||
for i in range(1, limit):
|
||||
await table.update([{"a": i}])
|
||||
await table.size() # force update to process
|
||||
assert (await table.size()) == limit
|
||||
assert view_updated.call_count == limit - 1
|
||||
rex = await view.to_records()
|
||||
assert rex == [{"a": i} for i in range(limit)]
|
||||
|
||||
# view/table delete callbacks
|
||||
view_deleted = Mock()
|
||||
table_deleted = Mock()
|
||||
await table.on_delete(table_deleted)
|
||||
await view.on_delete(view_deleted)
|
||||
with pytest.raises(PerspectiveError) as excinfo:
|
||||
await table.delete()
|
||||
assert excinfo.match(r"Cannot delete table with views")
|
||||
await view.delete()
|
||||
view_deleted.assert_called_once()
|
||||
await table.delete()
|
||||
table_deleted.assert_called_once()
|
||||
@@ -0,0 +1,124 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import websocket
|
||||
|
||||
import tornado.websocket
|
||||
import tornado.web
|
||||
import tornado.ioloop
|
||||
|
||||
import perspective
|
||||
import perspective.handlers.tornado
|
||||
|
||||
PORT = 8082
|
||||
|
||||
|
||||
def test_big_multi_thing(superstore):
|
||||
async def init_table(client):
|
||||
global SERVER_DATA
|
||||
global SERVER_TABLE
|
||||
|
||||
SERVER_DATA = "x,y\n1,2\n3,4"
|
||||
# with open(file_path, mode="rb") as file:
|
||||
SERVER_TABLE = client.table(SERVER_DATA, name="superstore")
|
||||
|
||||
global ws
|
||||
ws = websocket.WebSocketApp(
|
||||
"ws://localhost:{}/websocket".format(PORT),
|
||||
on_open=on_open,
|
||||
on_message=on_message,
|
||||
# on_error=on_error,
|
||||
# on_close=on_close,
|
||||
)
|
||||
|
||||
global ws_thread
|
||||
ws_thread = threading.Thread(target=ws.run_forever)
|
||||
ws_thread.start()
|
||||
|
||||
def server_thread():
|
||||
def make_app(perspective_server):
|
||||
return tornado.web.Application(
|
||||
[
|
||||
(
|
||||
r"/websocket",
|
||||
perspective.handlers.tornado.PerspectiveTornadoHandler,
|
||||
{"perspective_server": perspective_server},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
perspective_server = perspective.Server()
|
||||
app = make_app(perspective_server)
|
||||
global server
|
||||
server = app.listen(PORT, "0.0.0.0")
|
||||
|
||||
global server_loop
|
||||
server_loop = tornado.ioloop.IOLoop.current()
|
||||
client = perspective_server.new_local_client()
|
||||
server_loop.call_later(0, init_table, client)
|
||||
server_loop.start()
|
||||
|
||||
server_thread = threading.Thread(target=server_thread)
|
||||
server_thread.start()
|
||||
|
||||
client_loop = asyncio.new_event_loop()
|
||||
client_loop.set_debug(True)
|
||||
client_thread = threading.Thread(target=client_loop.run_forever)
|
||||
client_thread.start()
|
||||
|
||||
async def send_request(msg):
|
||||
global ws
|
||||
ws.send(msg, websocket.ABNF.OPCODE_BINARY)
|
||||
|
||||
def on_message(ws, message):
|
||||
async def poke_client():
|
||||
await client.handle_response(message)
|
||||
|
||||
asyncio.run_coroutine_threadsafe(poke_client(), client_loop)
|
||||
|
||||
# def on_error(ws, error):
|
||||
# print(f"Error!: {error}")
|
||||
|
||||
# def on_close(ws, close_status_code, close_msg):
|
||||
# print("Connection closed")
|
||||
|
||||
def on_open(ws):
|
||||
global client
|
||||
client = perspective.AsyncClient(send_request)
|
||||
asyncio.run_coroutine_threadsafe(test(client), client_loop)
|
||||
|
||||
global count
|
||||
count = 0
|
||||
|
||||
def update(x):
|
||||
global count
|
||||
count += 1
|
||||
|
||||
async def test(client):
|
||||
table = await client.open_table("superstore")
|
||||
view = await table.view()
|
||||
await view.on_update(update)
|
||||
SERVER_TABLE.update(SERVER_DATA)
|
||||
assert await table.size() == 4
|
||||
assert count == 1
|
||||
await server.close_all_connections()
|
||||
client_loop.stop()
|
||||
|
||||
client_thread.join()
|
||||
client_loop.close()
|
||||
ws.close()
|
||||
ws_thread.join()
|
||||
server_loop.add_callback(server_loop.stop)
|
||||
server_thread.join()
|
||||
server_loop.close()
|
||||
@@ -0,0 +1,276 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from datetime import datetime, date, timezone
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pyarrow as pa
|
||||
from pytest import fixture
|
||||
from random import random, randint, choice
|
||||
from faker import Faker
|
||||
|
||||
|
||||
# Our tests construct naive datetimes everywhere
|
||||
# so setting it here is an easy way to fix it globally.
|
||||
import os
|
||||
|
||||
# Perspective used to support datetime.date and datetime.datetime
|
||||
# as Table() constructor arguments, but now we forward the parameters
|
||||
# directly to JSON.loads. So to make sure the tests dont need to be
|
||||
# so utterly transmogrified, we have this little hack :)
|
||||
import json
|
||||
|
||||
|
||||
os.environ["TZ"] = "UTC"
|
||||
|
||||
|
||||
def new_encoder(self, obj):
|
||||
if isinstance(obj, datetime):
|
||||
return str(obj)
|
||||
elif isinstance(obj, date):
|
||||
return str(obj)
|
||||
else:
|
||||
return old(self, obj)
|
||||
|
||||
|
||||
old = json.JSONEncoder.default
|
||||
json.JSONEncoder.default = new_encoder
|
||||
|
||||
fake = Faker()
|
||||
|
||||
|
||||
def _make_date_time_index(size, time_unit):
|
||||
return pd.date_range("2000-01-01", periods=size, freq=time_unit)
|
||||
|
||||
|
||||
def _make_period_index(size, time_unit):
|
||||
return pd.period_range(start="2000", periods=size, freq=time_unit)
|
||||
|
||||
|
||||
def _make_dataframe(index, size=10):
|
||||
"""Create a new random dataframe of `size` and with a DateTimeIndex of
|
||||
frequency `time_unit`.
|
||||
"""
|
||||
return pd.DataFrame(
|
||||
index=index,
|
||||
data={
|
||||
"a": np.random.rand(size),
|
||||
"b": np.random.rand(size),
|
||||
"c": np.random.rand(size),
|
||||
"d": np.random.rand(size),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class Util:
|
||||
@staticmethod
|
||||
def make_arrow(names, data, types=None, legacy=False):
|
||||
"""Create an arrow binary that can be loaded and manipulated from memory.
|
||||
|
||||
Args:
|
||||
names (list): a list of str column names
|
||||
data (list): a list of lists containing data for each column
|
||||
types (list): an optional list of `pyarrow.type` function references.
|
||||
Types will be inferred if not provided.
|
||||
legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
|
||||
|
||||
Returns:
|
||||
bytes : a bytes object containing the arrow-serialized output.
|
||||
"""
|
||||
stream = pa.BufferOutputStream()
|
||||
arrays = []
|
||||
|
||||
for idx, column in enumerate(data):
|
||||
# only apply types if array is present
|
||||
kwargs = {}
|
||||
if types:
|
||||
kwargs["type"] = types[idx]
|
||||
arrays.append(pa.array(column, **kwargs))
|
||||
|
||||
batch = pa.RecordBatch.from_arrays(arrays, names)
|
||||
table = pa.Table.from_batches([batch])
|
||||
writer = pa.RecordBatchStreamWriter(
|
||||
stream, table.schema
|
||||
)
|
||||
|
||||
writer.write_table(table)
|
||||
writer.close()
|
||||
return stream.getvalue().to_pybytes()
|
||||
|
||||
@staticmethod
|
||||
def make_arrow_from_pandas(df, schema=None, legacy=False):
|
||||
"""Create a pyarrow Table from a Pandas dataframe.
|
||||
|
||||
Args:
|
||||
df (:obj:`pandas.DataFrame`)
|
||||
schema (:obj:`pyarrow.Schema`)
|
||||
legacy (bool): unused; retained for backwards compatibility.
|
||||
|
||||
Returns:
|
||||
pyarrow.Table
|
||||
"""
|
||||
return pa.Table.from_pandas(df, schema=schema)
|
||||
|
||||
@staticmethod
|
||||
def make_dictionary_arrow(names, data, types=None, legacy=False):
|
||||
"""Create an arrow binary that can be loaded and manipulated from memory, with
|
||||
each column being a dictionary array of `str` values and `int` indices.
|
||||
|
||||
Args:
|
||||
names (list): a list of str column names
|
||||
data (list:tuple): a list of tuples, the first value being a list of indices,
|
||||
and the second value being a list of values.
|
||||
types (list:list:pyarrow.func): a list of lists, containing the indices type and
|
||||
dictionary value type for each array.
|
||||
legacy (bool): if True, use legacy IPC format (pre-pyarrow 0.15). Defaults to False.
|
||||
|
||||
Returns:
|
||||
bytes : a bytes object containing the arrow-serialized output.
|
||||
"""
|
||||
stream = pa.BufferOutputStream()
|
||||
|
||||
arrays = []
|
||||
for idx, column in enumerate(data):
|
||||
indice_type = pa.int64()
|
||||
value_type = pa.string()
|
||||
|
||||
if types is not None:
|
||||
indice_type = types[idx][0]
|
||||
value_type = types[idx][1]
|
||||
|
||||
indices = pa.array(column[0], type=indice_type)
|
||||
values = pa.array(column[1], type=value_type)
|
||||
parray = pa.DictionaryArray.from_arrays(indices, values)
|
||||
arrays.append(parray)
|
||||
|
||||
batch = pa.RecordBatch.from_arrays(arrays, names)
|
||||
table = pa.Table.from_batches([batch])
|
||||
writer = pa.RecordBatchStreamWriter(
|
||||
stream, table.schema
|
||||
)
|
||||
|
||||
writer.write_table(table)
|
||||
writer.close()
|
||||
return stream.getvalue().to_pybytes()
|
||||
|
||||
@staticmethod
|
||||
def to_timestamp(obj):
|
||||
"""Return an integer timestamp based on a date/datetime object.
|
||||
|
||||
A `date` is a timezone-agnostic calendar day and serializes to epoch
|
||||
ms at *UTC* midnight regardless of the host process timezone, so it
|
||||
is converted here in UTC. A naive `datetime` is a wall-clock reading
|
||||
in the process-local timezone (callers express expected instants
|
||||
this way, e.g. `aware.astimezone(TZ).replace(tzinfo=None)`), so it is
|
||||
converted via local `timestamp()`.
|
||||
"""
|
||||
classname = obj.__class__.__name__
|
||||
if classname == "date":
|
||||
return int(
|
||||
datetime(
|
||||
obj.year, obj.month, obj.day, tzinfo=timezone.utc
|
||||
).timestamp()
|
||||
* 1000
|
||||
)
|
||||
elif classname == "datetime":
|
||||
return int(obj.timestamp() * 1000)
|
||||
else:
|
||||
return -1
|
||||
|
||||
@staticmethod
|
||||
def make_dataframe(size=10, freq="D"):
|
||||
index = _make_date_time_index(size, freq)
|
||||
return _make_dataframe(index, size)
|
||||
|
||||
@staticmethod
|
||||
def make_period_dataframe(size=10):
|
||||
index = _make_period_index(size, "M")
|
||||
return _make_dataframe(index, size)
|
||||
|
||||
@staticmethod
|
||||
def make_series(size=10, freq="D"):
|
||||
index = _make_date_time_index(size, freq)
|
||||
return pd.Series(data=np.random.rand(size), index=index)
|
||||
|
||||
|
||||
class Sentinel(object):
|
||||
"""Generic sentinel class for testing side-effectful code in Python 2 and
|
||||
3.
|
||||
"""
|
||||
|
||||
def __init__(self, value):
|
||||
self.value = value
|
||||
|
||||
def get(self):
|
||||
return self.value
|
||||
|
||||
def set(self, new_value):
|
||||
self.value = new_value
|
||||
|
||||
|
||||
@fixture()
|
||||
def sentinel():
|
||||
"""Pass `sentinel` into a test and call it with `value` to create a new
|
||||
instance of the Sentinel class.
|
||||
|
||||
Example:
|
||||
>>> def test_with_sentinel(self, sentinel):
|
||||
>>> s = sentinel(True)
|
||||
>>> s.set(False)
|
||||
>>> s.get() # returns False
|
||||
"""
|
||||
|
||||
def _sentinel(value):
|
||||
return Sentinel(value)
|
||||
|
||||
return _sentinel
|
||||
|
||||
|
||||
@fixture
|
||||
def util():
|
||||
"""Pass the `Util` class in to a test."""
|
||||
return Util
|
||||
|
||||
|
||||
@fixture
|
||||
def superstore(count=100):
|
||||
data = []
|
||||
for id in range(count):
|
||||
dat = {}
|
||||
dat["Row ID"] = id
|
||||
dat["Order ID"] = "{}-{}".format(fake.ein(), fake.zipcode())
|
||||
dat["Order Date"] = fake.date_this_year()
|
||||
dat["Ship Date"] = fake.date_between_dates(dat["Order Date"]).strftime(
|
||||
"%Y-%m-%d"
|
||||
)
|
||||
dat["Order Date"] = dat["Order Date"].strftime("%Y-%m-%d")
|
||||
dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"])
|
||||
dat["Ship Mode"] = choice(["First Class", "Standard Class", "Second Class"])
|
||||
dat["Customer ID"] = fake.zipcode()
|
||||
dat["Segment"] = choice(["A", "B", "C", "D"])
|
||||
dat["Country"] = "US"
|
||||
dat["City"] = fake.city()
|
||||
dat["State"] = fake.state()
|
||||
dat["Postal Code"] = fake.zipcode()
|
||||
dat["Region"] = choice(["Region %d" % i for i in range(5)])
|
||||
dat["Product ID"] = fake.bban()
|
||||
sector = choice(["Industrials", "Technology", "Financials"])
|
||||
industry = choice(["A", "B", "C"])
|
||||
dat["Category"] = sector
|
||||
dat["Sub-Category"] = industry
|
||||
dat["Sales"] = randint(1, 100) * 100
|
||||
dat["Quantity"] = randint(1, 100) * 10
|
||||
dat["Discount"] = round(random() * 100, 2)
|
||||
dat["Profit"] = round(random() * 1000, 2)
|
||||
data.append(dat)
|
||||
return pd.DataFrame(data)
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,351 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import queue
|
||||
import random
|
||||
import threading
|
||||
from functools import partial
|
||||
import tornado
|
||||
|
||||
from perspective import (
|
||||
Server,
|
||||
Client,
|
||||
)
|
||||
from pytest import mark
|
||||
|
||||
|
||||
def syncify(f):
|
||||
"""Given a function `f` that must be run on `TestAsync.loop`, queue `f` on
|
||||
the loop, block until it is evaluated, and return the result.
|
||||
"""
|
||||
sem = queue.Queue()
|
||||
|
||||
def _syncify_task():
|
||||
assert threading.current_thread().ident == TestAsync.thread.ident
|
||||
result = f()
|
||||
TestAsync.loop.add_callback(lambda: sem.put(result))
|
||||
|
||||
def _syncify():
|
||||
TestAsync.loop.add_callback(_syncify_task)
|
||||
return sem.get()
|
||||
|
||||
return _syncify
|
||||
|
||||
|
||||
data = [{"a": i, "b": i * 0.5, "c": str(i)} for i in range(10)]
|
||||
|
||||
|
||||
class TestAsync(object):
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
import asyncio
|
||||
|
||||
# Storing the current loop, which was set up by the pytest-asyncio
|
||||
# tests running in tests/async, delays its cleanup, which foregoes a
|
||||
# pytest exception about an unclosed socket. A cleaner way to fix this
|
||||
# probably exists (use an event loop fixture? convert these tests to
|
||||
# pytest-asyncio?)
|
||||
cls.save_old_loop_to_prevent_unclosed_socket_exception = (
|
||||
asyncio.get_event_loop()
|
||||
)
|
||||
asyncio.set_event_loop(asyncio.new_event_loop())
|
||||
cls.loop = tornado.ioloop.IOLoop.current()
|
||||
cls.thread = threading.Thread(target=cls.loop.start)
|
||||
cls.thread.daemon = True
|
||||
cls.thread.start()
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
cls.loop.add_callback(lambda: tornado.ioloop.IOLoop.current().stop())
|
||||
cls.thread.join()
|
||||
cls.loop.close(all_fds=True)
|
||||
|
||||
@classmethod
|
||||
def loop_is_running(cls):
|
||||
return cls.loop.asyncio_loop.is_running()
|
||||
|
||||
def test_async_queue_process(self):
|
||||
server = Server()
|
||||
client = Client.from_server(
|
||||
server,
|
||||
)
|
||||
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
|
||||
@syncify
|
||||
def _task():
|
||||
assert tbl.size() == 0
|
||||
for i in range(5):
|
||||
tbl.update([data[i]])
|
||||
return tbl.size()
|
||||
|
||||
assert _task() == 5
|
||||
tbl.delete()
|
||||
|
||||
def test_async_queue_process_csv(self):
|
||||
"""Make sure GIL release during CSV loading works"""
|
||||
server = Server()
|
||||
client = Client.from_server(
|
||||
server,
|
||||
)
|
||||
tbl = client.table("x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false")
|
||||
|
||||
@syncify
|
||||
def _task():
|
||||
assert tbl.size() == 4
|
||||
for i in range(5):
|
||||
tbl.update("x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false")
|
||||
return tbl.size()
|
||||
|
||||
assert _task() == 24
|
||||
|
||||
tbl.delete()
|
||||
|
||||
def test_async_call_loop(self):
|
||||
server = Server()
|
||||
client = Client.from_server(
|
||||
server,
|
||||
)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
tbl.update(data)
|
||||
|
||||
@syncify
|
||||
def _task():
|
||||
return tbl.size()
|
||||
|
||||
assert _task() == 10
|
||||
tbl.delete()
|
||||
|
||||
def test_async_multiple_managers_queue_process(self):
|
||||
server = Server()
|
||||
client = Client.from_server(server)
|
||||
server2 = Server()
|
||||
client2 = Client.from_server(server2)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
||||
|
||||
@syncify
|
||||
def _update_task():
|
||||
for i in range(5):
|
||||
tbl.update([data[i]])
|
||||
tbl2.update([data[i]])
|
||||
return tbl.size()
|
||||
|
||||
assert _update_task() == 5
|
||||
|
||||
@syncify
|
||||
def _flush_to_process():
|
||||
view = tbl2.view()
|
||||
records = view.to_records()
|
||||
for i in range(5):
|
||||
tbl2.update([data[i]])
|
||||
|
||||
view.delete()
|
||||
return records
|
||||
|
||||
assert _flush_to_process() == data[:5]
|
||||
|
||||
@syncify
|
||||
def _delete_task():
|
||||
tbl2.delete()
|
||||
tbl.delete()
|
||||
|
||||
_delete_task()
|
||||
|
||||
@mark.skip(
|
||||
reason="This test is failing because we're not calling process after each update like before"
|
||||
)
|
||||
def test_async_multiple_managers_mixed_queue_process(self):
|
||||
sentinel = {"called": 0}
|
||||
|
||||
def sync_queue_process(f, *args, **kwargs):
|
||||
sentinel["called"] += 1
|
||||
f(*args, **kwargs)
|
||||
|
||||
server = Server()
|
||||
client = Client.from_server(server)
|
||||
|
||||
server2 = Server()
|
||||
client2 = Client.from_server(server2)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
||||
|
||||
@syncify
|
||||
def _tbl_task():
|
||||
for i in range(5):
|
||||
tbl.update([data[i]])
|
||||
return tbl.size()
|
||||
|
||||
assert _tbl_task() == 5
|
||||
|
||||
for i in range(5):
|
||||
tbl2.update([data[i]])
|
||||
|
||||
assert sentinel["called"] == 5
|
||||
|
||||
@syncify
|
||||
def _tbl_task2():
|
||||
view = tbl.view()
|
||||
records = view.to_records()
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
return records
|
||||
|
||||
assert _tbl_task2() == data[:5]
|
||||
|
||||
view = tbl2.view()
|
||||
assert view.to_records() == data[:5]
|
||||
|
||||
view.delete()
|
||||
tbl2.delete()
|
||||
|
||||
@mark.skip(
|
||||
reason="This test is failing because we're not calling process after each update like before"
|
||||
)
|
||||
def test_async_multiple_managers_delayed_process(self):
|
||||
sentinel = {"async": 0, "sync": 0}
|
||||
|
||||
def _counter(key, f, *args, **kwargs):
|
||||
sentinel[key] += 1
|
||||
return f(*args, **kwargs)
|
||||
|
||||
short_delay_queue_process = partial(_counter, "sync")
|
||||
long_delay_queue_process = partial(
|
||||
TestAsync.loop.add_timeout, 1, _counter, "async"
|
||||
)
|
||||
|
||||
server = Server(on_poll_request=short_delay_queue_process)
|
||||
client = Client.from_server(server)
|
||||
|
||||
server2 = Server(on_poll_request=long_delay_queue_process)
|
||||
client2 = Client.from_server(server2)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
||||
|
||||
@syncify
|
||||
def _tbl_task():
|
||||
for i in range(10):
|
||||
tbl2.update([data[i]])
|
||||
|
||||
_tbl_task()
|
||||
for i in range(10):
|
||||
tbl.update([data[i]])
|
||||
|
||||
@syncify
|
||||
def _tbl_task2():
|
||||
size = tbl2.size()
|
||||
tbl2.delete()
|
||||
return size
|
||||
|
||||
assert _tbl_task2() == 10
|
||||
assert tbl.size() == 10
|
||||
assert sentinel["async"] == 1
|
||||
assert sentinel["sync"] == 10
|
||||
|
||||
tbl.delete()
|
||||
|
||||
def test_async_single_manager_tables_chained(self):
|
||||
def call_loop(fn, *args):
|
||||
TestAsync.loop.add_callback(fn, *args)
|
||||
|
||||
server = Server()
|
||||
client = Client.from_server(server)
|
||||
columns = {"index": "integer", "num1": "integer", "num2": "integer"}
|
||||
# tbl = client.table(columns, index="index")
|
||||
tbl = client.table(columns)
|
||||
view = tbl.view()
|
||||
tbl2 = client.table(view.to_arrow())
|
||||
|
||||
def _update(port_id, delta):
|
||||
print("Updating tbl2", delta)
|
||||
tbl2.update(delta)
|
||||
|
||||
view.on_update(_update, mode="row")
|
||||
for i in range(1000):
|
||||
call_loop(tbl.update, [{"index": i, "num1": i, "num2": 2 * i}])
|
||||
i += 1
|
||||
|
||||
call_loop(tbl.size)
|
||||
|
||||
q = queue.Queue()
|
||||
call_loop(q.put, True)
|
||||
q.get()
|
||||
|
||||
@syncify
|
||||
def _tbl_task2():
|
||||
size = tbl2.size()
|
||||
return size
|
||||
|
||||
assert _tbl_task2() == 1000
|
||||
# assert tbl2.size() == 1000
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
tbl2.delete()
|
||||
|
||||
def test_async_queue_process_multiple_ports(self):
|
||||
server = Server()
|
||||
client = Client.from_server(server)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
port_ids = [0]
|
||||
port_data = [{"a": 0, "b": 0, "c": "0"}]
|
||||
|
||||
for i in range(10):
|
||||
port_id = tbl.make_port()
|
||||
port_ids.append(port_id)
|
||||
port_data.append({"a": port_id, "b": port_id * 1.5, "c": str(port_id)})
|
||||
|
||||
assert port_ids == list(range(0, 11))
|
||||
|
||||
assert syncify(lambda: tbl.size())() == 0
|
||||
|
||||
random.shuffle(port_ids)
|
||||
|
||||
@syncify
|
||||
def _tbl_task():
|
||||
for port_id in port_ids:
|
||||
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
|
||||
tbl.update([port_data[idx]], port_id=port_id)
|
||||
size = tbl.size()
|
||||
tbl.delete()
|
||||
return size
|
||||
|
||||
assert len(port_ids) == 11
|
||||
assert _tbl_task() == 11
|
||||
|
||||
def test_async_multiple_managers_queue_process_multiple_ports(self):
|
||||
server = Server()
|
||||
client = Client.from_server(server)
|
||||
|
||||
server2 = Server()
|
||||
client2 = Client.from_server(server2)
|
||||
tbl = client.table({"a": "integer", "b": "float", "c": "string"})
|
||||
tbl2 = client2.table({"a": "integer", "b": "float", "c": "string"})
|
||||
port_ids = [0]
|
||||
port_data = [{"a": 0, "b": 0, "c": "0"}]
|
||||
|
||||
for i in range(10):
|
||||
port_id = tbl.make_port()
|
||||
port_id2 = tbl2.make_port()
|
||||
assert port_id == port_id2
|
||||
port_ids.append(port_id)
|
||||
port_data.append({"a": port_id, "b": port_id * 1.5, "c": str(port_id)})
|
||||
|
||||
@syncify
|
||||
def _task():
|
||||
random.shuffle(port_ids)
|
||||
for port_id in port_ids:
|
||||
idx = port_id if port_id < len(port_ids) else len(port_ids) - 1
|
||||
tbl.update([port_data[idx]], port_id=port_id)
|
||||
tbl2.update([port_data[idx]], port_id=port_id)
|
||||
return (tbl.size(), tbl2.size())
|
||||
|
||||
assert _task() == (11, 11)
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,201 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from perspective import Server
|
||||
|
||||
import asyncio
|
||||
import random
|
||||
import threading
|
||||
|
||||
from random import sample
|
||||
from string import ascii_letters
|
||||
from threading import Thread
|
||||
from time import sleep
|
||||
|
||||
class TestServer(object):
|
||||
def test_sync_updates_with_loop_callback_are_sync(self):
|
||||
def feed(table):
|
||||
y = 1000
|
||||
while y > 0:
|
||||
y -= 1
|
||||
table.update([{"a": random.randint(0, 10), "index": y}])
|
||||
|
||||
perspective_server = Server()
|
||||
loop = asyncio.new_event_loop()
|
||||
thread = threading.Thread(target=loop.run_forever)
|
||||
thread.start()
|
||||
|
||||
client = perspective_server.new_local_client()
|
||||
table = client.table(
|
||||
{"a": "integer", "index": "integer"}, index="index", name="test"
|
||||
)
|
||||
|
||||
view = table.view()
|
||||
global count
|
||||
count = 0
|
||||
|
||||
def update(x):
|
||||
global count
|
||||
count += 1
|
||||
|
||||
view.on_update(update)
|
||||
feed(table)
|
||||
assert table.size() == 1000
|
||||
assert count == 1000
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
thread.join()
|
||||
loop.close()
|
||||
|
||||
def test_concurrent_updates(self):
|
||||
async def feed(table, loop):
|
||||
y = 1000
|
||||
while y > 0:
|
||||
y -= 1
|
||||
table.update([{"a": random.randint(0, 10), "index": y}])
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
perspective_server = Server()
|
||||
loop = asyncio.new_event_loop()
|
||||
thread = threading.Thread(target=loop.run_forever)
|
||||
thread.start()
|
||||
|
||||
client = perspective_server.new_local_client()
|
||||
table = client.table(
|
||||
{"a": "integer", "index": "integer"}, index="index", name="test"
|
||||
)
|
||||
|
||||
view = table.view()
|
||||
global count
|
||||
count = 0
|
||||
|
||||
def update(x):
|
||||
global count
|
||||
count += 1
|
||||
|
||||
view.on_update(update)
|
||||
asyncio.run_coroutine_threadsafe(feed(table, loop), loop).result()
|
||||
assert table.size() == 1000
|
||||
assert count == 1000
|
||||
loop.call_soon_threadsafe(loop.stop)
|
||||
thread.join()
|
||||
loop.close()
|
||||
|
||||
def test_concurrent_updates_with_limit_tables_are_threadsafe(self):
|
||||
TEST_ITERATIONS = 100
|
||||
global running
|
||||
perspective_server = Server()
|
||||
client = perspective_server.new_local_client()
|
||||
table = client.table(
|
||||
{"col{}".format(i): "integer" for i in range(100)}, limit=100
|
||||
)
|
||||
|
||||
running = True
|
||||
|
||||
# Create an updating thread that overlaps the index alot
|
||||
def feed(table):
|
||||
row = {"col{}".format(i): random.randint(0, 100) for i in range(100)}
|
||||
while running:
|
||||
table.update([row for _ in range(100)])
|
||||
|
||||
thread = threading.Thread(target=feed, args=(table,))
|
||||
thread.start()
|
||||
|
||||
results = []
|
||||
|
||||
# Create a thread that serialized the table alot, checking for nulls
|
||||
def feed2(table):
|
||||
global running
|
||||
view = table.view()
|
||||
while len(results) < TEST_ITERATIONS:
|
||||
arr = view.to_arrow()
|
||||
table2 = client.table(arr)
|
||||
view2 = table2.view()
|
||||
json = view2.to_json(end_row=1)
|
||||
view2.delete()
|
||||
table2.delete()
|
||||
results.append(json)
|
||||
|
||||
view.delete()
|
||||
running = False
|
||||
|
||||
thread2 = threading.Thread(target=feed2, args=(table,))
|
||||
thread2.start()
|
||||
|
||||
thread.join()
|
||||
thread2.join()
|
||||
|
||||
assert table.size() == 100
|
||||
for result in results:
|
||||
for row in result:
|
||||
for col, val in row.items():
|
||||
assert val is not None
|
||||
|
||||
table.delete()
|
||||
|
||||
def test_concurrent_view_creation_on_separate_servers_are_threadsafe(self):
|
||||
def run_perspective():
|
||||
x = 1000
|
||||
s = Server()
|
||||
c = s.new_local_client()
|
||||
t = c.table({"a": "string", "b": int})
|
||||
t.view(expressions=["//tmp\nfalse"])
|
||||
while x > 0:
|
||||
t.update([{"a": "foo", "b": 42}])
|
||||
x -= 1
|
||||
|
||||
thread1 = threading.Thread(target=run_perspective, daemon=True)
|
||||
thread2 = threading.Thread(target=run_perspective, daemon=True)
|
||||
thread1.start()
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
|
||||
def test_concurrent_view_creation_with_updates_are_threadsafe(self):
|
||||
global running
|
||||
s = Server()
|
||||
schema = {
|
||||
"a": "string",
|
||||
"b": "string",
|
||||
"c": "string",
|
||||
}
|
||||
|
||||
group_bys = ["a", "b", "c"]
|
||||
c = s.new_local_client()
|
||||
t = c.table(schema, limit=10000)
|
||||
running = True
|
||||
|
||||
def gen_views():
|
||||
global running
|
||||
for _ in range(100):
|
||||
t.view(columns=list(schema.keys()), group_by=group_bys)
|
||||
sleep(0.01)
|
||||
running = False
|
||||
|
||||
def run_psp():
|
||||
global running
|
||||
while running:
|
||||
t.update(
|
||||
[
|
||||
{
|
||||
"a": "".join(sample(ascii_letters, 4)),
|
||||
"b": "".join(sample(ascii_letters, 4)),
|
||||
"c": "".join(sample(ascii_letters, 4)),
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
thread1 = Thread(target=run_psp, daemon=True)
|
||||
thread2 = Thread(target=gen_views, daemon=True)
|
||||
thread1.start()
|
||||
thread2.start()
|
||||
thread1.join()
|
||||
thread2.join()
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,110 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from perspective import Server, Client, ProxySession, AsyncClient
|
||||
import pytest
|
||||
|
||||
|
||||
client = Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
data = {"a": [1, 2, 3], "b": ["a", "b", "c"]}
|
||||
|
||||
|
||||
class TestProxySession(object):
|
||||
def test_proxy_client(self):
|
||||
server = Server()
|
||||
client = server.new_local_client()
|
||||
|
||||
def handle_request(bytes):
|
||||
sub_session.handle_request(bytes)
|
||||
|
||||
def handle_response(bytes):
|
||||
sub_client.handle_response(bytes)
|
||||
|
||||
sub_session = ProxySession(client, handle_response)
|
||||
sub_client = Client(handle_request, sub_session.close)
|
||||
table = sub_client.table(data, name="table1")
|
||||
assert table.schema() == {"a": "integer", "b": "string"}
|
||||
|
||||
def test_proxy_and_local_client_can_share_handles(self):
|
||||
server = Server()
|
||||
client = server.new_local_client()
|
||||
|
||||
def handle_request(bytes):
|
||||
sub_session.handle_request(bytes)
|
||||
|
||||
def handle_response(bytes):
|
||||
sub_client.handle_response(bytes)
|
||||
|
||||
sub_session = ProxySession(client, handle_response)
|
||||
sub_client = Client(handle_request, sub_session.close)
|
||||
table = sub_client.table(data, name="table1")
|
||||
table2 = client.open_table("table1")
|
||||
assert table.size() == 3
|
||||
table2.update([{"a": 4, "d": 5}])
|
||||
assert table.size() == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_request_async(self):
|
||||
"""tests use of ProxySession.handle_request_async() in an AsyncClient callback"""
|
||||
server = Server()
|
||||
client = server.new_local_client()
|
||||
|
||||
def handle_response(bytes):
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(sub_client.handle_response(bytes))
|
||||
|
||||
sub_session = ProxySession(client, handle_response)
|
||||
sub_client = AsyncClient(sub_session.handle_request_async, sub_session.close)
|
||||
table = await sub_client.table(data, name="table1")
|
||||
table2 = client.open_table("table1")
|
||||
assert (await table.size()) == 3
|
||||
table2.update([{"a": 4, "d": 5}])
|
||||
assert (await table.size()) == 4
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dismabiguate_message_ids_in_proxy_session(self):
|
||||
"""tests concurrent calls from clients on Session + ProxySession"""
|
||||
server = Server()
|
||||
client = server.new_local_client()
|
||||
|
||||
def send_response(bytes):
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(sub_client.handle_response(bytes))
|
||||
|
||||
async def send_request(bytes):
|
||||
await sub_session.handle_request_async(bytes)
|
||||
|
||||
sub_session = ProxySession(client, send_response)
|
||||
sub_client = AsyncClient(send_request, sub_session.close)
|
||||
|
||||
table = client.table("x\n1", name="test_table")
|
||||
table2 = await sub_client.open_table("test_table")
|
||||
view = table.view()
|
||||
view2 = await table2.view()
|
||||
|
||||
sentinel = []
|
||||
|
||||
def callback1(*args):
|
||||
sentinel.append("Callback 1")
|
||||
|
||||
def callback2(*args):
|
||||
sentinel.append("Callback 2")
|
||||
|
||||
view.on_update(callback1)
|
||||
await view2.on_update(callback2)
|
||||
await table2.update("x\n2")
|
||||
assert sentinel == ["Callback 1", "Callback 2"]
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,402 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import sys
|
||||
from random import randint, choice
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class CustomObjectStore(object):
|
||||
def __init__(self, value):
|
||||
self._value = value
|
||||
|
||||
def _psp_dtype_(self):
|
||||
return "object"
|
||||
|
||||
def __int__(self):
|
||||
return int(self._value)
|
||||
|
||||
def __repr__(self):
|
||||
return "test" if self._value == 1 else "test{}".format(self._value)
|
||||
|
||||
|
||||
def run():
|
||||
t = CustomObjectStore(1)
|
||||
t2 = CustomObjectStore(2)
|
||||
|
||||
assert sys.getrefcount(t) == 2
|
||||
assert sys.getrefcount(t2) == 2
|
||||
|
||||
data = {"a": [0], "b": [t]}
|
||||
assert sys.getrefcount(t) == 3
|
||||
|
||||
tbl = Table(data, index="a")
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
assert tbl.schema() == {"a": int, "b": object}
|
||||
assert tbl.size() == 1
|
||||
assert tbl.view().to_columns() == {"a": [0], "b": [t]}
|
||||
|
||||
# Count references
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
print("t:", id(t))
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
print(sys.getrefcount(t2), "should be", 2)
|
||||
print("t2:", id(t2))
|
||||
assert sys.getrefcount(t2) == 2
|
||||
|
||||
tbl.update([{"a": i, "b": None} for i in range(1, 6)])
|
||||
|
||||
assert tbl.size() == 6
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
print()
|
||||
tbl.update([data])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
print()
|
||||
tbl.update([data])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
print()
|
||||
tbl.update([data])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
assert sys.getrefcount(t) == 4
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 2 for the table
|
||||
print(sys.getrefcount(t), "should be", 5)
|
||||
assert sys.getrefcount(t) == 5
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 2 for the table
|
||||
print(sys.getrefcount(t), "should be", 5)
|
||||
assert sys.getrefcount(t) == 5
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 3, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
assert sys.getrefcount(t) == 6
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 3, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
assert sys.getrefcount(t) == 6
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 5, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 4for the table
|
||||
print(sys.getrefcount(t), "should be", 7)
|
||||
# assert sys.getrefcount(t) == 7
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
# clear some, overwrite some with same
|
||||
tbl.update([{"a": 0, "b": t}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 4 for the table
|
||||
print(sys.getrefcount(t), "should be", 7)
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 1, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 2, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 4)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 2, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 2, "b": t2}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 4)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 2, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 3 for the table
|
||||
print(sys.getrefcount(t), "should be", 6)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 3, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 2 for the table
|
||||
print(sys.getrefcount(t), "should be", 5)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 3, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 2 for the table
|
||||
print(sys.getrefcount(t), "should be", 5)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
True,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 5, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.update([{"a": 5, "b": None}])
|
||||
# 1 for `t`, 1 for `data`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t), "should be", 4)
|
||||
# 1 for `t2`, 1 for argument to sys.getrefcount, and 1 for the table
|
||||
print(sys.getrefcount(t2), "should be", 3)
|
||||
print(tbl.view().to_columns()["b"])
|
||||
assert list(_ is not None for _ in tbl.view().to_columns()["b"]) == [
|
||||
True,
|
||||
True,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
False,
|
||||
]
|
||||
|
||||
print()
|
||||
tbl.clear()
|
||||
assert tbl.size() == 0
|
||||
assert tbl.view().to_columns() == {}
|
||||
# 1 for `t`, one for `data`, one for argument to sys.getrefcount
|
||||
print(sys.getrefcount(t), "should be", 3)
|
||||
assert sys.getrefcount(t) == 3
|
||||
|
||||
|
||||
def run2():
|
||||
t = CustomObjectStore(1)
|
||||
t_ref_count = 2
|
||||
assert sys.getrefcount(t) == t_ref_count
|
||||
|
||||
indexes = set([0])
|
||||
|
||||
tbl = Table({"a": [0], "b": [t]}, index="a")
|
||||
assert sys.getrefcount(t) == 3
|
||||
t_ref_count += 1
|
||||
|
||||
assert tbl.schema() == {"a": int, "b": object}
|
||||
assert tbl.size() == 1
|
||||
assert tbl.view().to_columns() == {"a": [0], "b": [t]}
|
||||
|
||||
# seed a few to check
|
||||
tbl.remove([1])
|
||||
tbl.remove([1])
|
||||
tbl.remove([1])
|
||||
|
||||
for _ in range(10):
|
||||
pick = randint(1, 2) if indexes else 1
|
||||
if pick == 1:
|
||||
ind = randint(1, 10)
|
||||
while ind in indexes:
|
||||
ind = randint(1, 100)
|
||||
|
||||
print(
|
||||
"adding", ind, "refcount", t_ref_count, "should be", sys.getrefcount(t)
|
||||
)
|
||||
tbl.update({"a": [ind], "b": [t]})
|
||||
t_ref_count += 1
|
||||
indexes.add(ind)
|
||||
assert sys.getrefcount(t) == t_ref_count
|
||||
|
||||
else:
|
||||
ind = choice(list(indexes))
|
||||
indexes.remove(ind)
|
||||
tbl.remove([ind])
|
||||
t_ref_count -= 1
|
||||
print(
|
||||
"removing",
|
||||
ind,
|
||||
"refcount",
|
||||
t_ref_count,
|
||||
"should be",
|
||||
sys.getrefcount(t),
|
||||
)
|
||||
assert sys.getrefcount(t) == t_ref_count
|
||||
|
||||
print(t_ref_count)
|
||||
print(tbl.view().to_columns())
|
||||
|
||||
assert sys.getrefcount(t) == t_ref_count
|
||||
|
||||
print()
|
||||
tbl.clear()
|
||||
assert tbl.size() == 0
|
||||
assert tbl.view().to_columns() == {}
|
||||
# 1 for `t`, one for `data`, one for argument to sys.getrefcount
|
||||
print(sys.getrefcount(t), "should be", 2)
|
||||
assert sys.getrefcount(t) == 2
|
||||
@@ -0,0 +1,89 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
|
||||
|
||||
class TestViewColumnPaths(object):
|
||||
def test_column_paths(self, superstore):
|
||||
tbl = client.table(superstore)
|
||||
view = tbl.view()
|
||||
paths = view.column_paths()
|
||||
assert paths == [
|
||||
"index",
|
||||
"Row ID",
|
||||
"Order ID",
|
||||
"Order Date",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"Customer ID",
|
||||
"Segment",
|
||||
"Country",
|
||||
"City",
|
||||
"State",
|
||||
"Postal Code",
|
||||
"Region",
|
||||
"Product ID",
|
||||
"Category",
|
||||
"Sub-Category",
|
||||
"Sales",
|
||||
"Quantity",
|
||||
"Discount",
|
||||
"Profit",
|
||||
]
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
def test_column_paths_split_by(self, superstore):
|
||||
tbl = client.table(superstore)
|
||||
view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"])
|
||||
paths = view.column_paths()
|
||||
assert paths == [
|
||||
"First Class|Sales",
|
||||
"Second Class|Sales",
|
||||
"Standard Class|Sales",
|
||||
]
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
def test_column_paths_split_by_range(self, superstore):
|
||||
tbl = client.table(superstore)
|
||||
view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"])
|
||||
paths = view.column_paths(start_col=1)
|
||||
assert paths == [
|
||||
"Second Class|Sales",
|
||||
"Standard Class|Sales",
|
||||
]
|
||||
|
||||
view.delete()
|
||||
view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"])
|
||||
paths = view.column_paths(start_col=1, end_col=2)
|
||||
assert paths == [
|
||||
"Second Class|Sales",
|
||||
"Standard Class|Sales",
|
||||
]
|
||||
|
||||
view.delete()
|
||||
tbl = client.table(superstore)
|
||||
view = tbl.view(group_by=["State"], columns=["Sales"], split_by=["Ship Mode"])
|
||||
paths = view.column_paths(end_col=1)
|
||||
assert paths == [
|
||||
"First Class|Sales",
|
||||
"Second Class|Sales",
|
||||
]
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
@@ -0,0 +1,124 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestDelete(object):
|
||||
# delete
|
||||
|
||||
def test_table_delete(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.delete()
|
||||
# don't segfault
|
||||
|
||||
def test_table_delete_callback(self, sentinel):
|
||||
s = sentinel(False)
|
||||
|
||||
def callback():
|
||||
s.set(True)
|
||||
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.on_delete(callback)
|
||||
tbl.delete()
|
||||
assert s.get() is True
|
||||
|
||||
def test_table_delete_with_view(self, sentinel):
|
||||
s = sentinel(False)
|
||||
|
||||
def callback():
|
||||
s.set(True)
|
||||
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.on_delete(callback)
|
||||
view = tbl.view()
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
assert s.get() is True
|
||||
|
||||
def test_table_delete_multiple_callback(self, sentinel):
|
||||
s1 = sentinel(False)
|
||||
s2 = sentinel(False)
|
||||
|
||||
def callback1():
|
||||
s1.set(True)
|
||||
|
||||
def callback2():
|
||||
s2.set(True)
|
||||
|
||||
tbl = Table([{"a": 1}])
|
||||
tbl.on_delete(callback1)
|
||||
tbl.on_delete(callback2)
|
||||
|
||||
tbl.delete()
|
||||
|
||||
assert s1.get() is True
|
||||
assert s2.get() is True
|
||||
|
||||
def test_table_remove_delete_callback(self, sentinel):
|
||||
s = sentinel(False)
|
||||
|
||||
def callback():
|
||||
s.set(True)
|
||||
|
||||
tbl = Table([{"a": 1}])
|
||||
callback_id = tbl.on_delete(callback)
|
||||
tbl.remove_delete(callback_id)
|
||||
|
||||
tbl.delete()
|
||||
|
||||
assert s.get() is False
|
||||
|
||||
def test_view_delete_multiple_callback(self, sentinel):
|
||||
s1 = sentinel(False)
|
||||
s2 = sentinel(False)
|
||||
|
||||
def callback1():
|
||||
s1.set(True)
|
||||
|
||||
def callback2():
|
||||
s2.set(True)
|
||||
|
||||
tbl = Table([{"a": 1}])
|
||||
view = tbl.view()
|
||||
|
||||
view.on_delete(callback1)
|
||||
view.on_delete(callback2)
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
assert s1.get() is True
|
||||
assert s2.get() is True
|
||||
|
||||
def test_view_remove_delete_callback(self, sentinel):
|
||||
s = sentinel(False)
|
||||
|
||||
def callback():
|
||||
s.set(True)
|
||||
|
||||
tbl = Table([{"a": 1}])
|
||||
view = tbl.view()
|
||||
|
||||
callback_id = view.on_delete(callback)
|
||||
view.remove_delete(callback_id)
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
assert s.get() is False
|
||||
@@ -0,0 +1,65 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from pytest import raises
|
||||
from perspective import PerspectiveError
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestException(object):
|
||||
def test_instantiation_exception_table(self):
|
||||
from perspective import Table
|
||||
with raises(TypeError) as ex:
|
||||
Table()
|
||||
assert "Do not call Table's constructor directly" in str(ex.value)
|
||||
|
||||
def test_instantiation_exception_view(self):
|
||||
from perspective import View
|
||||
with raises(TypeError) as ex:
|
||||
View()
|
||||
assert "Do not call View's constructor directly" in str(ex.value)
|
||||
|
||||
def test_exception_from_core(self):
|
||||
tbl = Table({"a": [1, 2, 3]})
|
||||
|
||||
with raises(PerspectiveError) as ex:
|
||||
# creating view with unknown column should throw
|
||||
tbl.view(group_by=["b"])
|
||||
|
||||
assert str(ex.value) == "Abort(): Invalid column 'b' found in View group_by.\n"
|
||||
|
||||
def test_exception_from_core_catch_generic(self):
|
||||
tbl = Table({"a": [1, 2, 3]})
|
||||
# `PerspectiveCppError` should inherit from `Exception`
|
||||
with raises(Exception) as ex:
|
||||
tbl.view(group_by=["b"])
|
||||
|
||||
assert str(ex.value) == "Abort(): Invalid column 'b' found in View group_by.\n"
|
||||
|
||||
def test_exception_from_core_correct_types(self):
|
||||
tbl = Table({"a": [1, 2, 3]})
|
||||
|
||||
# `PerspectiveError` should be raised from the Python layer
|
||||
with raises(PerspectiveError) as ex:
|
||||
tbl.view()
|
||||
tbl.delete()
|
||||
|
||||
assert str(ex.value) == "Abort(): Cannot delete table with views"
|
||||
|
||||
with raises(PerspectiveError) as ex:
|
||||
tbl.view(group_by=["b"])
|
||||
|
||||
assert str(ex.value) == "Abort(): Invalid column 'b' found in View group_by.\n"
|
||||
@@ -0,0 +1,115 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
|
||||
|
||||
class TestJoin:
|
||||
def test_inner_join_two_tables(self):
|
||||
left = client.table(
|
||||
[{"id": 1, "x": 10}, {"id": 2, "x": 20}, {"id": 3, "x": 30}]
|
||||
)
|
||||
right = client.table(
|
||||
[{"id": 1, "y": "a"}, {"id": 2, "y": "b"}, {"id": 4, "y": "d"}]
|
||||
)
|
||||
joined = client.join(left, right, "id")
|
||||
view = joined.view()
|
||||
json = view.to_json()
|
||||
assert len(json) == 2
|
||||
view.delete()
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
|
||||
def test_join_has_correct_schema(self):
|
||||
left = client.table({"id": "integer", "x": "float"})
|
||||
right = client.table({"id": "integer", "y": "string"})
|
||||
joined = client.join(left, right, "id")
|
||||
schema = joined.schema()
|
||||
assert schema == {"id": "integer", "x": "float", "y": "string"}
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
|
||||
def test_join_reacts_to_left_updates(self):
|
||||
left = client.table(
|
||||
[{"id": 1, "x": 10}, {"id": 2, "x": 20}]
|
||||
)
|
||||
right = client.table(
|
||||
[{"id": 1, "y": "a"}, {"id": 2, "y": "b"}]
|
||||
)
|
||||
joined = client.join(left, right, "id")
|
||||
view = joined.view()
|
||||
left.update([{"id": 1, "x": 99}])
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"id": 1, "x": 10, "y": "a"},
|
||||
{"id": 2, "x": 20, "y": "b"},
|
||||
{"id": 1, "x": 99, "y": "a"},
|
||||
]
|
||||
view.delete()
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
|
||||
def test_left_join(self):
|
||||
left = client.table(
|
||||
[{"id": 1, "x": 10}, {"id": 2, "x": 20}, {"id": 3, "x": 30}]
|
||||
)
|
||||
right = client.table(
|
||||
[{"id": 1, "y": "a"}, {"id": 2, "y": "b"}]
|
||||
)
|
||||
joined = client.join(left, right, "id", "left")
|
||||
view = joined.view()
|
||||
json = view.to_json()
|
||||
assert len(json) == 3
|
||||
view.delete()
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
|
||||
def test_join_by_table_names(self):
|
||||
left = client.table(
|
||||
[{"id": 1, "x": 10}, {"id": 2, "x": 20}],
|
||||
name="left_py",
|
||||
)
|
||||
right = client.table(
|
||||
[{"id": 1, "y": "a"}, {"id": 2, "y": "b"}],
|
||||
name="right_py",
|
||||
)
|
||||
joined = client.join("left_py", "right_py", "id")
|
||||
view = joined.view()
|
||||
json = view.to_json()
|
||||
assert len(json) == 2
|
||||
view.delete()
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
|
||||
def test_join_mixed_table_and_string(self):
|
||||
left = client.table(
|
||||
[{"id": 1, "x": 10}, {"id": 2, "x": 20}]
|
||||
)
|
||||
right = client.table(
|
||||
[{"id": 1, "y": "a"}, {"id": 2, "y": "b"}],
|
||||
name="right_py_mixed",
|
||||
)
|
||||
joined = client.join(left, "right_py_mixed", "id")
|
||||
view = joined.view()
|
||||
json = view.to_json()
|
||||
assert len(json) == 2
|
||||
view.delete()
|
||||
joined.delete()
|
||||
right.delete()
|
||||
left.delete()
|
||||
@@ -0,0 +1,105 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
|
||||
import psutil
|
||||
import os
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestDelete(object):
|
||||
# delete
|
||||
|
||||
def test_table_delete(self):
|
||||
process = psutil.Process(os.getpid())
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.delete()
|
||||
mem = process.memory_info().rss
|
||||
|
||||
for x in range(10000):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.delete()
|
||||
|
||||
mem2 = process.memory_info().rss
|
||||
|
||||
# assert 1 < (max2 / max) < 1.01
|
||||
assert (mem2 - mem) < 2000000
|
||||
|
||||
def test_table_delete_with_view(self, sentinel):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
|
||||
process = psutil.Process(os.getpid())
|
||||
mem = process.memory_info().rss
|
||||
for x in range(10000):
|
||||
view = tbl.view()
|
||||
view.delete()
|
||||
|
||||
tbl.delete()
|
||||
mem2 = process.memory_info().rss
|
||||
assert (mem2 - mem) < 2000000
|
||||
|
||||
|
||||
class TestExpressionVocab(object):
|
||||
# Regression tests for the `t_expression_vocab` leak fixed in PR #3175.
|
||||
# Pre-fix, updates against a view with a string-producing expression
|
||||
# interned each result into a new 4KB vocab page once the prior page
|
||||
# filled, and never deduplicated against older pages. RSS grew without
|
||||
# bound. The fix adds cross-page deduplication; these tests assert the
|
||||
# steady state is flat.
|
||||
|
||||
def test_string_expression_update_no_leak(self):
|
||||
long_literal = "X" * 256
|
||||
tbl = Table({"x": "integer", "c": "string"}, index="x")
|
||||
view = tbl.view(expressions={"e": 'concat("c", \'' + long_literal + "')"})
|
||||
|
||||
for _ in range(100):
|
||||
tbl.update([{"x": 1, "c": "value"}])
|
||||
|
||||
process = psutil.Process(os.getpid())
|
||||
mem = process.memory_info().rss
|
||||
for _ in range(5000):
|
||||
tbl.update([{"x": 1, "c": "value"}])
|
||||
mem2 = process.memory_info().rss
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
assert (mem2 - mem) < 2000000
|
||||
|
||||
def test_string_expression_update_bounded_vocab_no_leak(self):
|
||||
# Cycles through a small fixed set of distinct values. Exercises the
|
||||
# cross-page `string_exists` lookup the fix relies on: once the
|
||||
# vocabulary is fully populated, subsequent interns must resolve to
|
||||
# existing pointers rather than allocating new ones.
|
||||
values = ["alpha", "bravo", "charlie", "delta", "echo"]
|
||||
tbl = Table({"x": "integer", "c": "string"}, index="x")
|
||||
view = tbl.view(expressions={"e": 'upper("c")'})
|
||||
|
||||
for i in range(100):
|
||||
tbl.update([{"x": 1, "c": values[i % len(values)]}])
|
||||
|
||||
process = psutil.Process(os.getpid())
|
||||
mem = process.memory_info().rss
|
||||
for i in range(5000):
|
||||
tbl.update([{"x": 1, "c": values[i % len(values)]}])
|
||||
mem2 = process.memory_info().rss
|
||||
|
||||
view.delete()
|
||||
tbl.delete()
|
||||
|
||||
assert (mem2 - mem) < 2000000
|
||||
@@ -0,0 +1,178 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import random
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
data = {"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"], "c": [True, False, True, False]}
|
||||
|
||||
|
||||
class TestPorts(object):
|
||||
def test_make_port_sequential(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
def test_make_port_sequential_and_update(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
for i in range(1, 11):
|
||||
table.update({"a": [i], "b": ["a"], "c": [True]}, port_id=i)
|
||||
|
||||
view = table.view()
|
||||
result = view.to_columns()
|
||||
|
||||
assert result == {
|
||||
"a": [1, 2, 3, 4] + [i for i in range(1, 11)],
|
||||
"b": ["a", "b", "c", "d"] + ["a" for i in range(10)],
|
||||
"c": [True, False, True, False] + [True for i in range(10)],
|
||||
}
|
||||
|
||||
def test_arbitary_port_updates(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
port = random.randint(0, 10)
|
||||
|
||||
table.update(data, port_id=port)
|
||||
|
||||
assert table.size() == 8
|
||||
|
||||
assert table.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4] * 2,
|
||||
"b": ["a", "b", "c", "d"] * 2,
|
||||
"c": [True, False, True, False] * 2,
|
||||
}
|
||||
|
||||
def test_ports_should_only_notify_if_they_have_a_queued_update(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
view = table.view()
|
||||
ports_to_update = [random.randint(0, 10) for i in range(5)]
|
||||
|
||||
def callback(port_id):
|
||||
assert port_id in ports_to_update
|
||||
|
||||
view.on_update(callback)
|
||||
|
||||
for port in ports_to_update:
|
||||
table.update(data, port_id=port)
|
||||
|
||||
def test_ports_should_have_unique_deltas(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
view = table.view()
|
||||
ports_to_update = [random.randint(0, 10) for i in range(5)]
|
||||
unique_data = {
|
||||
port: [{"a": port, "b": str(port), "c": True}] for port in ports_to_update
|
||||
}
|
||||
|
||||
def callback(port_id, delta):
|
||||
assert port_id in ports_to_update
|
||||
_t = Table(delta)
|
||||
_v = _t.view()
|
||||
assert _v.to_records() == unique_data[port]
|
||||
_v.delete()
|
||||
_t.delete()
|
||||
|
||||
view.on_update(callback, mode="row")
|
||||
|
||||
for port in ports_to_update:
|
||||
table.update(unique_data[port], port_id=port)
|
||||
|
||||
def test_ports_should_queue_updates_properly(self):
|
||||
table = Table(data)
|
||||
port_ids = []
|
||||
|
||||
for i in range(10):
|
||||
port_ids.append(table.make_port())
|
||||
|
||||
assert port_ids == list(range(1, 11))
|
||||
|
||||
view = table.view()
|
||||
ports_to_update = [random.randint(0, 10) for i in range(5)]
|
||||
|
||||
def callback(port_id):
|
||||
assert port_id in ports_to_update
|
||||
|
||||
view.on_update(callback)
|
||||
|
||||
for port in ports_to_update:
|
||||
table.update(data, port_id=port)
|
||||
|
||||
def test_ports_multiple_tables_with_different_ports(self):
|
||||
server = Table(data)
|
||||
client = Table(data)
|
||||
|
||||
for i in range(random.randint(5, 15)):
|
||||
# reserve an arbitary number of ports
|
||||
server.make_port()
|
||||
|
||||
# port for client is now far above the ports "ON" the client, as the
|
||||
# client ports will begin creation at 1.
|
||||
server_port_for_client = server.make_port()
|
||||
client_port = client.make_port()
|
||||
|
||||
server_view = server.view()
|
||||
client_view = client.view()
|
||||
|
||||
# when the client updates, check whether the port id matches that
|
||||
# of the server, and complete the test.
|
||||
def client_callback(port_id, delta):
|
||||
if port_id == client_port:
|
||||
print("UPDATING SERVER")
|
||||
server.update(delta, port_id=server_port_for_client)
|
||||
|
||||
# when the server updates, pass the update back to the client
|
||||
def server_callback(port_id, delta):
|
||||
print("UPDATING CLIENT")
|
||||
assert port_id == server_port_for_client
|
||||
assert server.size() == 8
|
||||
server_view.delete()
|
||||
server.delete()
|
||||
client_view.delete()
|
||||
client.delete()
|
||||
|
||||
client_view.on_update(client_callback, mode="row")
|
||||
server_view.on_update(server_callback, mode="row")
|
||||
|
||||
client.update(data, port_id=client_port)
|
||||
@@ -0,0 +1,102 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestRemove(object):
|
||||
def test_remove_all(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
tbl.remove(["abc"])
|
||||
assert tbl.view().to_records() == []
|
||||
# assert tbl.size() == 0
|
||||
|
||||
def test_remove_nonsequential(self):
|
||||
tbl = Table(
|
||||
[{"a": "abc", "b": 123}, {"a": "def", "b": 456}, {"a": "efg", "b": 789}],
|
||||
index="a",
|
||||
)
|
||||
tbl.remove(["abc", "efg"])
|
||||
assert tbl.view().to_records() == [{"a": "def", "b": 456}]
|
||||
# assert tbl.size() == 1
|
||||
|
||||
def test_remove_multiple_single(self):
|
||||
tbl = Table({"a": "integer", "b": "string"}, index="a")
|
||||
for i in range(0, 10):
|
||||
tbl.update([{"a": i, "b": str(i)}])
|
||||
for i in range(1, 10):
|
||||
tbl.remove([i])
|
||||
assert tbl.view().to_records() == [{"a": 0, "b": "0"}]
|
||||
# assert tbl.size() == 0
|
||||
|
||||
def test_remove_expressions(self):
|
||||
schema = {"key": "string", "delta$": "float", "business_line": "string"}
|
||||
data = [
|
||||
{
|
||||
"key": "A",
|
||||
"delta$": 46412.3804275,
|
||||
},
|
||||
{
|
||||
"key": "B",
|
||||
"delta$": 2317615.875,
|
||||
},
|
||||
]
|
||||
|
||||
table = Table(schema, index="key")
|
||||
table.update(data)
|
||||
table.remove(["A"])
|
||||
view = table.view(
|
||||
group_by=["business_line"],
|
||||
columns=["delta$", "alias"],
|
||||
expressions={
|
||||
"alias": '"delta$"',
|
||||
},
|
||||
)
|
||||
|
||||
records = view.to_records()
|
||||
assert records == [
|
||||
{"__ROW_PATH__": [], "delta$": 2317615.875, "alias": 2317615.875},
|
||||
{"__ROW_PATH__": [None], "delta$": 2317615.875, "alias": 2317615.875},
|
||||
]
|
||||
|
||||
def test_remove_expressions_after_view(self):
|
||||
schema = {"key": "string", "delta$": "float", "business_line": "string"}
|
||||
data = [
|
||||
{
|
||||
"key": "A",
|
||||
"delta$": 46412.3804275,
|
||||
},
|
||||
{
|
||||
"key": "B",
|
||||
"delta$": 2317615.875,
|
||||
},
|
||||
]
|
||||
|
||||
table = Table(schema, index="key")
|
||||
table.update(data)
|
||||
view = table.view(
|
||||
group_by=["business_line"],
|
||||
columns=["delta$", "alias"],
|
||||
expressions={
|
||||
"alias": '"delta$"',
|
||||
},
|
||||
)
|
||||
|
||||
table.remove(["A"])
|
||||
records = view.to_records()
|
||||
assert records == [
|
||||
{"__ROW_PATH__": [], "delta$": 2317615.875, "alias": 2317615.875},
|
||||
{"__ROW_PATH__": [None], "delta$": 2317615.875, "alias": 2317615.875},
|
||||
]
|
||||
@@ -0,0 +1,641 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from datetime import date, datetime
|
||||
import perspective
|
||||
from pytest import mark, raises
|
||||
import pytest
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestTable:
|
||||
# table constructors
|
||||
|
||||
@mark.skip
|
||||
def test_empty_table(self):
|
||||
tbl = Table([])
|
||||
assert tbl.size() == 0
|
||||
|
||||
def test_table_not_iterable(self):
|
||||
data = {"a": 1}
|
||||
with raises(TypeError):
|
||||
Table(data)
|
||||
|
||||
# def test_table_synchronous_process(self):
|
||||
# tbl = Table({"a": [1, 2, 3]})
|
||||
# assert _PerspectiveStateManager.TO_PROCESS == {}
|
||||
# tbl.update({"a": [4, 5, 6]})
|
||||
# assert _PerspectiveStateManager.TO_PROCESS == {}
|
||||
|
||||
def test_table_csv(self):
|
||||
data = "x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false"
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"x": "integer", "y": "string", "z": "boolean"}
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {
|
||||
"x": [1, 2, 3, 4],
|
||||
"y": ["a", "b", "c", "d"],
|
||||
"z": [True, False, True, False],
|
||||
}
|
||||
|
||||
def test_table_csv_with_nulls(self):
|
||||
tbl = Table("x,y\n1,")
|
||||
assert tbl.schema() == {"x": "integer", "y": "string"}
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {"x": [1], "y": [None]}
|
||||
|
||||
def test_table_csv_with_nulls_updated(self):
|
||||
tbl = Table("x,y\n1,", index="x")
|
||||
assert tbl.schema() == {"x": "integer", "y": "string"}
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {"x": [1], "y": [None]}
|
||||
tbl.update("x,y\n1,abc\n2,123")
|
||||
assert view.to_columns() == {"x": [1, 2], "y": ["abc", "123"]}
|
||||
|
||||
def test_table_correct_csv_nan_end(self):
|
||||
tbl = Table("string,integer\n,1\n,2\nabc,3")
|
||||
assert tbl.schema() == {"string": "string", "integer": "integer"}
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {
|
||||
"string": ["", "", "abc"],
|
||||
"integer": [1, 2, 3],
|
||||
}
|
||||
|
||||
def test_table_correct_csv_nan_intermittent(self):
|
||||
tbl = Table("string,float\nabc,\n,2.5\nghi,")
|
||||
assert tbl.schema() == {"string": "string", "float": "float"}
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {
|
||||
"string": ["abc", "", "ghi"],
|
||||
"float": [None, 2.5, None],
|
||||
}
|
||||
|
||||
def test_table_records_from_string_with_format_override(self):
|
||||
data = '{"x": [1,2,3], "y": [4,5,6]}'
|
||||
tbl = Table(data, format="columns")
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {
|
||||
"x": [1, 2, 3],
|
||||
"y": [4, 5, 6],
|
||||
}
|
||||
|
||||
def test_table_string_column_with_nulls_update_and_filter(self):
|
||||
tbl = Table(
|
||||
[
|
||||
{"a": "1", "b": 2, "c": "3"},
|
||||
{"a": "2", "b": 3, "c": "4"},
|
||||
{"a": "3", "b": 3, "c": None},
|
||||
],
|
||||
index="a",
|
||||
)
|
||||
view = tbl.view(filter=[["c", "==", "4"]])
|
||||
records = view.to_records()
|
||||
tbl.update([{"a": "4", "b": 10}])
|
||||
assert records == view.to_records()
|
||||
|
||||
def test_table_int(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "integer", "b": "integer"}
|
||||
|
||||
def test_table_int_column_names(self):
|
||||
data = {"a": [1, 2, 3], 0: [4, 5, 6]}
|
||||
Table(data)
|
||||
|
||||
def test_table_nones(self):
|
||||
none_data = [{"a": 1, "b": None}, {"a": None, "b": 2}]
|
||||
tbl = Table(none_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "integer", "b": "integer"}
|
||||
|
||||
def test_table_bool(self):
|
||||
bool_data = [{"a": True, "b": False}, {"a": True, "b": True}]
|
||||
tbl = Table(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "boolean", "b": "boolean"}
|
||||
|
||||
def test_table_bool_str(self):
|
||||
bool_data = [{"a": "True", "b": "False"}, {"a": "True", "b": "True"}]
|
||||
tbl = Table(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "boolean", "b": "boolean"}
|
||||
assert tbl.view().to_columns() == {"a": [True, True], "b": [False, True]}
|
||||
|
||||
def test_table_float(self):
|
||||
float_data = [{"a": 1.5, "b": 2.5}, {"a": 3.2, "b": 3.1}]
|
||||
tbl = Table(float_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "float", "b": "float"}
|
||||
|
||||
def test_table_str(self):
|
||||
str_data = [{"a": "b", "b": "b"}, {"a": "3", "b": "3"}]
|
||||
tbl = Table(str_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
|
||||
def test_table_str_with_escape(self):
|
||||
str_data = [
|
||||
{"a": 'abc"def"', "b": 'abc"def"'},
|
||||
{"a": "abc'def'", "b": "abc'def'"},
|
||||
]
|
||||
tbl = Table(str_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_records() == str_data
|
||||
|
||||
def test_table_str_unicode(self):
|
||||
str_data = [{"a": "ȀȁȀȃȀȁȀȃȀȁȀȃȀȁȀȃ", "b": "ЖДфйЖДфйЖДфйЖДфй"}]
|
||||
tbl = Table(str_data)
|
||||
assert tbl.size() == 1
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_records() == str_data
|
||||
|
||||
def test_table_date(self):
|
||||
str_data = [{"a": date.today(), "b": date.today()}]
|
||||
tbl = Table(str_data)
|
||||
assert tbl.size() == 1
|
||||
assert tbl.schema() == {"a": "date", "b": "date"}
|
||||
|
||||
def test_table_datetime(self):
|
||||
str_data = [{"a": datetime.now(), "b": datetime.now()}]
|
||||
tbl = Table(str_data)
|
||||
assert tbl.size() == 1
|
||||
assert tbl.schema() == {"a": "datetime", "b": "datetime"}
|
||||
|
||||
def test_table_columnar(self):
|
||||
data = {"a": [1, 2, 3], "b": [4, 5, 6]}
|
||||
tbl = Table(data)
|
||||
assert tbl.columns() == ["a", "b"]
|
||||
assert tbl.size() == 3
|
||||
assert tbl.schema() == {"a": "integer", "b": "integer"}
|
||||
|
||||
def test_table_columnar_mixed_length(self):
|
||||
data = [{"a": 1.5, "b": 2.5}, {"a": 3.2}]
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "float", "b": "float"}
|
||||
assert tbl.view().to_records() == [{"a": 1.5, "b": 2.5}, {"a": 3.2, "b": None}]
|
||||
|
||||
# schema
|
||||
|
||||
def test_table_schema(self):
|
||||
data = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
def test_table_readable_string_schema(self):
|
||||
data = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
def test_table_output_readable_schema(self):
|
||||
data = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
def test_table_mixed_schema(self):
|
||||
data = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
def test_table_output_string_schema(self):
|
||||
data = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
def test_table_symmetric_schema(self):
|
||||
data = {
|
||||
"a": [1, 2, 3],
|
||||
"b": [1.5, 2.5, 3.5],
|
||||
"c": ["a", "b", "c"],
|
||||
"d": [True, False, True],
|
||||
"e": [date.today(), date.today(), date.today()],
|
||||
"f": [datetime.now(), datetime.now(), datetime.now()],
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
schema = tbl.schema()
|
||||
|
||||
assert schema == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl2 = Table(schema)
|
||||
|
||||
assert tbl2.schema() == schema
|
||||
|
||||
def test_table_symmetric_string_schema(self):
|
||||
data = {
|
||||
"a": [1, 2, 3],
|
||||
"b": [1.5, 2.5, 3.5],
|
||||
"c": ["a", "b", "c"],
|
||||
"d": [True, False, True],
|
||||
"e": [date.today(), date.today(), date.today()],
|
||||
"f": [datetime.now(), datetime.now(), datetime.now()],
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
schema = tbl.schema()
|
||||
|
||||
assert schema == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
tbl2 = Table(schema)
|
||||
|
||||
assert tbl2.schema() == schema
|
||||
|
||||
def test_table_python_schema(self):
|
||||
data = {
|
||||
"a": int,
|
||||
"b": float,
|
||||
"c": str,
|
||||
"d": bool,
|
||||
"e": date,
|
||||
"f": datetime,
|
||||
}
|
||||
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "string",
|
||||
"d": "boolean",
|
||||
"e": "date",
|
||||
"f": "datetime",
|
||||
}
|
||||
|
||||
# is_valid_filter
|
||||
|
||||
# def test_table_is_valid_filter_str(self):
|
||||
# filter = ["a", "<", 1]
|
||||
# data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
# tbl = Table(data)
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# def test_table_not_is_valid_filter_str(self):
|
||||
# filter = ["a", "<", None]
|
||||
# data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
# tbl = Table(data)
|
||||
# assert tbl.is_valid_filter(filter) is False
|
||||
|
||||
# def test_table_is_valid_filter_filter_op(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_IS_NULL]
|
||||
# data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
# tbl = Table(data)
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# def test_table_not_is_valid_filter_filter_op(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, None]
|
||||
# data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
# tbl = Table(data)
|
||||
# assert tbl.is_valid_filter(filter) is False
|
||||
|
||||
# def test_table_is_valid_filter_date(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, date.today()]
|
||||
# tbl = Table({"a": "date"})
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# def test_table_not_is_valid_filter_date(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, None]
|
||||
# tbl = Table({"a": "date"})
|
||||
# assert tbl.is_valid_filter(filter) is False
|
||||
|
||||
# def test_table_is_valid_filter_datetime(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, datetime.now()]
|
||||
# tbl = Table({"a": "datetime"})
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# def test_table_not_is_valid_filter_datetime(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, None]
|
||||
# tbl = Table({"a": "datetime"})
|
||||
# assert tbl.is_valid_filter(filter) is False
|
||||
|
||||
# def test_table_is_valid_filter_datetime_str(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, "7/11/2019 5:30PM"]
|
||||
# tbl = Table({"a": "datetime"})
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# def test_table_not_is_valid_filter_datetime_str(self):
|
||||
# filter = ["a", t_filter_op.FILTER_OP_GT, None]
|
||||
# tbl = Table({"a": "datetime"})
|
||||
# assert tbl.is_valid_filter(filter) is False
|
||||
|
||||
# def test_table_is_valid_filter_ignores_not_in_schema(self):
|
||||
# filter = ["not in schema", "<", 1]
|
||||
# data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
# tbl = Table(data)
|
||||
# assert tbl.is_valid_filter(filter) is True
|
||||
|
||||
# index
|
||||
|
||||
def test_table_index(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 1, "b": 4}]
|
||||
tbl = Table(data, index="a")
|
||||
assert tbl.size() == 1
|
||||
assert tbl.view().to_records() == [{"a": 1, "b": 4}]
|
||||
|
||||
def test_table_index_from_schema(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 1, "b": 4}]
|
||||
tbl = Table({"a": "integer", "b": "integer"}, index="a")
|
||||
assert tbl.size() == 0
|
||||
tbl.update(data)
|
||||
assert tbl.view().to_records() == [{"a": 1, "b": 4}]
|
||||
|
||||
# index with None in column
|
||||
|
||||
def test_table_index_int_with_none(self):
|
||||
tbl = Table({"a": [0, 1, 2, None, None], "b": [4, 3, 2, 1, 0]}, index="a")
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [None, 0, 1, 2],
|
||||
"b": [0, 4, 3, 2],
|
||||
} # second `None` replaces first
|
||||
|
||||
def test_table_index_float_with_none(self):
|
||||
tbl = Table({"a": [0.0, 1.5, 2.5, None, None], "b": [4, 3, 2, 1, 0]}, index="a")
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [None, 0, 1.5, 2.5],
|
||||
"b": [0, 4, 3, 2],
|
||||
} # second `None` replaces first
|
||||
|
||||
def test_table_index_bool_with_none(self):
|
||||
# bools cannot be used as primary key columns
|
||||
with raises(perspective.PerspectiveError):
|
||||
Table({"a": [True, False, None, True], "b": [4, 3, 2, 1]}, index="a")
|
||||
|
||||
def test_table_index_date_with_none(self):
|
||||
tbl = Table(
|
||||
{
|
||||
"a": [date(2019, 7, 11), None, date(2019, 3, 12), date(2011, 3, 10)],
|
||||
"b": [4, 3, 2, 1],
|
||||
},
|
||||
index="a",
|
||||
)
|
||||
|
||||
def ts(x):
|
||||
return int(datetime.timestamp(x) * 1000)
|
||||
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
None,
|
||||
ts(datetime(2011, 3, 10)),
|
||||
ts(datetime(2019, 3, 12)),
|
||||
ts(datetime(2019, 7, 11)),
|
||||
],
|
||||
"b": [3, 1, 2, 4],
|
||||
}
|
||||
|
||||
def test_table_index_datetime_with_none(self, util):
|
||||
tbl = Table(
|
||||
{
|
||||
"a": [
|
||||
datetime(2019, 7, 11, 15, 30),
|
||||
None,
|
||||
datetime(2019, 7, 11, 12, 10),
|
||||
datetime(2019, 7, 11, 5, 0),
|
||||
],
|
||||
"b": [4, 3, 2, 1],
|
||||
},
|
||||
index="a",
|
||||
)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
None,
|
||||
util.to_timestamp(datetime(2019, 7, 11, 5, 0)),
|
||||
util.to_timestamp(datetime(2019, 7, 11, 12, 10)),
|
||||
util.to_timestamp(datetime(2019, 7, 11, 15, 30)),
|
||||
],
|
||||
"b": [3, 1, 2, 4],
|
||||
}
|
||||
|
||||
def test_table_index_str_with_none(self):
|
||||
tbl = Table({"a": ["", "a", None, "b"], "b": [4, 3, 2, 1]}, index="a")
|
||||
assert tbl.view().to_columns() == {"a": [None, "", "a", "b"], "b": [2, 4, 3, 1]}
|
||||
|
||||
def test_table_get_index(self):
|
||||
tbl = Table({"a": ["", "a", None, "b"], "b": [4, 3, 2, 1]}, index="a")
|
||||
assert tbl.get_index() == "a"
|
||||
|
||||
def test_table_get_index_none(self):
|
||||
tbl = Table({"a": ["", "a", None, "b"], "b": [4, 3, 2, 1]})
|
||||
assert tbl.get_index() is None
|
||||
|
||||
# limit
|
||||
|
||||
def test_table_limit(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data, limit=1)
|
||||
assert tbl.size() == 1
|
||||
assert tbl.view().to_records() == [{"a": 3, "b": 4}]
|
||||
|
||||
def test_table_get_limit(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data, limit=1)
|
||||
assert tbl.get_limit() == 1
|
||||
|
||||
def test_table_get_limit_none(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
assert tbl.get_limit() is None
|
||||
|
||||
# num_views
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_table_get_num_views(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
assert tbl.get_num_views() == 0
|
||||
v1 = tbl.view()
|
||||
v2 = tbl.view()
|
||||
v3 = tbl.view()
|
||||
assert tbl.get_num_views() == 3
|
||||
|
||||
v1.delete()
|
||||
v2.delete()
|
||||
assert tbl.get_num_views() == 1
|
||||
failed = False
|
||||
try:
|
||||
tbl.delete()
|
||||
except perspective.PerspectiveError:
|
||||
failed = True
|
||||
assert failed
|
||||
v3.delete()
|
||||
assert tbl.get_num_views() == 0
|
||||
tbl.delete()
|
||||
|
||||
# clear
|
||||
|
||||
def test_table_clear(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
tbl.clear()
|
||||
view = tbl.view()
|
||||
assert view.to_records() == []
|
||||
|
||||
# replace
|
||||
|
||||
def test_table_replace(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data2 = [{"a": 3, "b": 4}, {"a": 1, "b": 2}]
|
||||
tbl = Table(data)
|
||||
tbl.replace(data2)
|
||||
assert tbl.view().to_records() == data2
|
||||
|
||||
def test_table_replace_views_should_preserve(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data2 = [{"a": 3, "b": 4}, {"a": 1, "b": 2}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view(group_by=["a"], split_by=["b"])
|
||||
first = view.to_records()
|
||||
tbl.replace(data2)
|
||||
assert view.to_records() == first
|
||||
|
||||
def test_table_replace_should_fire_on_update(self, sentinel):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data2 = [{"a": 3, "b": 4}, {"a": 1, "b": 2}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
|
||||
s = sentinel(False)
|
||||
|
||||
def updater(port_id):
|
||||
s.set(True)
|
||||
|
||||
view.on_update(updater)
|
||||
tbl.replace(data2)
|
||||
tbl.size()
|
||||
assert s.get()
|
||||
|
||||
def test_table_replace_should_fire_on_update_with_delta(self, sentinel):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data2 = [{"a": 3, "b": 4}, {"a": 1, "b": 2}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
|
||||
s = sentinel(False)
|
||||
|
||||
def updater(port_id, delta):
|
||||
# assert port_id == 0
|
||||
table2 = Table(delta)
|
||||
assert table2.view().to_records() == data2
|
||||
s.set(True)
|
||||
|
||||
view.on_update(updater, mode="row")
|
||||
tbl.replace(data2)
|
||||
tbl.size()
|
||||
assert s.get() is True
|
||||
|
||||
def test_float32_table_construction(self):
|
||||
float_data = [{"a": 1.5, "b": 2.5}, {"a": 3.2, "b": 3.1}]
|
||||
tbl = Table(float_data, index="a")
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "float", "b": "float"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import pytest
|
||||
|
||||
pytest.main(["-vv", "-s", __file__])
|
||||
@@ -0,0 +1,500 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import os.path
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from perspective.tests.conftest import Util
|
||||
import pyarrow as pa
|
||||
from datetime import date, datetime
|
||||
import perspective as psp
|
||||
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
DATE32_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "date32.arrow")
|
||||
DATE64_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "date64.arrow")
|
||||
DICT_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "dict.arrow")
|
||||
|
||||
|
||||
names = ["a", "b", "c", "d"]
|
||||
|
||||
# Create sample data for every integer type
|
||||
ALL_INTEGERS_DATA = {
|
||||
"int8": pa.array([1, 2, 3], type=pa.int8()),
|
||||
"int16": pa.array([1000, 2000, 3000], type=pa.int16()),
|
||||
"int32": pa.array([100000, 200000, 300000], type=pa.int32()),
|
||||
"int64": pa.array([10000000000, 20000000000, 30000000000], type=pa.int64()),
|
||||
"uint8": pa.array([1, 2, 3], type=pa.uint8()),
|
||||
"uint16": pa.array([1000, 2000, 3000], type=pa.uint16()),
|
||||
"uint32": pa.array([100000, 200000, 300000], type=pa.uint32()),
|
||||
"uint64": pa.array([10000000000, 20000000000, 30000000000], type=pa.uint64()),
|
||||
"float32": pa.array([100000.0, 200000.0, 300000.0], type=pa.float32()),
|
||||
"float64": pa.array(
|
||||
[10000000000.0, 20000000000.0, 30000000000.0], type=pa.float64()
|
||||
),
|
||||
}
|
||||
|
||||
ALL_INTEGERS_TABLE = pa.Table.from_pydict(ALL_INTEGERS_DATA)
|
||||
|
||||
|
||||
class TestTableArrow(object):
|
||||
def test_table_with_integer_types(self):
|
||||
tbl = Table(ALL_INTEGERS_TABLE)
|
||||
assert tbl.size() == 3
|
||||
assert tbl.schema() == {
|
||||
"int8": "integer",
|
||||
"int16": "integer",
|
||||
"int32": "integer",
|
||||
"int64": "integer",
|
||||
"uint8": "integer",
|
||||
"uint16": "integer",
|
||||
"uint32": "integer",
|
||||
"uint64": "integer",
|
||||
"float32": "float",
|
||||
"float64": "float",
|
||||
}
|
||||
for k, values in ALL_INTEGERS_DATA.items():
|
||||
v = tbl.view(filter=[[k, "==", values[0].as_py()]])
|
||||
assert len(v.to_json()) == 1
|
||||
|
||||
def test_table_arrow_loads_date32_file(self, util: Util):
|
||||
with open(DATE32_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl = Table(file.read())
|
||||
assert tbl.schema() == {
|
||||
"jan-2019": "date",
|
||||
"feb-2020": "date",
|
||||
"mar-2019": "date",
|
||||
"apr-2020": "date",
|
||||
}
|
||||
assert tbl.size() == 31
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {
|
||||
"jan-2019": [
|
||||
util.to_timestamp(datetime(2019, 1, i)) for i in range(1, 32)
|
||||
],
|
||||
"feb-2020": [
|
||||
util.to_timestamp(datetime(2020, 2, i)) for i in range(1, 30)
|
||||
]
|
||||
+ [None, None],
|
||||
"mar-2019": [
|
||||
util.to_timestamp(datetime(2019, 3, i)) for i in range(1, 32)
|
||||
],
|
||||
"apr-2020": [
|
||||
util.to_timestamp(datetime(2020, 4, i)) for i in range(1, 31)
|
||||
]
|
||||
+ [None],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_date64_file(self, util: Util):
|
||||
with open(DATE64_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl = Table(file.read())
|
||||
assert tbl.schema() == {
|
||||
"jan-2019": "date",
|
||||
"feb-2020": "date",
|
||||
"mar-2019": "date",
|
||||
"apr-2020": "date",
|
||||
}
|
||||
assert tbl.size() == 31
|
||||
view = tbl.view()
|
||||
assert view.to_columns() == {
|
||||
"jan-2019": [
|
||||
util.to_timestamp(datetime(2019, 1, i)) for i in range(1, 32)
|
||||
],
|
||||
"feb-2020": [
|
||||
util.to_timestamp(datetime(2020, 2, i)) for i in range(1, 30)
|
||||
]
|
||||
+ [None, None],
|
||||
"mar-2019": [
|
||||
util.to_timestamp(datetime(2019, 3, i)) for i in range(1, 32)
|
||||
],
|
||||
"apr-2020": [
|
||||
util.to_timestamp(datetime(2020, 4, i)) for i in range(1, 31)
|
||||
]
|
||||
+ [None],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dict_file(self):
|
||||
with open(DICT_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl = Table(file.read())
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.size() == 5
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None, "abc"],
|
||||
"b": ["klm", "hij", None, "hij", "klm"],
|
||||
}
|
||||
|
||||
# streams
|
||||
|
||||
def test_table_arrow_loads_int_stream(self, util):
|
||||
data = [list(range(10)) for i in range(4)]
|
||||
arrow_data = util.make_arrow(names, data)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "integer",
|
||||
"c": "integer",
|
||||
"d": "integer",
|
||||
}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": data[0],
|
||||
"b": data[1],
|
||||
"c": data[2],
|
||||
"d": data[3],
|
||||
}
|
||||
|
||||
def test_empty_arrow(self, util):
|
||||
table = pa.table(
|
||||
{
|
||||
"col1": [1, 2, 3],
|
||||
"col2": ["abc", "foo", "bar"],
|
||||
}
|
||||
)
|
||||
|
||||
empty_table = table.schema.empty_table()
|
||||
assert client.table(table, name="table2").size() == 3
|
||||
assert client.table(empty_table, name="table_empty_bad").size() == 0
|
||||
assert client.table(table, name="table3").schema() == {
|
||||
"col1": "integer",
|
||||
"col2": "string",
|
||||
}
|
||||
|
||||
assert client.table(empty_table, name="table4").schema() == {
|
||||
"col1": "integer",
|
||||
"col2": "string",
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_float_stream(self, util):
|
||||
data = [[i for i in range(10)], [i * 1.5 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
}
|
||||
assert tbl.view().to_columns() == {"a": data[0], "b": data[1]}
|
||||
|
||||
def test_table_arrow_loads_decimal_stream(self, util):
|
||||
data = [[i * 1000 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.decimal128(4)])
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "float",
|
||||
}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_bool_stream(self, util):
|
||||
data = [[True if i % 2 == 0 else False for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "boolean"}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_date32_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date32()])
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_date64_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date64()])
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_timestamp_all_formats_stream(self, util):
|
||||
data = [
|
||||
[datetime(2019, 2, i, 9) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 10) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 11) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 12) for i in range(1, 11)],
|
||||
]
|
||||
arrow_data = util.make_arrow(
|
||||
names,
|
||||
data,
|
||||
types=[
|
||||
pa.timestamp("s"),
|
||||
pa.timestamp("ms"),
|
||||
pa.timestamp("us"),
|
||||
pa.timestamp("ns"),
|
||||
],
|
||||
)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "datetime",
|
||||
"b": "datetime",
|
||||
"c": "datetime",
|
||||
"d": "datetime",
|
||||
}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(i) for i in data[0]],
|
||||
"b": [util.to_timestamp(i) for i in data[1]],
|
||||
"c": [util.to_timestamp(i) for i in data[2]],
|
||||
"d": [util.to_timestamp(i) for i in data[3]],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_string_stream(self, util):
|
||||
data = [[str(i) for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.string()])
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_int8(self, util):
|
||||
data = [
|
||||
([0, 1, 1, None], ["abc", "def"]),
|
||||
([0, 1, None, 2], ["xx", "yy", "zz"]),
|
||||
]
|
||||
types = [[pa.int8(), pa.string()]] * 2
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data, types=types)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None],
|
||||
"b": ["xx", "yy", None, "zz"],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_int16(self, util):
|
||||
data = [
|
||||
([0, 1, 1, None], ["abc", "def"]),
|
||||
([0, 1, None, 2], ["xx", "yy", "zz"]),
|
||||
]
|
||||
types = [[pa.int16(), pa.string()]] * 2
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data, types=types)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None],
|
||||
"b": ["xx", "yy", None, "zz"],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_int32(self, util):
|
||||
data = [
|
||||
([0, 1, 1, None], ["abc", "def"]),
|
||||
([0, 1, None, 2], ["xx", "yy", "zz"]),
|
||||
]
|
||||
types = [[pa.int32(), pa.string()]] * 2
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data, types=types)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None],
|
||||
"b": ["xx", "yy", None, "zz"],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_int64(self, util):
|
||||
data = [
|
||||
([0, 1, 1, None], ["abc", "def"]),
|
||||
([0, 1, None, 2], ["xx", "yy", "zz"]),
|
||||
]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None],
|
||||
"b": ["xx", "yy", None, "zz"],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_nones(self, util):
|
||||
data = [([None, 0, 1, 2], ["", "abc", "def"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a"], data)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
assert tbl.view().to_columns() == {"a": [None, "", "abc", "def"]}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_nones_indexed(self, util):
|
||||
data = [
|
||||
([1, None, 0, 2], ["", "abc", "def"]),
|
||||
([2, 1, 0, None], ["", "hij", "klm"]),
|
||||
] # ["abc", None, "", "def"] # ["klm", "hij", "", None]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data, index="a") # column "a" is sorted
|
||||
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [None, "", "abc", "def"],
|
||||
"b": ["hij", "", "klm", None],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_dictionary_stream_nones_indexed_2(self, util):
|
||||
"""Test the other column, just in case."""
|
||||
data = [
|
||||
([1, None, 0, 2], ["", "abc", "def"]),
|
||||
([2, 1, 0, None], ["", "hij", "klm"]),
|
||||
] # ["abc", None, "", "def"] # ["klm", "hij", "", None]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data, index="b") # column "b" is sorted
|
||||
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["def", "", None, "abc"],
|
||||
"b": [None, "", "hij", "klm"],
|
||||
}
|
||||
|
||||
# legacy
|
||||
|
||||
def test_table_arrow_loads_int_legacy(self, util):
|
||||
data = [list(range(10)) for i in range(4)]
|
||||
arrow_data = util.make_arrow(names, data, legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "integer",
|
||||
"c": "integer",
|
||||
"d": "integer",
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_float_legacy(self, util):
|
||||
data = [[i for i in range(10)], [i * 1.5 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a", "b"], data, legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
}
|
||||
assert tbl.view().to_columns() == {"a": data[0], "b": data[1]}
|
||||
|
||||
def test_table_arrow_loads_decimal128_legacy(self, util):
|
||||
data = [[i * 1000 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.decimal128(4)], legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "float",
|
||||
}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_bool_legacy(self, util):
|
||||
data = [[True if i % 2 == 0 else False for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "boolean"}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_date32_legacy(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date32()], legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_date64_legacy(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date64()], legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_timestamp_all_formats_legacy(self, util):
|
||||
data = [
|
||||
[datetime(2019, 2, i, 9) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 10) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 11) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 12) for i in range(1, 11)],
|
||||
]
|
||||
arrow_data = util.make_arrow(
|
||||
names,
|
||||
data,
|
||||
types=[
|
||||
pa.timestamp("s"),
|
||||
pa.timestamp("ms"),
|
||||
pa.timestamp("us"),
|
||||
pa.timestamp("ns"),
|
||||
],
|
||||
legacy=True,
|
||||
)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {
|
||||
"a": "datetime",
|
||||
"b": "datetime",
|
||||
"c": "datetime",
|
||||
"d": "datetime",
|
||||
}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(i) for i in data[0]],
|
||||
"b": [util.to_timestamp(i) for i in data[1]],
|
||||
"c": [util.to_timestamp(i) for i in data[2]],
|
||||
"d": [util.to_timestamp(i) for i in data[3]],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_string_legacy(self, util):
|
||||
data = [[str(i) for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.string()], legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_table_arrow_loads_dictionary_legacy(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data, legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "string", "b": "string"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["a", "b", "b", None],
|
||||
"b": ["x", "y", None, "z"],
|
||||
}
|
||||
|
||||
def test_table_arrow_loads_arrow_from_df_with_nan(self):
|
||||
data = pd.DataFrame({"a": [1.5, 2.5, np.nan, 3.5, 4.5, np.nan, np.nan, np.nan]})
|
||||
|
||||
arrow_table = pa.Table.from_pandas(data, preserve_index=False)
|
||||
|
||||
assert arrow_table["a"].null_count == 4
|
||||
|
||||
tbl = Table(arrow_table)
|
||||
assert tbl.size() == 8
|
||||
|
||||
# check types
|
||||
assert tbl.schema() == {"a": "float"}
|
||||
|
||||
# check nans
|
||||
json = tbl.view().to_columns()
|
||||
|
||||
assert json["a"] == [1.5, 2.5, None, 3.5, 4.5, None, None, None]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,201 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from datetime import date, datetime
|
||||
from pytest import mark
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestTableInfer(object):
|
||||
def test_table_infer_int(self):
|
||||
data = {"a": [None, None, None, None, 1, 0, 1, 1, 1]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer"}
|
||||
|
||||
def test_table_infer_float(self):
|
||||
data = {"a": [None, None, None, None, 1.0, 2.0]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "float"}
|
||||
|
||||
def test_table_infer_bool(self):
|
||||
bool_data = [{"a": True, "b": False}, {"a": True, "b": True}]
|
||||
tbl = Table(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "boolean", "b": "boolean"}
|
||||
|
||||
def test_table_infer_bool_str(self):
|
||||
bool_data = [{"a": "True", "b": "False"}, {"a": "True", "b": "True"}]
|
||||
tbl = Table(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "boolean", "b": "boolean"}
|
||||
|
||||
@mark.skip(reason="Unsupported python specific behavior")
|
||||
def test_table_bool_infer_str_all_formats_from_schema(self):
|
||||
bool_data = [
|
||||
{"a": "True", "b": "False"},
|
||||
{"a": "t", "b": "f"},
|
||||
{"a": "true", "b": "false"},
|
||||
{"a": 1, "b": 0},
|
||||
{"a": "on", "b": "off"},
|
||||
]
|
||||
tbl = Table(bool_data)
|
||||
assert tbl.schema() == {"a": "boolean", "b": "boolean"}
|
||||
assert tbl.size() == 5
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [True, True, True, True, True],
|
||||
"b": [False, False, False, False, False],
|
||||
}
|
||||
|
||||
def test_table_infer_bool_variant(self):
|
||||
data = {"a": [None, None, None, None, True, True, True]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "boolean"}
|
||||
|
||||
def test_table_infer_str(self):
|
||||
data = {"a": [None, None, None, None, None, None, "abc"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
@mark.skip(reason="Time is not a valid JSON type")
|
||||
def test_table_infer_time_as_string(self):
|
||||
# time objects are inferred as string
|
||||
data = {
|
||||
"a": [
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
datetime(2019, 7, 11, 12, 30, 5).time(),
|
||||
]
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
@mark.skip(reason="Unsupported python specific behavior")
|
||||
def test_table_infer_date_from_datetime(self):
|
||||
# inferrence on non-pandas datasets defaults to datetime
|
||||
data = {"a": [None, None, None, None, None, None, datetime(2019, 7, 11)]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_date_from_date(self):
|
||||
# pass in a `date` to make sure it infers as date
|
||||
data = {"a": [None, None, None, None, None, None, date(2019, 7, 11)]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_valid_date(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/31/2019"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_ambiguous_date(self):
|
||||
data = {"a": [None, None, None, None, None, None, "01/03/2019"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_ymd_date(self):
|
||||
data = {"a": [None, None, None, None, None, None, "2019/01/03"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_invalid_date(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/55/2019"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
def test_table_infer_date_edge(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/31/2019 00:00:00"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_datetime_edge(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/31/2019 00:00:01"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_valid_datetime(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/31/2019 07:30:00"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_iso_datetime(self):
|
||||
data = {"a": [None, None, None, None, None, None, "2019/07/25T09:00:00"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_datetime_separators(self):
|
||||
data = {
|
||||
"a": [
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
"2019-07-25T09:00:00",
|
||||
"2019/07/25T09:00:00",
|
||||
]
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_datetime_tz(self):
|
||||
data = {"a": [None, None, None, None, None, "2019-07-25T09:00:00-05:00"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_infer_invalid_datetime(self):
|
||||
data = {"a": [None, None, None, None, None, None, "08/31/2019 25:30:00"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
def test_table_infer_mixed_date(self):
|
||||
data = {"a": [None, None, None, None, None, "08/11/2019"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_infer_mixed_datetime(self):
|
||||
data = {"a": [None, None, None, None, None, "08/11/2019 13:14:15"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
@mark.skip(reason="Numeric strings are not inferred as numbers")
|
||||
def test_table_strict_datetime_infer(self):
|
||||
data = {"a": ["10", "9", "8", "7", "6", "5", "4", "3", "2", "1"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
def test_table_strict_date_infer(self):
|
||||
data = {"a": ["2019 09 10"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
def test_table_strict_datetime_separator_infer(self):
|
||||
data = {"a": ["2019-10-01 7:30"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
|
||||
def test_table_datetime_infer_no_false_positive(self):
|
||||
data = {"a": [" . - / but clearly not a date"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
|
||||
@mark.skip
|
||||
def test_table_datetime_infer_from_string_with_time(self):
|
||||
data = {"a": ["11:00 ABCD"]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
@@ -0,0 +1,45 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestTableInfer(object):
|
||||
def test_table_limit_wraparound_does_not_respect_partial_none(self):
|
||||
t = Table({"a": "float", "b": "float"}, limit=3)
|
||||
t.update([{"a": 10}, {"b": 1}, {"a": 20}, {"a": None, "b": 2}])
|
||||
d1 = t.view().to_json()
|
||||
|
||||
t2 = Table({"a": "float", "b": "float"}, limit=3)
|
||||
t2.update([{"a": 10}, {"b": 1}, {"a": 20}, {"b": 2}])
|
||||
d2 = t2.view().to_json()
|
||||
|
||||
assert d1[0] == d2[0]
|
||||
assert d1[1:] == d2[1:]
|
||||
|
||||
def test_table_limit_wraparound_does_not_respect_partial(self):
|
||||
t = Table({"a": "float", "b": "float"}, limit=3)
|
||||
t.update([{"a": 10}, {"b": 1}, {"a": 20}, {"a": 10, "b": 2}])
|
||||
d1 = t.view().to_columns()
|
||||
|
||||
t2 = Table({"a": "float", "b": "float"}, limit=3)
|
||||
t2.update([{"a": 10}, {"b": 1}, {"a": 20}, {"b": 2}])
|
||||
d2 = t2.view().to_columns()
|
||||
|
||||
assert d1 != d2
|
||||
|
||||
def test_table_limit_with_json(self):
|
||||
t = Table({"a": [1, 2, 3]}, limit=1)
|
||||
assert t.size() == 1
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,304 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import perspective as psp
|
||||
from pytest import mark
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
def _perspective_temp_dirs():
|
||||
"""All `perspective_*` directories currently in the OS temp directory.
|
||||
|
||||
`BACKING_STORE_DISK` columns are written to a unique
|
||||
`<tempdir>/perspective_<uuid>/` directory.
|
||||
"""
|
||||
tmp = tempfile.gettempdir()
|
||||
try:
|
||||
entries = os.listdir(tmp)
|
||||
except OSError:
|
||||
return set()
|
||||
|
||||
print(tmp)
|
||||
return {
|
||||
os.path.join(tmp, e)
|
||||
for e in entries
|
||||
if e.startswith("perspective_") and os.path.isdir(os.path.join(tmp, e))
|
||||
}
|
||||
|
||||
|
||||
class TestTableOnDisk:
|
||||
def test_page_to_disk_schema_and_view(self):
|
||||
data = {
|
||||
"x": [1, 2, 3, 4],
|
||||
"y": ["a", "b", "c", "d"],
|
||||
"z": [True, False, True, False],
|
||||
}
|
||||
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
assert disk.schema() == mem.schema()
|
||||
assert disk.view().to_columns() == mem.view().to_columns()
|
||||
|
||||
def test_page_to_disk_csv(self):
|
||||
data = "x,y,z\n1,a,true\n2,b,false\n3,c,true\n4,d,false"
|
||||
tbl = Table(data, page_to_disk=True)
|
||||
assert tbl.schema() == {"x": "integer", "y": "string", "z": "boolean"}
|
||||
assert tbl.view().to_columns() == {
|
||||
"x": [1, 2, 3, 4],
|
||||
"y": ["a", "b", "c", "d"],
|
||||
"z": [True, False, True, False],
|
||||
}
|
||||
|
||||
def test_page_to_disk_update_indexed(self):
|
||||
mem = Table({"x": [1, 2, 3], "y": [1.0, 2.0, 3.0]}, index="x")
|
||||
disk = Table(
|
||||
{"x": [1, 2, 3], "y": [1.0, 2.0, 3.0]}, index="x", page_to_disk=True
|
||||
)
|
||||
update = {"x": [2, 4], "y": [20.0, 40.0]}
|
||||
mem.update(update)
|
||||
disk.update(update)
|
||||
assert disk.view().to_columns() == mem.view().to_columns()
|
||||
assert disk.size() == mem.size()
|
||||
|
||||
def test_page_to_disk_group_by_aggregation(self):
|
||||
data = {"g": ["a", "b", "a", "b", "a"], "v": [1, 2, 3, 4, 5]}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
config = {"group_by": ["g"], "columns": ["v"], "aggregates": {"v": "sum"}}
|
||||
assert disk.view(**config).to_columns() == mem.view(**config).to_columns()
|
||||
|
||||
def test_page_to_disk_arrow_roundtrip(self):
|
||||
data = {"x": list(range(100)), "y": [float(i) / 2 for i in range(100)]}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
assert disk.view().to_arrow() == mem.view().to_arrow()
|
||||
|
||||
def test_page_to_disk_creates_backing_files(self):
|
||||
before = _perspective_temp_dirs()
|
||||
tbl = Table({"x": [1, 2, 3], "y": ["a", "b", "c"]}, page_to_disk=True)
|
||||
|
||||
# Touch the table so it is not optimized away before we inspect the FS.
|
||||
assert tbl.size() == 3
|
||||
after = _perspective_temp_dirs()
|
||||
new_dirs = after - before
|
||||
assert new_dirs, "expected a new perspective_<uuid> directory on disk"
|
||||
assert any(os.listdir(d) for d in new_dirs), "expected column files on disk"
|
||||
|
||||
def test_memory_table_creates_no_backing_files(self):
|
||||
before = _perspective_temp_dirs()
|
||||
tbl = Table({"x": [1, 2, 3]})
|
||||
assert tbl.size() == 3
|
||||
after = _perspective_temp_dirs()
|
||||
assert after == before, "in-memory table must not write to disk"
|
||||
|
||||
def test_page_to_disk_growth_forces_resize(self):
|
||||
tbl = Table({"x": [0], "y": [0.0]}, index="x", page_to_disk=True)
|
||||
n = 50000
|
||||
tbl.update({"x": list(range(n)), "y": [float(i) for i in range(n)]})
|
||||
assert tbl.size() == n
|
||||
cols = tbl.view().to_columns()
|
||||
assert cols["x"][0] == 0
|
||||
assert cols["x"][-1] == n - 1
|
||||
assert cols["y"][-1] == float(n - 1)
|
||||
|
||||
def test_page_to_disk_clone_keeps_master_files(self):
|
||||
# Cloning a disk-backed column (e.g. the masked clone that serializing a
|
||||
# table with removed rows performs) must give the clone its OWN backing
|
||||
# file. Otherwise the clone aliases — and on teardown `rmfile`s — the
|
||||
# master's file, silently unlinking the master's named backing store.
|
||||
before = _perspective_temp_dirs()
|
||||
tbl = Table(
|
||||
{"x": [1, 2, 3, 4], "y": [10.0, 20.0, 30.0, 40.0]},
|
||||
index="x",
|
||||
page_to_disk=True,
|
||||
)
|
||||
view = tbl.view()
|
||||
new_dirs = _perspective_temp_dirs() - before
|
||||
master_dirs = [
|
||||
d
|
||||
for d in new_dirs
|
||||
if not os.path.basename(d).startswith("perspective_expr_")
|
||||
]
|
||||
assert len(master_dirs) == 1
|
||||
master_dir = master_dirs[0]
|
||||
master_files = set(os.listdir(master_dir))
|
||||
assert master_files, "expected master backing files on disk"
|
||||
|
||||
# Removing rows triggers a masked clone of the disk master columns.
|
||||
tbl.remove([2, 3])
|
||||
assert view.to_columns()["x"] == [1, 4]
|
||||
# The master's own backing files must survive the clone's teardown.
|
||||
survived = set(os.listdir(master_dir))
|
||||
assert master_files <= survived, (
|
||||
"clone teardown unlinked master backing files: {}".format(
|
||||
master_files - survived
|
||||
)
|
||||
)
|
||||
|
||||
# And the master must remain usable for subsequent updates.
|
||||
tbl.update({"x": [5, 6], "y": [50.0, 60.0]})
|
||||
assert sorted(view.to_columns()["x"]) == [1, 4, 5, 6]
|
||||
|
||||
def test_page_to_disk_larger_dataset_matches_memory(self):
|
||||
n = 20000
|
||||
data = {
|
||||
"i": list(range(n)),
|
||||
"f": [float(i) * 1.5 for i in range(n)],
|
||||
"s": ["row_{}".format(i % 97) for i in range(n)],
|
||||
}
|
||||
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
assert disk.view().to_arrow() == mem.view().to_arrow()
|
||||
|
||||
|
||||
def _perspective_expr_dirs():
|
||||
"""`perspective_expr_*` directories — the on-disk expression `m_master`."""
|
||||
tmp = tempfile.gettempdir()
|
||||
try:
|
||||
entries = os.listdir(tmp)
|
||||
except OSError:
|
||||
return set()
|
||||
|
||||
return {
|
||||
os.path.join(tmp, e)
|
||||
for e in entries
|
||||
if e.startswith("perspective_expr_") and os.path.isdir(os.path.join(tmp, e))
|
||||
}
|
||||
|
||||
|
||||
class TestTableOnDiskExpressions:
|
||||
def test_expression_numeric_equivalence(self):
|
||||
data = {"x": [1, 2, 3, 4], "y": [10.0, 20.0, 30.0, 40.0]}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
exprs = {"sum": '"x" + "y"', "prod": '"x" * "y"'}
|
||||
assert (
|
||||
disk.view(expressions=exprs).to_columns()
|
||||
== mem.view(expressions=exprs).to_columns()
|
||||
)
|
||||
|
||||
def test_expression_string_equivalence(self):
|
||||
data = {"a": ["foo", "bar", "baz"], "b": ["AA", "BB", "CC"]}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
exprs = {"up": 'upper("a")', "lo": 'lower("b")'}
|
||||
assert (
|
||||
disk.view(expressions=exprs).to_columns()
|
||||
== mem.view(expressions=exprs).to_columns()
|
||||
)
|
||||
|
||||
def test_expression_with_group_by_equivalence(self):
|
||||
data = {"g": ["a", "b", "a", "b"], "v": [1, 2, 3, 4]}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
config = {
|
||||
"expressions": {"v2": '"v" * 2'},
|
||||
"group_by": ["g"],
|
||||
"columns": ["v2"],
|
||||
"aggregates": {"v2": "sum"},
|
||||
}
|
||||
|
||||
assert disk.view(**config).to_columns() == mem.view(**config).to_columns()
|
||||
|
||||
def test_expression_master_is_page_to_disk(self):
|
||||
before = _perspective_expr_dirs()
|
||||
tbl = Table({"x": [1, 2, 3], "y": [10.0, 20.0, 30.0]}, page_to_disk=True)
|
||||
view = tbl.view(expressions={"e": '"x" + "y"'})
|
||||
assert view.to_columns()["e"] == [11.0, 22.0, 33.0]
|
||||
new_dirs = _perspective_expr_dirs() - before
|
||||
assert new_dirs, (
|
||||
"expected a perspective_expr_<uuid> dir for the expression master"
|
||||
)
|
||||
|
||||
assert any(os.listdir(d) for d in new_dirs), (
|
||||
"expected expression backing files on disk"
|
||||
)
|
||||
|
||||
def test_memory_expression_creates_no_expr_dir(self):
|
||||
before = _perspective_expr_dirs()
|
||||
tbl = Table({"x": [1, 2, 3], "y": [10.0, 20.0, 30.0]})
|
||||
view = tbl.view(expressions={"e": '"x" + "y"'})
|
||||
assert view.to_columns()["e"] == [11.0, 22.0, 33.0]
|
||||
assert _perspective_expr_dirs() == before, (
|
||||
"in-memory table must not write expression data to disk"
|
||||
)
|
||||
|
||||
def test_expression_page_to_disk_update_and_larger(self):
|
||||
n = 10000
|
||||
tbl = Table({"x": [0], "y": [0.0]}, index="x", page_to_disk=True)
|
||||
tbl.update({"x": list(range(n)), "y": [float(i) for i in range(n)]})
|
||||
cols = tbl.view(expressions={"e": '"x" + "y"'}).to_columns()
|
||||
assert cols["e"][0] == 0.0
|
||||
assert cols["e"][-1] == float((n - 1) + (n - 1))
|
||||
|
||||
|
||||
class TestResidency:
|
||||
# The residency manager evicts disk-backed column buffers to their files
|
||||
# when over the `PSP_MEMORY_BUDGET` and restores them transparently on
|
||||
# access. Under a tiny budget, eviction fires aggressively and data must
|
||||
# still round-trip identically to an in-memory table.
|
||||
|
||||
@mark.skip(reason="No secret hooks in the engine")
|
||||
def test_residency_evicts_and_data_is_correct(self):
|
||||
stats_fd, stats_path = tempfile.mkstemp(prefix="psp_residency_")
|
||||
os.close(stats_fd)
|
||||
os.environ["PSP_MEMORY_BUDGET"] = "1024"
|
||||
os.environ["PSP_RESIDENCY_STATS_FILE"] = stats_path
|
||||
try:
|
||||
n = 5000
|
||||
data = {
|
||||
"x": list(range(n)),
|
||||
"y": [float(i) * 1.5 for i in range(n)],
|
||||
"s": ["row_{}".format(i % 50) for i in range(n)],
|
||||
}
|
||||
mem = Table(data)
|
||||
disk = Table(data, page_to_disk=True)
|
||||
|
||||
# Each request is a safepoint that trims to budget; data read back
|
||||
# must be correct (round-tripped through evict -> restore).
|
||||
assert disk.view().to_columns() == mem.view().to_columns()
|
||||
assert disk.view().to_arrow() == mem.view().to_arrow()
|
||||
|
||||
upd = {
|
||||
"x": list(range(n, n + 200)),
|
||||
"y": [float(i) for i in range(n, n + 200)],
|
||||
"s": ["upd_{}".format(i) for i in range(200)],
|
||||
}
|
||||
mem.update(upd)
|
||||
disk.update(upd)
|
||||
assert disk.view().to_columns() == mem.view().to_columns()
|
||||
|
||||
# An expression view on an evicted disk table must still compute.
|
||||
exprs = {"e": '"x" + "y"'}
|
||||
assert (
|
||||
disk.view(expressions=exprs).to_columns()
|
||||
== mem.view(expressions=exprs).to_columns()
|
||||
)
|
||||
|
||||
# Confirm eviction actually occurred (otherwise the test is vacuous).
|
||||
with open(stats_path) as f:
|
||||
stats = f.read()
|
||||
evictions = int(stats.split("evictions=")[1].split()[0])
|
||||
assert evictions > 0, "expected evictions under a tiny budget: " + stats
|
||||
finally:
|
||||
os.environ.pop("PSP_MEMORY_BUDGET", None)
|
||||
os.environ.pop("PSP_RESIDENCY_STATS_FILE", None)
|
||||
os.remove(stats_path)
|
||||
# Drain a safepoint with residency disabled so any evicted stores
|
||||
# from other live tables are restored before subsequent tests.
|
||||
Table({"_": [0]}).view().to_columns()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,251 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from datetime import date, datetime
|
||||
import numpy as np
|
||||
import polars as pl
|
||||
from pytest import mark
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
def arrow_bytes_to_polars(view):
|
||||
import pyarrow
|
||||
|
||||
with pyarrow.ipc.open_stream(pyarrow.BufferReader(view.to_arrow())) as reader:
|
||||
return pl.from_dataframe(reader.read_pandas())
|
||||
|
||||
|
||||
class TestTablePolars(object):
|
||||
def test_empty_table(self):
|
||||
tbl = Table([])
|
||||
assert tbl.size() == 0
|
||||
assert tbl.schema() == {}
|
||||
|
||||
def test_table_dataframe(self):
|
||||
d = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data = pl.DataFrame(d)
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "integer", "b": "integer"}
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": 1, "b": 2},
|
||||
{"a": 3, "b": 4},
|
||||
]
|
||||
|
||||
def test_table_lazyframe(self):
|
||||
d = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
data = pl.DataFrame(d).lazy()
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.schema() == {"a": "integer", "b": "integer"}
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": 1, "b": 2},
|
||||
{"a": 3, "b": 4},
|
||||
]
|
||||
|
||||
def test_table_dataframe_column_order(self):
|
||||
d = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}]
|
||||
data = pl.DataFrame(d).select(["b", "c", "a", "d"])
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.columns() == ["b", "c", "a", "d"]
|
||||
|
||||
def test_table_dataframe_selective_column_order(self):
|
||||
d = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}]
|
||||
data = pl.DataFrame(d).select(["b", "c", "a"])
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.columns() == ["b", "c", "a"]
|
||||
|
||||
def test_table_dataframe_does_not_mutate(self):
|
||||
# make sure we don't mutate the dataframe that a user passes in
|
||||
data = pl.DataFrame(
|
||||
{
|
||||
"a": [None, 1, None, 2],
|
||||
"b": [1.5, None, 2.5, None],
|
||||
}
|
||||
)
|
||||
assert data["a"].to_list() == [None, 1, None, 2]
|
||||
assert data["b"].to_list() == [1.5, None, 2.5, None]
|
||||
|
||||
tbl = Table(data)
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
|
||||
assert data["a"].to_list() == [None, 1, None, 2]
|
||||
assert data["b"].to_list() == [1.5, None, 2.5, None]
|
||||
|
||||
def test_table_polars_from_schema_int(self):
|
||||
data = [None, 1, None, 2, None, 3, 4]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "integer"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == data
|
||||
|
||||
def test_table_polars_from_schema_bool(self):
|
||||
data = [True, False, True, False]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "boolean"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == data
|
||||
|
||||
def test_table_polars_from_schema_float(self):
|
||||
data = [None, 1.5, None, 2.5, None, 3.5, 4.5]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "float"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == data
|
||||
|
||||
def test_table_polars_from_schema_float_all_nan(self):
|
||||
data = [np.nan, np.nan, np.nan, np.nan]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "float"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == [None, None, None, None]
|
||||
|
||||
def test_table_polars_from_schema_float_to_int(self):
|
||||
data = [None, 1.5, None, 2.5, None, 3.5, 4.5]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "integer"})
|
||||
table.update(df)
|
||||
# truncates decimal
|
||||
assert table.view().to_columns()["a"] == [None, 1, None, 2, None, 3, 4]
|
||||
|
||||
def test_table_polars_from_schema_int_to_float(self):
|
||||
data = [None, 1, None, 2, None, 3, 4]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "float"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == [None, 1.0, None, 2.0, None, 3.0, 4.0]
|
||||
|
||||
def test_table_polars_from_schema_date(self, util):
|
||||
data = [date(2019, 8, 15), None, date(2019, 8, 16)]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "date"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == [
|
||||
util.to_timestamp(datetime(2019, 8, 15)),
|
||||
None,
|
||||
util.to_timestamp(datetime(2019, 8, 16)),
|
||||
]
|
||||
|
||||
def test_table_polars_from_schema_str(self):
|
||||
data = ["a", None, "b", None, "c"]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table({"a": "string"})
|
||||
table.update(df)
|
||||
assert table.view().to_columns()["a"] == data
|
||||
|
||||
def test_table_polars_none(self):
|
||||
data = [None, None, None]
|
||||
df = pl.DataFrame({"a": data})
|
||||
table = Table(df)
|
||||
assert table.view().to_columns()["a"] == data
|
||||
|
||||
def test_table_polars_symmetric_table(self):
|
||||
# make sure that updates are symmetric to table creation
|
||||
df = pl.DataFrame({"a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5]})
|
||||
t1 = Table(df)
|
||||
t2 = Table({"a": "integer", "b": "float"})
|
||||
t2.update(df)
|
||||
assert t1.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": [1.5, 2.5, 3.5, 4.5],
|
||||
}
|
||||
|
||||
def test_table_polars_symmetric_stacked_updates(self):
|
||||
# make sure that updates are symmetric to table creation
|
||||
df = pl.DataFrame({"a": [1, 2, 3, 4], "b": [1.5, 2.5, 3.5, 4.5]})
|
||||
|
||||
t1 = Table(df)
|
||||
t1.update(df)
|
||||
|
||||
t2 = Table({"a": "integer", "b": "float"})
|
||||
t2.update(df)
|
||||
t2.update(df)
|
||||
|
||||
assert t1.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4, 1, 2, 3, 4],
|
||||
"b": [1.5, 2.5, 3.5, 4.5, 1.5, 2.5, 3.5, 4.5],
|
||||
}
|
||||
|
||||
@mark.skip(reason="Not supported, polars doesnt like input")
|
||||
def test_table_polars_transitive(self):
|
||||
# serialized output -> table -> serialized output
|
||||
records = {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": [1.5, 2.5, 3.5, 4.5],
|
||||
"c": [np.nan, np.nan, "abc", np.nan],
|
||||
"d": [None, True, None, False],
|
||||
"e": [
|
||||
float("nan"),
|
||||
datetime(2019, 7, 11, 12, 30),
|
||||
float("nan"),
|
||||
datetime(2019, 7, 11, 12, 30),
|
||||
],
|
||||
}
|
||||
|
||||
df = pl.DataFrame(records, strict=False)
|
||||
t1 = Table(df)
|
||||
out1 = arrow_bytes_to_polars(t1.view(columns=["a", "b", "c", "d", "e"]))
|
||||
t2 = Table(out1)
|
||||
assert t1.schema() == t2.schema()
|
||||
out2 = t2.view().to_columns()
|
||||
assert t1.view().to_columns() == out2
|
||||
|
||||
# dtype=object should have correct inferred types
|
||||
|
||||
def test_table_polars_object_to_int(self):
|
||||
df = pl.DataFrame({"a": [1, 2, None, 2, None, 3, 4]})
|
||||
table = Table(df)
|
||||
assert table.schema() == {"a": "integer"}
|
||||
assert table.view().to_columns()["a"] == [1, 2, None, 2, None, 3, 4]
|
||||
|
||||
def test_table_polars_object_to_float(self):
|
||||
df = pl.DataFrame({"a": [None, 1, None, 2, None, 3, 4]})
|
||||
table = Table(df)
|
||||
assert table.schema() == {"a": "integer"}
|
||||
assert table.view().to_columns()["a"] == [None, 1.0, None, 2.0, None, 3.0, 4.0]
|
||||
|
||||
def test_table_polars_object_to_bool(self):
|
||||
df = pl.DataFrame({"a": [True, False, True, False, True, False]})
|
||||
table = Table(df)
|
||||
assert table.schema() == {"a": "boolean"}
|
||||
assert table.view().to_columns()["a"] == [True, False, True, False, True, False]
|
||||
|
||||
def test_table_polars_object_to_datetime(self):
|
||||
df = pl.DataFrame(
|
||||
{
|
||||
"a": [
|
||||
datetime(2019, 7, 11, 1, 2, 3),
|
||||
datetime(2019, 7, 12, 1, 2, 3),
|
||||
None,
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
table = Table(df)
|
||||
assert table.schema() == {"a": "datetime"}
|
||||
assert table.view().to_columns()["a"] == [
|
||||
datetime(2019, 7, 11, 1, 2, 3).timestamp() * 1000,
|
||||
datetime(2019, 7, 12, 1, 2, 3).timestamp() * 1000,
|
||||
None,
|
||||
]
|
||||
|
||||
def test_table_polars_object_to_str(self):
|
||||
df = pl.DataFrame({"a": np.array(["abc", "def", None, "ghi"], dtype=object)})
|
||||
table = Table(df)
|
||||
assert table.schema() == {"a": "string"}
|
||||
assert table.view().to_columns()["a"] == ["abc", "def", None, "ghi"]
|
||||
@@ -0,0 +1,130 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from perspective import PerspectiveError
|
||||
from datetime import date, datetime
|
||||
from pytest import approx, mark, raises
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
def date_timestamp(date):
|
||||
return int(datetime.combine(date, datetime.min.time()).timestamp()) * 1000
|
||||
|
||||
|
||||
def compare_delta(received, expected):
|
||||
"""Compare an arrow-serialized row delta by constructing a Table."""
|
||||
tbl = Table(received)
|
||||
assert tbl.view().to_columns() == expected
|
||||
|
||||
|
||||
class TestView(object):
|
||||
def test_view_zero(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
tbl2 = Table(view)
|
||||
view2 = tbl2.view()
|
||||
dimms = view2.dimensions()
|
||||
assert dimms["num_view_rows"] == 2
|
||||
assert dimms["num_view_columns"] == 2
|
||||
assert view2.schema() == {"a": "integer", "b": "integer"}
|
||||
assert view2.to_records() == data
|
||||
|
||||
def test_view_one(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view(group_by=["a"])
|
||||
tbl2 = Table(view)
|
||||
view2 = tbl2.view()
|
||||
|
||||
dimms = view2.dimensions()
|
||||
assert dimms["num_view_rows"] == 3
|
||||
assert dimms["num_view_columns"] == 3
|
||||
assert view2.schema() == {
|
||||
"a (Group by 1)": "integer",
|
||||
"a": "integer",
|
||||
"b": "integer",
|
||||
}
|
||||
|
||||
assert view2.to_records() == [
|
||||
{"a (Group by 1)": None, "a": 4, "b": 6},
|
||||
{"a (Group by 1)": 1, "a": 1, "b": 2},
|
||||
{"a (Group by 1)": 3, "a": 3, "b": 4},
|
||||
]
|
||||
|
||||
def test_view_two(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view(group_by=["a"], split_by=["b"])
|
||||
tbl2 = Table(view)
|
||||
view2 = tbl2.view()
|
||||
dimms = view2.dimensions()
|
||||
assert dimms["num_view_rows"] == 3
|
||||
assert dimms["num_view_columns"] == 5
|
||||
assert view2.schema() == {
|
||||
"a (Group by 1)": "integer",
|
||||
"2|a": "integer",
|
||||
"2|b": "integer",
|
||||
"4|a": "integer",
|
||||
"4|b": "integer",
|
||||
}
|
||||
|
||||
assert view2.to_records() == [
|
||||
{
|
||||
"a (Group by 1)": None,
|
||||
"2|a": 1,
|
||||
"2|b": 2,
|
||||
"4|a": 3,
|
||||
"4|b": 4,
|
||||
},
|
||||
{
|
||||
"a (Group by 1)": 1,
|
||||
"2|a": 1,
|
||||
"2|b": 2,
|
||||
"4|a": None,
|
||||
"4|b": None,
|
||||
},
|
||||
{
|
||||
"a (Group by 1)": 3,
|
||||
"2|a": None,
|
||||
"2|b": None,
|
||||
"4|a": 3,
|
||||
"4|b": 4,
|
||||
},
|
||||
]
|
||||
|
||||
def test_view_two_column_only(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view(split_by=["b"])
|
||||
tbl2 = Table(view)
|
||||
view2 = tbl2.view()
|
||||
dimms = view2.dimensions()
|
||||
assert dimms["num_view_rows"] == 2
|
||||
assert dimms["num_view_columns"] == 4
|
||||
assert view2.schema() == {
|
||||
"2|a": "integer",
|
||||
"2|b": "integer",
|
||||
"4|a": "integer",
|
||||
"4|b": "integer",
|
||||
}
|
||||
|
||||
assert view2.to_records() == [
|
||||
{"2|a": 1, "2|b": 2, "4|a": None, "4|b": None},
|
||||
{"2|a": None, "2|b": None, "4|a": 3, "4|b": 4},
|
||||
]
|
||||
@@ -0,0 +1,417 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import pyarrow as pa
|
||||
from datetime import date, datetime
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestToArrow(object):
|
||||
def test_to_arrow_nones_symmetric(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_big_numbers_symmetric(self):
|
||||
data = {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": [
|
||||
1.7976931348623157e308,
|
||||
1.7976931348623157e308,
|
||||
1.7976931348623157e308,
|
||||
1.7976931348623157e308,
|
||||
],
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_boolean_symmetric(self):
|
||||
data = {"a": [True, False, None, False, True, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "boolean"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_str_symmetric(self):
|
||||
data = {"a": ["a", "b", "c", "d", "e", None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_str_dict(self):
|
||||
data = {
|
||||
"a": ["abcdefg", "abcdefg", "h"],
|
||||
"b": ["aaa", "bbb", "bbb"],
|
||||
"c": ["hello", "world", "world"],
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "string", "b": "string", "c": "string"}
|
||||
arr = tbl.view().to_arrow()
|
||||
|
||||
# assert that we are actually generating dict arrays
|
||||
buf = pa.BufferReader(arr)
|
||||
reader = pa.ipc.open_stream(buf)
|
||||
arrow_table = reader.read_all()
|
||||
arrow_schema = arrow_table.schema
|
||||
|
||||
for name in ("a", "b", "c"):
|
||||
arrow_type = arrow_schema.field(name).type
|
||||
assert pa.types.is_dictionary(arrow_type)
|
||||
|
||||
# assert that data is symmetrical
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_date_symmetric(self):
|
||||
# data = {"a": [date(2019, 7, 11), date(2016, 2, 29), date(2019, 12, 10)]}
|
||||
# tbl = Table(data)
|
||||
tbl = Table({"a": "date"})
|
||||
# data = {"a": map(lambda x: x.getTime(), [date(2019, 7, 11), date(2016, 2, 29), date(2019, 12, 10)])}
|
||||
data = {"a": [date(2019, 7, 11), date(2016, 2, 29), date(2019, 12, 10)]}
|
||||
tbl.update(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
|
||||
def ts(x):
|
||||
return int(datetime.timestamp(x) * 1000)
|
||||
|
||||
assert tbl2.schema() == tbl.schema()
|
||||
assert tbl2.view().to_columns() == {
|
||||
"a": [
|
||||
ts(datetime(2019, 7, 11)),
|
||||
ts(datetime(2016, 2, 29)),
|
||||
ts(datetime(2019, 12, 10)),
|
||||
]
|
||||
}
|
||||
|
||||
def test_to_arrow_date_symmetric_january(self, util):
|
||||
data = {"a": [date(2019, 1, 1), date(2016, 1, 1), date(2019, 1, 1)]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.schema() == tbl.schema()
|
||||
assert tbl2.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(x)
|
||||
for x in [
|
||||
datetime(2019, 1, 1),
|
||||
datetime(2016, 1, 1),
|
||||
datetime(2019, 1, 1),
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
def test_to_arrow_datetime_symmetric(self, util):
|
||||
data = {
|
||||
"a": [
|
||||
datetime(2019, 7, 11, 12, 30),
|
||||
datetime(2016, 2, 29, 11, 0),
|
||||
datetime(2019, 12, 10, 12, 0),
|
||||
]
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "datetime"}
|
||||
arr = tbl.view().to_arrow()
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.schema() == tbl.schema()
|
||||
assert tbl2.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(x)
|
||||
for x in [
|
||||
datetime(2019, 7, 11, 12, 30),
|
||||
datetime(2016, 2, 29, 11, 0),
|
||||
datetime(2019, 12, 10, 12, 0),
|
||||
]
|
||||
]
|
||||
}
|
||||
|
||||
def test_to_arrow_one_symmetric(self):
|
||||
data = {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": ["a", "b", "c", "d"],
|
||||
"c": [
|
||||
datetime(2019, 7, 11, 12, 0),
|
||||
datetime(2019, 7, 11, 12, 10),
|
||||
datetime(2019, 7, 11, 12, 20),
|
||||
datetime(2019, 7, 11, 12, 30),
|
||||
],
|
||||
}
|
||||
tbl = Table(data)
|
||||
view = tbl.view(group_by=["a"])
|
||||
arrow = view.to_arrow()
|
||||
tbl2 = Table(arrow)
|
||||
assert tbl2.schema() == {
|
||||
"a (Group by 1)": "integer",
|
||||
"a": "integer",
|
||||
"b": "integer",
|
||||
"c": "integer",
|
||||
}
|
||||
d = view.to_columns()
|
||||
d["a (Group by 1)"] = [
|
||||
x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")
|
||||
]
|
||||
assert tbl2.view().to_columns() == d
|
||||
|
||||
def test_to_arrow_two_symmetric(self):
|
||||
data = {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": ["hello", "world", "hello2", "world2"],
|
||||
"c": [datetime(2019, 7, 11, 12, i) for i in range(0, 40, 10)],
|
||||
}
|
||||
tbl = Table(data)
|
||||
view = tbl.view(group_by=["a"], split_by=["b"])
|
||||
arrow = view.to_arrow()
|
||||
tbl2 = Table(arrow)
|
||||
assert tbl2.schema() == {
|
||||
"a (Group by 1)": "integer",
|
||||
"hello|a": "integer",
|
||||
"hello|b": "integer",
|
||||
"hello|c": "integer",
|
||||
"world|a": "integer",
|
||||
"world|b": "integer",
|
||||
"world|c": "integer",
|
||||
"hello2|a": "integer",
|
||||
"hello2|b": "integer",
|
||||
"hello2|c": "integer",
|
||||
"world2|a": "integer",
|
||||
"world2|b": "integer",
|
||||
"world2|c": "integer",
|
||||
}
|
||||
d = view.to_columns()
|
||||
d["a (Group by 1)"] = [
|
||||
x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")
|
||||
]
|
||||
assert tbl2.view().to_columns() == d
|
||||
|
||||
def test_to_arrow_column_only_symmetric(self):
|
||||
data = {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": ["a", "b", "c", "d"],
|
||||
"c": [datetime(2019, 7, 11, 12, i) for i in range(0, 40, 10)],
|
||||
}
|
||||
tbl = Table(data)
|
||||
view = tbl.view(split_by=["a"])
|
||||
arrow = view.to_arrow()
|
||||
tbl2 = Table(arrow)
|
||||
assert tbl2.schema() == {
|
||||
"1|a": "integer",
|
||||
"1|b": "string",
|
||||
"1|c": "datetime",
|
||||
"2|a": "integer",
|
||||
"2|b": "string",
|
||||
"2|c": "datetime",
|
||||
"3|a": "integer",
|
||||
"3|b": "string",
|
||||
"3|c": "datetime",
|
||||
"4|a": "integer",
|
||||
"4|b": "string",
|
||||
"4|c": "datetime",
|
||||
}
|
||||
d = view.to_columns()
|
||||
assert tbl2.view().to_columns() == d
|
||||
|
||||
# start and end row
|
||||
def test_to_arrow_start_row(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_row=3)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"a": data["a"][3:], "b": data["b"][3:]}
|
||||
|
||||
def test_to_arrow_end_row(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(end_row=2)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"a": data["a"][:2], "b": data["b"][:2]}
|
||||
|
||||
def test_to_arrow_start_end_row(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_row=2, end_row=3)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"a": data["a"][2:3], "b": data["b"][2:3]}
|
||||
|
||||
def test_to_arrow_start_end_row_equiv(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_row=2, end_row=2)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"a": [], "b": []}
|
||||
|
||||
def test_to_arrow_start_row_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_row=-1)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_end_row_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(end_row=6)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_start_end_row_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_row=-1, end_row=6)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_start_col(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=1)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"b": data["b"]}
|
||||
|
||||
def test_to_arrow_end_col(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(end_col=1)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"a": data["a"]}
|
||||
|
||||
def test_to_arrow_start_end_col(self):
|
||||
data = {
|
||||
"a": [None, 1, None, 2, 3],
|
||||
"b": [1.5, 2.5, None, 3.5, None],
|
||||
"c": [None, 1, None, 2, 3],
|
||||
"d": [1.5, 2.5, None, 3.5, None],
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "integer",
|
||||
"d": "float",
|
||||
}
|
||||
arr = tbl.view().to_arrow(start_col=1, end_col=3)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {"b": data["b"], "c": data["c"]}
|
||||
|
||||
def test_to_arrow_start_col_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=-1)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_end_col_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(end_col=6)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_start_end_col_invalid(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=-1, end_col=6)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == data
|
||||
|
||||
def test_to_arrow_start_end_col_equiv_row(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=1, end_col=1, start_row=2, end_row=3)
|
||||
tbl2 = Table(arr)
|
||||
# start/end col is a range - thus start=end provides no columns
|
||||
assert tbl2.view().to_columns() == {}
|
||||
|
||||
def test_to_arrow_start_end_col_equiv(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=1, end_col=1)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == {}
|
||||
|
||||
def test_to_arrow_start_end_row_end_col(self):
|
||||
data = {"a": [None, 1, None, 2, 3], "b": [1.5, 2.5, None, 3.5, None]}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
arr = tbl.view().to_arrow(end_col=1, start_row=2, end_row=3)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == tbl.view().to_columns(
|
||||
end_col=1, start_row=2, end_row=3
|
||||
)
|
||||
|
||||
def test_to_arrow_start_end_col_start_row(self):
|
||||
data = {
|
||||
"a": [None, 1, None, 2, 3],
|
||||
"b": [1.5, 2.5, None, 3.5, None],
|
||||
"c": [1.5, 2.5, None, 4.5, None],
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float", "c": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=1, end_col=2, start_row=2)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == tbl.view().to_columns(
|
||||
start_col=1, end_col=2, start_row=2
|
||||
)
|
||||
|
||||
def test_to_arrow_start_end_col_end_row(self):
|
||||
data = {
|
||||
"a": [None, 1, None, 2, 3],
|
||||
"b": [1.5, 2.5, None, 3.5, None],
|
||||
"c": [1.5, 2.5, None, 4.5, None],
|
||||
}
|
||||
tbl = Table(data)
|
||||
assert tbl.schema() == {"a": "integer", "b": "float", "c": "float"}
|
||||
arr = tbl.view().to_arrow(start_col=1, end_col=2, end_row=2)
|
||||
tbl2 = Table(arr)
|
||||
assert tbl2.view().to_columns() == tbl.view().to_columns(
|
||||
start_col=1, end_col=2, end_row=2
|
||||
)
|
||||
|
||||
def test_to_arrow_one_mean(self):
|
||||
data = {"a": [1, 2, 3, 4], "b": ["a", "a", "b", "b"]}
|
||||
|
||||
table = Table(data)
|
||||
view = table.view(group_by=["b"], columns=["a"], aggregates={"a": "mean"})
|
||||
arrow = view.to_arrow()
|
||||
|
||||
table2 = Table(arrow)
|
||||
view2 = table2.view()
|
||||
result = view2.to_columns()
|
||||
|
||||
assert result == {"b (Group by 1)": [None, "a", "b"], "a": [2.5, 1.5, 3.5]}
|
||||
@@ -0,0 +1,32 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestToArrowLZ4(object):
|
||||
def test_to_arrow_lz4_roundtrip(self, superstore):
|
||||
original_tbl = Table(superstore.to_dict(orient="records"))
|
||||
arrow_uncompressed = original_tbl.view().to_arrow(compression=None)
|
||||
|
||||
tbl = Table(arrow_uncompressed)
|
||||
arr = tbl.view().to_arrow(compression="lz4")
|
||||
assert len(arr) < len(arrow_uncompressed)
|
||||
tbl2 = Table(arr)
|
||||
arr2 = tbl2.view().to_arrow(compression=None)
|
||||
assert len(arr2) > len(arr)
|
||||
tbl3 = Table(arr)
|
||||
arr3 = tbl3.view().to_arrow(compression="lz4")
|
||||
assert len(arr3) == len(arr)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestToPolars(object):
|
||||
def test_to_polars(self, superstore):
|
||||
original_tbl = Table(superstore.to_dict(orient="records"))
|
||||
df = original_tbl.view().to_polars()
|
||||
tbl = Table(df)
|
||||
assert tbl.size() == original_tbl.size()
|
||||
df2 = tbl.view().to_polars()
|
||||
assert df.equals(df2)
|
||||
@@ -0,0 +1,536 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import numpy as np
|
||||
import pyarrow as pa
|
||||
import pandas as pd
|
||||
from datetime import date, datetime
|
||||
from pytest import mark
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestUpdate(object):
|
||||
@mark.skip(reason="Fix for null values update yet to be implemented")
|
||||
def test_update_with_missing_or_null_values(self):
|
||||
tbl = Table([{"a": "1", "b": "2"}, {"a": "3", "b": "4"}], index="a")
|
||||
tbl.update([{"a": "1", "b": None}])
|
||||
assert tbl.view().to_records() == [{"a": "1", "b": None}, {"a": "3", "b": "4"}]
|
||||
|
||||
data = pd.DataFrame({"a": ["1"], "b": [None]})
|
||||
|
||||
arrow_table = pa.Table.from_pandas(data, preserve_index=False)
|
||||
|
||||
tbl.update(arrow_table)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.view().to_records() == [{"a": "1", "b": ""}, {"a": "3", "b": "4"}]
|
||||
|
||||
def test_update_from_schema(self):
|
||||
tbl = Table({"a": "string", "b": "integer"})
|
||||
tbl.update([{"a": "abc", "b": 123}])
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 123}]
|
||||
|
||||
def test_update_columnar_from_schema(self):
|
||||
tbl = Table({"a": "string", "b": "integer"})
|
||||
tbl.update({"a": ["abc"], "b": [123]})
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 123}]
|
||||
|
||||
def test_update_csv(self):
|
||||
tbl = Table({"a": "string", "b": "integer"})
|
||||
|
||||
view = tbl.view()
|
||||
tbl.update("a,b\nxyz,123\ndef,100000000")
|
||||
|
||||
assert view.to_columns() == {"a": ["xyz", "def"], "b": [123, 100000000]}
|
||||
|
||||
def test_update_csv_indexed(self):
|
||||
tbl = Table({"a": "string", "b": "float"}, index="a")
|
||||
|
||||
view = tbl.view()
|
||||
tbl.update("a,b\nxyz,1.23456718\ndef,100000000.1")
|
||||
|
||||
assert view.to_columns() == {
|
||||
"a": ["def", "xyz"],
|
||||
"b": [100000000.1, 1.23456718],
|
||||
}
|
||||
|
||||
tbl.update("a,b\nxyz,0.00000001\ndef,1234.5678\nefg,100.2")
|
||||
|
||||
assert view.to_columns() == {
|
||||
"a": ["def", "efg", "xyz"],
|
||||
"b": [1234.5678, 100.2, 0.00000001],
|
||||
}
|
||||
|
||||
def test_update_append(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}])
|
||||
tbl.update([{"a": "def", "b": 456}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": 123},
|
||||
{"a": "def", "b": 456},
|
||||
]
|
||||
|
||||
def test_update_partial(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
tbl.update([{"a": "abc", "b": 456}])
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
def test_update_partial_add_row(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
tbl.update([{"a": "abc", "b": 456}, {"a": "def"}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": 456},
|
||||
{"a": "def", "b": None},
|
||||
]
|
||||
|
||||
def test_update_partial_noop(self):
|
||||
tbl = Table([{"a": "abc", "b": 1, "c": 2}], index="a")
|
||||
tbl.update([{"a": "abc", "b": 456}])
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 456, "c": 2}]
|
||||
|
||||
def test_update_partial_unset(self):
|
||||
tbl = Table(
|
||||
[{"a": "abc", "b": 1, "c": 2}, {"a": "def", "b": 3, "c": 4}], index="a"
|
||||
)
|
||||
tbl.update([{"a": "abc"}, {"a": "def", "c": None}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": 1, "c": 2},
|
||||
{"a": "def", "b": 3, "c": None},
|
||||
]
|
||||
|
||||
def test_update_columnar_partial_add_row(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
|
||||
tbl.update({"a": ["abc", "def"], "b": [456, None]})
|
||||
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": 456},
|
||||
{"a": "def", "b": None},
|
||||
]
|
||||
|
||||
def test_update_columnar_partial_noop(self):
|
||||
tbl = Table([{"a": "abc", "b": 1, "c": 2}], index="a")
|
||||
|
||||
# no-op because "c" is not in the update dataset
|
||||
tbl.update({"a": ["abc"], "b": [456]})
|
||||
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 456, "c": 2}]
|
||||
|
||||
def test_update_columnar_partial_unset(self):
|
||||
tbl = Table(
|
||||
[{"a": "abc", "b": 1, "c": 2}, {"a": "def", "b": 3, "c": 4}], index="a"
|
||||
)
|
||||
|
||||
tbl.update({"a": ["abc"], "b": [None]})
|
||||
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": None, "c": 2},
|
||||
{"a": "def", "b": 3, "c": 4},
|
||||
]
|
||||
|
||||
def test_update_partial_subcolumn(self):
|
||||
tbl = Table([{"a": "abc", "b": 123, "c": 456}], index="a")
|
||||
tbl.update([{"a": "abc", "c": 1234}])
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 123, "c": 1234}]
|
||||
|
||||
def test_update_partial_subcolumn_dict(self):
|
||||
tbl = Table([{"a": "abc", "b": 123, "c": 456}], index="a")
|
||||
tbl.update({"a": ["abc"], "c": [1234]})
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 123, "c": 1234}]
|
||||
|
||||
def test_update_partial_cols_more_columns_than_table(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
tbl.update([{"a": "abc", "b": 456, "c": 789}])
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
def test_update_columnar_append(self):
|
||||
tbl = Table({"a": ["abc"], "b": [123]})
|
||||
tbl.update({"a": ["def"], "b": [456]})
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": "abc", "b": 123},
|
||||
{"a": "def", "b": 456},
|
||||
]
|
||||
|
||||
def test_update_columnar_partial(self):
|
||||
tbl = Table({"a": ["abc"], "b": [123]}, index="a")
|
||||
tbl.update({"a": ["abc"], "b": [456]})
|
||||
assert tbl.view().to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
# make sure already created views are notified properly
|
||||
def test_update_from_schema_notify(self):
|
||||
tbl = Table({"a": "string", "b": "integer"})
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 0
|
||||
tbl.update([{"a": "abc", "b": 123}])
|
||||
assert view.to_records() == [{"a": "abc", "b": 123}]
|
||||
|
||||
def test_update_columnar_from_schema_notify(self):
|
||||
tbl = Table({"a": "string", "b": "integer"})
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 0
|
||||
tbl.update({"a": ["abc"], "b": [123]})
|
||||
assert view.to_records() == [{"a": "abc", "b": 123}]
|
||||
|
||||
def test_update_append_notify(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}])
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 1
|
||||
tbl.update([{"a": "def", "b": 456}])
|
||||
assert view.to_records() == [{"a": "abc", "b": 123}, {"a": "def", "b": 456}]
|
||||
|
||||
def test_update_partial_notify(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 1
|
||||
tbl.update([{"a": "abc", "b": 456}])
|
||||
assert view.to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
def test_update_partial_cols_not_in_schema_notify(self):
|
||||
tbl = Table([{"a": "abc", "b": 123}], index="a")
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 1
|
||||
tbl.update([{"a": "abc", "b": 456, "c": 789}])
|
||||
assert view.to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
def test_update_columnar_append_notify(self):
|
||||
tbl = Table({"a": ["abc"], "b": [123]})
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 1
|
||||
tbl.update({"a": ["def"], "b": [456]})
|
||||
assert view.to_records() == [{"a": "abc", "b": 123}, {"a": "def", "b": 456}]
|
||||
|
||||
def test_update_columnar_partial_notify(self):
|
||||
tbl = Table({"a": ["abc"], "b": [123]}, index="a")
|
||||
view = tbl.view()
|
||||
assert view.num_rows() == 1
|
||||
tbl.update({"a": ["abc"], "b": [456]})
|
||||
assert view.to_records() == [{"a": "abc", "b": 456}]
|
||||
|
||||
# bool
|
||||
|
||||
def test_update_bool_from_schema(self):
|
||||
bool_data = [{"a": True, "b": False}, {"a": True, "b": True}]
|
||||
tbl = Table({"a": "boolean", "b": "boolean"})
|
||||
tbl.update(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.view().to_records() == bool_data
|
||||
|
||||
def test_update_bool_str_from_schema(self):
|
||||
bool_data = [{"a": "True", "b": "False"}, {"a": "True", "b": "True"}]
|
||||
tbl = Table({"a": "boolean", "b": "boolean"})
|
||||
tbl.update(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": True, "b": False},
|
||||
{"a": True, "b": True},
|
||||
]
|
||||
|
||||
@mark.skip(reason="We may not want to support this")
|
||||
def test_update_bool_str_all_formats_from_schema(self):
|
||||
bool_data = [
|
||||
{"a": "True", "b": "False"},
|
||||
{"a": "t", "b": "f"},
|
||||
{"a": "true", "b": "false"},
|
||||
{"a": 1, "b": 0},
|
||||
{"a": "on", "b": "off"},
|
||||
]
|
||||
tbl = Table({"a": "boolean", "b": "boolean"})
|
||||
tbl.update(bool_data)
|
||||
assert tbl.size() == 5
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [True, True, True, True, True],
|
||||
"b": [False, False, False, False, False],
|
||||
}
|
||||
|
||||
def test_update_bool_int_from_schema(self):
|
||||
bool_data = [{"a": 1, "b": 0}, {"a": 1, "b": 0}]
|
||||
tbl = Table({"a": "boolean", "b": "boolean"})
|
||||
tbl.update(bool_data)
|
||||
assert tbl.size() == 2
|
||||
assert tbl.view().to_columns() == {"a": [True, True], "b": [False, False]}
|
||||
|
||||
# dates and datetimes
|
||||
def test_update_date(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
tbl.update([{"a": date(2019, 7, 12)}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12))},
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy.
|
||||
def test_update_date_np(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
tbl.update([{"a": np.datetime64(date(2019, 7, 12))}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12))},
|
||||
]
|
||||
|
||||
def test_update_datetime(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": datetime(2019, 7, 12, 11, 0)}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11, 11, 0))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))},
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy.
|
||||
def test_update_datetime_np(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": np.datetime64(datetime(2019, 7, 12, 11, 0))}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11, 11, 0))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))},
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy.
|
||||
def test_update_datetime_np_ts(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": np.datetime64("2019-07-12T11:00")}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11, 11, 0))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))},
|
||||
]
|
||||
|
||||
def test_update_datetime_timestamp_seconds(self, util):
|
||||
ts = util.to_timestamp(datetime(2019, 7, 12, 11, 0, 0))
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": ts}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11, 11, 0))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))},
|
||||
]
|
||||
|
||||
def test_update_datetime_timestamp_ms(self, util):
|
||||
ts = util.to_timestamp(datetime(2019, 7, 12, 11, 0, 0))
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": ts}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 11, 11, 0))},
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))},
|
||||
]
|
||||
|
||||
# partial date & datetime updates
|
||||
|
||||
def test_update_date_partial(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)], "b": [1]}, index="b")
|
||||
tbl.update([{"a": date(2019, 7, 12), "b": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12)), "b": 1}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_date_np_partial(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)], "b": [1]}, index="b")
|
||||
tbl.update([{"a": np.datetime64(date(2019, 7, 12)), "b": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12)), "b": 1}
|
||||
]
|
||||
|
||||
def test_update_datetime_partial(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "b": [1]}, index="b")
|
||||
tbl.update([{"a": datetime(2019, 7, 12, 11, 0), "b": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0)), "b": 1}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_datetime_np_partial(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "b": [1]}, index="b")
|
||||
tbl.update([{"a": np.datetime64(datetime(2019, 7, 12, 11, 0)), "b": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0)), "b": 1}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_datetime_np_ts_partial(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "b": [1]}, index="b")
|
||||
tbl.update([{"a": np.datetime64("2019-07-12T11:00"), "b": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0)), "b": 1}
|
||||
]
|
||||
|
||||
def test_update_datetime_timestamp_seconds_partial(self, util):
|
||||
ts = util.to_timestamp(datetime(2019, 7, 12, 11, 0, 0))
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "idx": [1]}, index="idx")
|
||||
tbl.update([{"a": ts, "idx": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0)), "idx": 1}
|
||||
]
|
||||
|
||||
def test_update_datetime_timestamp_ms_partial(self, util):
|
||||
ts = util.to_timestamp(datetime(2019, 7, 12, 11, 0, 0))
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "idx": [1]}, index="idx")
|
||||
tbl.update([{"a": ts, "idx": 1}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0)), "idx": 1}
|
||||
]
|
||||
|
||||
# updating dates using implicit index
|
||||
|
||||
def test_update_date_partial_implicit(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
tbl.update([{"a": date(2019, 7, 12), "__INDEX__": 0}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12))}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_date_np_partial_implicit(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
tbl.update([{"a": np.datetime64(date(2019, 7, 12)), "__INDEX__": 0}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12))}
|
||||
]
|
||||
|
||||
def test_update_datetime_partial_implicit(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": datetime(2019, 7, 12, 11, 0), "__INDEX__": 0}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_datetime_np_partial_implicit(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": np.datetime64(datetime(2019, 7, 12, 11, 0)), "__INDEX__": 0}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))}
|
||||
]
|
||||
|
||||
@mark.skip # We do not support numpy anymore.
|
||||
def test_update_datetime_np_ts_partial_implicit(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
tbl.update([{"a": np.datetime64("2019-07-12T11:00"), "__INDEX__": 0}])
|
||||
assert tbl.view().to_records() == [
|
||||
{"a": util.to_timestamp(datetime(2019, 7, 12, 11, 0))}
|
||||
]
|
||||
|
||||
# implicit index
|
||||
|
||||
def test_update_implicit_index(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
tbl.update([{"__INDEX__": 0, "a": 3, "b": 15}])
|
||||
assert view.to_records() == [{"a": 3, "b": 15}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_implicit_index_dict_noop(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
|
||||
# "b" does not exist in dataset, so no-op
|
||||
tbl.update(
|
||||
{
|
||||
"__INDEX__": [0],
|
||||
"a": [3],
|
||||
}
|
||||
)
|
||||
assert view.to_records() == [{"a": 3, "b": 2}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_implicit_index_dict_unset_with_null(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
|
||||
# unset because "b" is null
|
||||
tbl.update({"__INDEX__": [0], "a": [3], "b": [None]})
|
||||
assert view.to_records() == [{"a": 3, "b": None}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_implicit_index_multi(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}, {"a": 4, "b": 5}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
tbl.update(
|
||||
[
|
||||
{
|
||||
"__INDEX__": 0,
|
||||
"a": 3,
|
||||
},
|
||||
{"__INDEX__": 2, "a": 5},
|
||||
]
|
||||
)
|
||||
assert view.to_records() == [
|
||||
{"a": 3, "b": 2},
|
||||
{"a": 2, "b": 3},
|
||||
{"a": 5, "b": 5},
|
||||
]
|
||||
|
||||
def test_update_implicit_index_symmetric(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data)
|
||||
view = tbl.view()
|
||||
records = view.to_records(index=True)
|
||||
idx = records[0]["__INDEX__"][0] # XXX: How should we grab this?
|
||||
tbl.update([{"__INDEX__": idx, "a": 3}])
|
||||
assert view.to_records() == [{"a": 3, "b": 2}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_explicit_index(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update([{"a": 1, "b": 3}])
|
||||
assert view.to_records() == [{"a": 1, "b": 3}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_explicit_index_multi(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update([{"a": 1, "b": 3}, {"a": 3, "b": 5}])
|
||||
assert view.to_records() == [
|
||||
{"a": 1, "b": 3},
|
||||
{"a": 2, "b": 3},
|
||||
{"a": 3, "b": 5},
|
||||
]
|
||||
|
||||
def test_update_explicit_index_multi_append(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update([{"a": 1, "b": 3}, {"a": 12, "b": 5}])
|
||||
assert view.to_records() == [
|
||||
{"a": 1, "b": 3},
|
||||
{"a": 2, "b": 3},
|
||||
{"a": 3, "b": 4},
|
||||
{"a": 12, "b": 5},
|
||||
]
|
||||
|
||||
def test_update_explicit_index_multi_append_noindex(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}, {"a": 3, "b": 4}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update([{"a": 1, "b": 3}, {"b": 5}])
|
||||
assert view.to_records() == [
|
||||
{"a": None, "b": 5},
|
||||
{"a": 1, "b": 3},
|
||||
{"a": 2, "b": 3},
|
||||
{"a": 3, "b": 4},
|
||||
]
|
||||
|
||||
def test_update_implicit_index_with_explicit_unset(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update([{"__INDEX__": 1, "b": 3}])
|
||||
assert view.to_records() == [{"a": 1, "b": 3}, {"a": 2, "b": 3}]
|
||||
|
||||
def test_update_implicit_index_with_explicit_set(self):
|
||||
data = [{"a": 1, "b": 2}, {"a": 2, "b": 3}]
|
||||
tbl = Table(data, index="a")
|
||||
view = tbl.view()
|
||||
tbl.update(
|
||||
[{"__INDEX__": 1, "a": 1, "b": 3}]
|
||||
) # should ignore re-specification of pkey
|
||||
assert view.to_records() == [{"a": 1, "b": 3}, {"a": 2, "b": 3}]
|
||||
@@ -0,0 +1,980 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import os
|
||||
import random
|
||||
import uuid
|
||||
import pyarrow as pa
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from datetime import date, datetime
|
||||
from pytest import mark
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
SOURCE_STREAM_ARROW = os.path.join(
|
||||
os.path.dirname(__file__), "arrow", "int_float_str.arrow"
|
||||
)
|
||||
SOURCE_FILE_ARROW = os.path.join(
|
||||
os.path.dirname(__file__), "arrow", "int_float_str.arrow"
|
||||
)
|
||||
PARTIAL_ARROW = os.path.join(
|
||||
os.path.dirname(__file__), "arrow", "int_float_str_update.arrow"
|
||||
)
|
||||
DICT_ARROW = os.path.join(os.path.dirname(__file__), "arrow", "dict.arrow")
|
||||
DICT_UPDATE_ARROW = os.path.join(
|
||||
os.path.dirname(__file__), "arrow", "dict_update.arrow"
|
||||
)
|
||||
|
||||
names = ["a", "b", "c", "d"]
|
||||
|
||||
|
||||
class TestUpdateArrow(object):
|
||||
# files
|
||||
|
||||
def test_update_arrow_updates_stream_file(self):
|
||||
tbl = Table({"a": "integer", "b": "float", "c": "string"})
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "integer", "b": "float", "c": "string"}
|
||||
|
||||
with open(SOURCE_FILE_ARROW, mode="rb") as file:
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4] * 2,
|
||||
"b": [1.5, 2.5, 3.5, 4.5] * 2,
|
||||
"c": ["a", "b", "c", "d"] * 2,
|
||||
}
|
||||
|
||||
def test_update_arrow_partial_updates_file(self):
|
||||
tbl = Table({"a": "integer", "b": "float", "c": "string"}, index="a")
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 4
|
||||
|
||||
with open(PARTIAL_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4],
|
||||
"b": [100.5, 2.5, 3.5, 400.5],
|
||||
"c": ["x", "b", "c", "y"],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_dict_file(self):
|
||||
tbl = Table({"a": "string", "b": "string"})
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 5
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None, "abc", None, "update1", "update2"],
|
||||
"b": ["klm", "hij", None, "hij", "klm", "update3", None, "update4"],
|
||||
}
|
||||
|
||||
@mark.skip
|
||||
def test_update_arrow_updates_dict_partial_file(self):
|
||||
tbl = None
|
||||
v = None
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl = Table(src.read(), index="a")
|
||||
v = tbl.view()
|
||||
assert v.num_rows() == 2
|
||||
assert v.to_columns() == {"a": ["abc", "def"], "b": ["klm", "hij"]}
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
v.num_rows() == 4
|
||||
assert v.to_columns() == {
|
||||
"a": ["abc", "def", "update1", "update2"],
|
||||
"b": ["klm", "hij", None, "update4"],
|
||||
}
|
||||
|
||||
# update with file arrow with more columns than in schema
|
||||
|
||||
def test_update_arrow_updates_more_columns_stream_file(self):
|
||||
tbl = Table(
|
||||
{
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
}
|
||||
)
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "integer", "b": "float"}
|
||||
|
||||
with open(SOURCE_FILE_ARROW, mode="rb") as file:
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4] * 2,
|
||||
"b": [1.5, 2.5, 3.5, 4.5] * 2,
|
||||
}
|
||||
|
||||
def test_update_arrow_partial_updates_more_columns_file(self):
|
||||
tbl = Table({"a": "integer", "c": "string"}, index="a")
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 4
|
||||
|
||||
with open(PARTIAL_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4],
|
||||
"c": ["x", "b", "c", "y"],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_dict_more_columns_file(self):
|
||||
tbl = Table(
|
||||
{
|
||||
"a": "string",
|
||||
}
|
||||
)
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 5
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None, "abc", None, "update1", "update2"]
|
||||
}
|
||||
|
||||
@mark.skip
|
||||
def test_update_arrow_updates_dict_more_columns_partial_file(self):
|
||||
tbl = Table({"a": "string"}, index="a")
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 4
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "update1", "update2"]
|
||||
}
|
||||
|
||||
# update with file arrow with less columns than in schema
|
||||
|
||||
def test_update_arrow_updates_less_columns_stream_file(self):
|
||||
tbl = Table(
|
||||
{
|
||||
"a": "integer",
|
||||
"x": "float",
|
||||
}
|
||||
)
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as file: # b is important -> binary
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.schema() == {"a": "integer", "x": "float"}
|
||||
|
||||
with open(SOURCE_FILE_ARROW, mode="rb") as file:
|
||||
tbl.update(file.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4] * 2,
|
||||
"x": [None for i in range(8)],
|
||||
}
|
||||
|
||||
def test_update_arrow_partial_updates_less_columns_file(self):
|
||||
tbl = Table({"a": "integer", "x": "string"}, index="a")
|
||||
|
||||
with open(SOURCE_STREAM_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 4
|
||||
|
||||
with open(PARTIAL_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4],
|
||||
"x": [None for i in range(4)],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_dict_less_columns_file(self):
|
||||
tbl = Table({"a": "string", "x": "string"})
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 5
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "def", None, "abc", None, "update1", "update2"],
|
||||
"x": [None for i in range(8)],
|
||||
}
|
||||
|
||||
@mark.skip
|
||||
def test_update_arrow_updates_dict_less_columns_partial_file(self):
|
||||
tbl = Table({"a": "string", "x": "string"}, index="a")
|
||||
|
||||
with open(DICT_ARROW, mode="rb") as src:
|
||||
tbl.update(src.read())
|
||||
assert tbl.size() == 4
|
||||
|
||||
with open(DICT_UPDATE_ARROW, mode="rb") as partial:
|
||||
tbl.update(partial.read())
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["abc", "def", "update1", "update2"],
|
||||
"x": [None for i in range(4)],
|
||||
}
|
||||
|
||||
# update int schema with int
|
||||
|
||||
def test_update_arrow_update_int_schema_with_uint8(self, util):
|
||||
array = [random.randint(0, 127) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint8)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint8()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_uint16(self, util):
|
||||
array = [random.randint(0, 32767) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint16)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint16()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_uint32(self, util):
|
||||
array = [random.randint(0, 2000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint32)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_uint64(self, util):
|
||||
array = [random.randint(0, 20000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint64)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_int8(self, util):
|
||||
array = [random.randint(-127, 127) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int8)})
|
||||
|
||||
schema = pa.schema({"a": pa.int8()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_int16(self, util):
|
||||
array = [random.randint(-32767, 32767) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int16)})
|
||||
|
||||
schema = pa.schema({"a": pa.int16()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_int32(self, util):
|
||||
array = [random.randint(-2000000, 2000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int32)})
|
||||
|
||||
schema = pa.schema({"a": pa.int32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_int_schema_with_int64(self, util):
|
||||
array = [random.randint(-20000000, 20000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int64)})
|
||||
|
||||
schema = pa.schema({"a": pa.int64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == [x * 1.0 for x in array]
|
||||
|
||||
# updating float schema with int
|
||||
|
||||
def test_update_arrow_update_float_schema_with_uint8(self, util):
|
||||
array = [random.randint(0, 127) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint8)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint8()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_uint16(self, util):
|
||||
array = [random.randint(0, 32767) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint16)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint16()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_uint32(self, util):
|
||||
array = [random.randint(0, 2000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint32)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_uint64(self, util):
|
||||
array = [random.randint(0, 20000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.uint64)})
|
||||
|
||||
schema = pa.schema({"a": pa.uint64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_int8(self, util):
|
||||
array = [random.randint(-127, 127) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int8)})
|
||||
|
||||
schema = pa.schema({"a": pa.int8()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_int16(self, util):
|
||||
array = [random.randint(-32767, 32767) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int16)})
|
||||
|
||||
schema = pa.schema({"a": pa.int16()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_int32(self, util):
|
||||
array = [random.randint(-2000000, 2000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int32)})
|
||||
|
||||
schema = pa.schema({"a": pa.int32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_int64(self, util):
|
||||
array = [random.randint(-20000000, 20000000) for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.int64)})
|
||||
|
||||
schema = pa.schema({"a": pa.int64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
# updating int schema with float
|
||||
def test_update_arrow_update_int_schema_with_float32(self, util):
|
||||
array = [random.randint(-2000000, 2000000) * 0.5 for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.float32)})
|
||||
|
||||
schema = pa.schema({"a": pa.float32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == [int(x) for x in array]
|
||||
|
||||
def test_update_arrow_update_int_schema_with_float64(self, util):
|
||||
array = [
|
||||
random.randint(-20000000, 20000000) * random.random() for i in range(100)
|
||||
]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.float64)})
|
||||
|
||||
schema = pa.schema({"a": pa.float64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == [int(x) for x in array]
|
||||
|
||||
# updating float schema with float
|
||||
|
||||
def test_update_arrow_update_float_schema_with_float32(self, util):
|
||||
array = [random.randint(-2000000, 2000000) * 0.5 for i in range(100)]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.float32)})
|
||||
|
||||
schema = pa.schema({"a": pa.float32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
def test_update_arrow_update_float_schema_with_float64(self, util):
|
||||
array = [
|
||||
random.randint(-20000000, 20000000) * random.random() for i in range(100)
|
||||
]
|
||||
data = pd.DataFrame({"a": np.array(array, dtype=np.float64)})
|
||||
|
||||
schema = pa.schema({"a": pa.float64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
tbl = Table({"a": "float"})
|
||||
tbl.update(arrow)
|
||||
assert tbl.view().to_columns()["a"] == array
|
||||
|
||||
# updating date schema
|
||||
|
||||
def test_update_arrow_update_date_schema_with_date32(self, util):
|
||||
array = [date(2019, 2, i) for i in range(1, 11)]
|
||||
data = pd.DataFrame({"a": array})
|
||||
|
||||
schema = pa.schema({"a": pa.date32()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
|
||||
tbl = Table({"a": "date"})
|
||||
|
||||
tbl.update(arrow)
|
||||
|
||||
assert tbl.view().to_columns()["a"] == [
|
||||
util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)
|
||||
]
|
||||
|
||||
def test_update_arrow_update_date_schema_with_date64(self, util):
|
||||
array = [date(2019, 2, i) for i in range(1, 11)]
|
||||
data = pd.DataFrame({"a": array})
|
||||
|
||||
schema = pa.schema({"a": pa.date64()})
|
||||
|
||||
arrow = util.make_arrow_from_pandas(data, schema)
|
||||
|
||||
tbl = Table({"a": "date"})
|
||||
|
||||
tbl.update(arrow)
|
||||
|
||||
assert tbl.view().to_columns()["a"] == [
|
||||
util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)
|
||||
]
|
||||
|
||||
def test_update_arrow_update_datetime_schema_with_timestamp(self, util):
|
||||
data = [
|
||||
[datetime(2019, 2, i, 9) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 10) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 11) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 12) for i in range(1, 11)],
|
||||
]
|
||||
|
||||
arrow_data = util.make_arrow(
|
||||
names,
|
||||
data,
|
||||
types=[
|
||||
pa.timestamp("s"),
|
||||
pa.timestamp("ms"),
|
||||
pa.timestamp("us"),
|
||||
pa.timestamp("ns"),
|
||||
],
|
||||
)
|
||||
|
||||
tbl = Table(
|
||||
{
|
||||
"a": "datetime",
|
||||
"b": "datetime",
|
||||
"c": "datetime",
|
||||
"d": "datetime",
|
||||
}
|
||||
)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(d) for d in data[0]],
|
||||
"b": [util.to_timestamp(d) for d in data[1]],
|
||||
"c": [util.to_timestamp(d) for d in data[2]],
|
||||
"d": [util.to_timestamp(d) for d in data[3]],
|
||||
}
|
||||
|
||||
# streams
|
||||
|
||||
def test_update_arrow_updates_int_stream(self, util):
|
||||
data = [list(range(10)) for i in range(4)]
|
||||
arrow_data = util.make_arrow(names, data)
|
||||
tbl = Table({"a": "integer", "b": "integer", "c": "integer", "d": "integer"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": data[0],
|
||||
"b": data[1],
|
||||
"c": data[2],
|
||||
"d": data[3],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_float_stream(self, util):
|
||||
data = [[i for i in range(10)], [i * 1.5 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a", "b"], data)
|
||||
tbl = Table(
|
||||
{
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
}
|
||||
)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {"a": data[0], "b": data[1]}
|
||||
|
||||
@mark.skip(reason="Decimal128 isn't part of our schema yet")
|
||||
def test_update_arrow_updates_decimal128_stream(self, util):
|
||||
data = [[i * 1000000000 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.decimal128(10)])
|
||||
tbl = Table({"a": "integer"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_update_arrow_updates_bool_stream(self, util):
|
||||
data = [[True if i % 2 == 0 else False for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data)
|
||||
tbl = Table({"a": "boolean"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_update_arrow_updates_date32_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date32()])
|
||||
tbl = Table({"a": "date"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [int(1000 * datetime(2019, 2, i).timestamp()) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_date64_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date64()])
|
||||
tbl = Table({"a": "date"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 2, i)) for i in range(1, 11)]
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_timestamp_all_formats_stream(self, util):
|
||||
data = [
|
||||
[datetime(2019, 2, i, 9) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 10) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 11) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 12) for i in range(1, 11)],
|
||||
]
|
||||
arrow_data = util.make_arrow(
|
||||
names,
|
||||
data,
|
||||
types=[
|
||||
pa.timestamp("s"),
|
||||
pa.timestamp("ms"),
|
||||
pa.timestamp("us"),
|
||||
pa.timestamp("ns"),
|
||||
],
|
||||
)
|
||||
tbl = Table(
|
||||
{"a": "datetime", "b": "datetime", "c": "datetime", "d": "datetime"}
|
||||
)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(d) for d in data[0]],
|
||||
"b": [util.to_timestamp(d) for d in data[1]],
|
||||
"c": [util.to_timestamp(d) for d in data[2]],
|
||||
"d": [util.to_timestamp(d) for d in data[3]],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_string_stream(self, util):
|
||||
data = [[str(i) for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.string()])
|
||||
tbl = Table({"a": "string"})
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 10
|
||||
assert tbl.view().to_columns() == {"a": data[0]}
|
||||
|
||||
def test_update_arrow_updates_dictionary_stream(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table({"a": "string", "b": "string"})
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["a", "b", "b", None],
|
||||
"b": ["x", "y", None, "z"],
|
||||
}
|
||||
|
||||
@mark.skip(reason="Arrow no longer supports partial updates per row")
|
||||
def test_update_arrow_partial_updates_dictionary_stream(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table({"a": "string", "b": "string"}, index="a")
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {"a": [None, "a", "b"], "b": ["z", "x", "y"]}
|
||||
|
||||
@mark.skip
|
||||
def test_update_arrow_partial_updates_dictionary_stream_duplicates(self, util):
|
||||
"""If there are duplicate values in the dictionary, primary keys
|
||||
may be duplicated if the column is used as an index. Skip this test
|
||||
for now - still looking for the best way to fix."""
|
||||
data = [
|
||||
([0, 1, 1, None, 2], ["a", "b", "a"]),
|
||||
([0, 1, None, 2, 1], ["x", "y", "z"]),
|
||||
]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
|
||||
tbl = Table({"a": "string", "b": "string"}, index="a")
|
||||
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {"a": [None, "a", "b"], "b": ["z", "x", "y"]}
|
||||
|
||||
def test_update_arrow_partial_updates_more_columns_dictionary_stream(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
|
||||
tbl = Table({"a": "string"}, index="a")
|
||||
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {"a": [None, "a", "b"]}
|
||||
|
||||
@mark.skip(reason="Arrow no longer supports partial updates per row")
|
||||
def test_update_arrow_partial_updates_less_columns_dictionary_stream(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table({"a": "string", "b": "string", "x": "string"}, index="a")
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [None, "a", "b"],
|
||||
"b": ["z", "x", "y"],
|
||||
"x": [None, None, None],
|
||||
}
|
||||
|
||||
def test_update_arrow_arbitary_order(self, util):
|
||||
data = [[1, 2, 3, 4], ["a", "b", "c", "d"], [1, 2, 3, 4], ["a", "b", "c", "d"]]
|
||||
update_data = [[5, 6], ["e", "f"], [5, 6], ["e", "f"]]
|
||||
arrow = util.make_arrow(["a", "b", "c", "d"], data)
|
||||
update_arrow = util.make_arrow(["c", "b", "a", "d"], update_data)
|
||||
tbl = Table(arrow)
|
||||
assert tbl.schema() == {
|
||||
"a": "integer",
|
||||
"b": "string",
|
||||
"c": "integer",
|
||||
"d": "string",
|
||||
}
|
||||
tbl.update(update_arrow)
|
||||
assert tbl.size() == 6
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4, 5, 6],
|
||||
"b": ["a", "b", "c", "d", "e", "f"],
|
||||
"c": [1, 2, 3, 4, 5, 6],
|
||||
"d": ["a", "b", "c", "d", "e", "f"],
|
||||
}
|
||||
|
||||
# append
|
||||
|
||||
def test_update_arrow_updates_append_int_stream(self, util):
|
||||
data = [list(range(10)) for i in range(4)]
|
||||
arrow_data = util.make_arrow(names, data)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": data[0] + data[0],
|
||||
"b": data[1] + data[1],
|
||||
"c": data[2] + data[2],
|
||||
"d": data[3] + data[3],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_float_stream(self, util):
|
||||
data = [[i for i in range(10)], [i * 1.5 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": data[0] + data[0],
|
||||
"b": data[1] + data[1],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_decimal_stream(self, util):
|
||||
data = [[i * 1000 for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.decimal128(4)])
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {"a": data[0] + data[0]}
|
||||
|
||||
def test_update_arrow_updates_append_bool_stream(self, util):
|
||||
data = [[True if i % 2 == 0 else False for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {"a": data[0] + data[0]}
|
||||
|
||||
def test_update_arrow_updates_append_date32_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
out_data = [datetime(2019, 2, i) for i in range(1, 11)]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date32()])
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(d) for d in out_data + out_data]
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_date64_stream(self, util):
|
||||
data = [[date(2019, 2, i) for i in range(1, 11)]]
|
||||
out_data = [datetime(2019, 2, i) for i in range(1, 11)]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.date64()])
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(d) for d in out_data + out_data]
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_timestamp_all_formats_stream(self, util):
|
||||
data = [
|
||||
[datetime(2019, 2, i, 9) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 10) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 11) for i in range(1, 11)],
|
||||
[datetime(2019, 2, i, 12) for i in range(1, 11)],
|
||||
]
|
||||
arrow_data = util.make_arrow(
|
||||
names,
|
||||
data,
|
||||
types=[
|
||||
pa.timestamp("s"),
|
||||
pa.timestamp("ms"),
|
||||
pa.timestamp("us"),
|
||||
pa.timestamp("ns"),
|
||||
],
|
||||
)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(d) for d in data[0] + data[0]],
|
||||
"b": [util.to_timestamp(d) for d in data[1] + data[1]],
|
||||
"c": [util.to_timestamp(d) for d in data[2] + data[2]],
|
||||
"d": [util.to_timestamp(d) for d in data[3] + data[3]],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_string_stream(self, util):
|
||||
data = [[str(i) for i in range(10)]]
|
||||
arrow_data = util.make_arrow(["a"], data, types=[pa.string()])
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
assert tbl.size() == 20
|
||||
assert tbl.view().to_columns() == {"a": data[0] + data[0]}
|
||||
|
||||
def test_update_arrow_updates_append_dictionary_stream(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["a", "b", "b", None, "a", "b", "b", None],
|
||||
"b": ["x", "y", None, "z", "x", "y", None, "z"],
|
||||
}
|
||||
|
||||
def test_update_arrow_updates_append_dictionary_stream_legacy(self, util):
|
||||
data = [([0, 1, 1, None], ["a", "b"]), ([0, 1, None, 2], ["x", "y", "z"])]
|
||||
arrow_data = util.make_dictionary_arrow(["a", "b"], data, legacy=True)
|
||||
tbl = Table(arrow_data)
|
||||
tbl.update(arrow_data)
|
||||
|
||||
assert tbl.size() == 8
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["a", "b", "b", None, "a", "b", "b", None],
|
||||
"b": ["x", "y", None, "z", "x", "y", None, "z"],
|
||||
}
|
||||
|
||||
# indexed
|
||||
|
||||
def test_update_arrow_partial_indexed(self, util):
|
||||
data = [[1, 2, 3, 4], ["a", "b", "c", "d"]]
|
||||
update_data = [[2, 4], ["x", "y"]]
|
||||
arrow = util.make_arrow(["a", "b"], data)
|
||||
update_arrow = util.make_arrow(["a", "b"], update_data)
|
||||
tbl = Table(arrow, index="a")
|
||||
assert tbl.schema() == {"a": "integer", "b": "string"}
|
||||
tbl.update(update_arrow)
|
||||
assert tbl.size() == 4
|
||||
assert tbl.view().to_columns() == {"a": [1, 2, 3, 4], "b": ["a", "x", "c", "y"]}
|
||||
|
||||
# update specific columns
|
||||
|
||||
def test_update_arrow_specific_column(self, util):
|
||||
data = [[1, 2, 3, 4], ["a", "b", "c", "d"]]
|
||||
update_data = [[2, 3, 4]]
|
||||
arrow = util.make_arrow(["a", "b"], data)
|
||||
update_arrow = util.make_arrow(["a"], update_data)
|
||||
tbl = Table(arrow)
|
||||
assert tbl.schema() == {"a": "integer", "b": "string"}
|
||||
tbl.update(update_arrow)
|
||||
assert tbl.size() == 7
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4, 2, 3, 4],
|
||||
"b": ["a", "b", "c", "d", None, None, None],
|
||||
}
|
||||
|
||||
# try to fuzz column order
|
||||
|
||||
def test_update_arrow_column_order_str(self, util):
|
||||
# use str so it doesn't get promoted
|
||||
data = [["a", "b", "c"] for i in range(10)]
|
||||
names = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]
|
||||
names_scrambled = names[::-1]
|
||||
arrow = util.make_arrow(names_scrambled, data)
|
||||
tbl = Table({name: "string" for name in names})
|
||||
tbl.update(arrow)
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {name: data[0] for name in names}
|
||||
|
||||
def test_update_arrow_column_order_int(self, util):
|
||||
data = [[1, 2, 3] for i in range(10)]
|
||||
names = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]
|
||||
names_scrambled = names[::-1]
|
||||
arrow = util.make_arrow(names_scrambled, data)
|
||||
tbl = Table({name: "integer" for name in names})
|
||||
tbl.update(arrow)
|
||||
assert tbl.size() == 3
|
||||
assert tbl.view().to_columns() == {name: data[0] for name in names}
|
||||
|
||||
def test_update_arrow_thread_safe_int_index(self, util):
|
||||
data = [["a", "b", "c"] for i in range(10)]
|
||||
data += [[1, 2, 3]]
|
||||
names = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "uid"]
|
||||
arrow = util.make_arrow(names, data)
|
||||
tbl = Table(arrow, index="uid")
|
||||
|
||||
for i in range(100):
|
||||
idx = (1, 2, 3)[random.randint(0, 2)]
|
||||
update_data = [
|
||||
[str(uuid.uuid4()) + str(random.randint(100, 1000000000))],
|
||||
[idx],
|
||||
]
|
||||
update_names = [names[random.randint(0, 9)], "uid"]
|
||||
update_arrow = util.make_arrow(update_names, update_data)
|
||||
tbl.update(update_arrow)
|
||||
|
||||
assert tbl.size() == 3
|
||||
|
||||
def test_update_arrow_thread_safe_datetime_index(self, util):
|
||||
data = [["a", "b", "c"] for i in range(10)]
|
||||
data += [
|
||||
[
|
||||
datetime(2020, 1, 15, 12, 17),
|
||||
datetime(2020, 1, 15, 12, 18),
|
||||
datetime(2020, 1, 15, 12, 19),
|
||||
]
|
||||
]
|
||||
names = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "uid"]
|
||||
arrow = util.make_arrow(names, data)
|
||||
tbl = Table(arrow, index="uid")
|
||||
|
||||
for i in range(100):
|
||||
idx = (
|
||||
datetime(2020, 1, 15, 12, 17),
|
||||
datetime(2020, 1, 15, 12, 18),
|
||||
datetime(2020, 1, 15, 12, 19),
|
||||
)[random.randint(0, 2)]
|
||||
update_data = [
|
||||
[str(uuid.uuid4()) + str(random.randint(100, 1000000000))],
|
||||
[idx],
|
||||
]
|
||||
update_names = [names[random.randint(0, 9)], "uid"]
|
||||
update_arrow = util.make_arrow(update_names, update_data)
|
||||
tbl.update(update_arrow)
|
||||
|
||||
assert tbl.size() == 3
|
||||
|
||||
def test_update_arrow_thread_safe_str_index(self, util):
|
||||
data = [["a", "b", "c"] for i in range(11)]
|
||||
names = ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "uid"]
|
||||
arrow = util.make_arrow(names, data)
|
||||
tbl = Table(arrow, index="uid")
|
||||
|
||||
for i in range(100):
|
||||
idx = ("a", "b", "c")[random.randint(0, 2)]
|
||||
update_data = [
|
||||
[str(uuid.uuid4()) + str(random.randint(100, 1000000000))],
|
||||
[idx],
|
||||
]
|
||||
update_names = [names[random.randint(0, 9)], "uid"]
|
||||
update_arrow = util.make_arrow(update_names, update_data)
|
||||
tbl.update(update_arrow)
|
||||
|
||||
assert tbl.size() == 3
|
||||
@@ -0,0 +1,211 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import date, datetime
|
||||
from pytest import mark
|
||||
import pytest
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestUpdatePandas(object):
|
||||
def test_update_df(self):
|
||||
tbl = Table({"a": [1, 2, 3, 4]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7, 8]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {"a": [1, 2, 3, 4, 5, 6, 7, 8]}
|
||||
|
||||
def test_update_df_i32_vs_i64(self):
|
||||
tbl = Table({"a": "integer"})
|
||||
|
||||
update_data = pd.DataFrame({"a": np.array([5, 6, 7, 8], dtype="int64")})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {"a": [5, 6, 7, 8]}
|
||||
|
||||
def test_update_df_bool(self):
|
||||
tbl = Table({"a": [True, False, True, False]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [True, False, True, False]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [True, False, True, False, True, False, True, False]
|
||||
}
|
||||
|
||||
def test_update_df_str(self):
|
||||
tbl = Table({"a": ["a", "b", "c", "d"]})
|
||||
|
||||
update_data = pd.DataFrame({"a": ["a", "b", "c", "d"]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": ["a", "b", "c", "d", "a", "b", "c", "d"]
|
||||
}
|
||||
|
||||
def test_update_df_date(self, util):
|
||||
# set_global_serializer(serializer)
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
update_data = pd.DataFrame({"a": [date(2019, 7, 12)]})
|
||||
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(d)
|
||||
for d in [datetime(2019, 7, 11), datetime(2019, 7, 12)]
|
||||
]
|
||||
}
|
||||
|
||||
@mark.skip(reason="this test relies on a lossy conversion from datetime -> date.")
|
||||
def test_update_df_date_timestamp(self, util):
|
||||
tbl = Table({"a": [date(2019, 7, 11)]})
|
||||
|
||||
assert tbl.schema() == {"a": "date"}
|
||||
|
||||
update_data = pd.DataFrame({"a": [datetime(2019, 7, 12, 0, 0, 0)]})
|
||||
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(datetime(2019, 7, 11)),
|
||||
util.to_timestamp(datetime(2019, 7, 12)),
|
||||
]
|
||||
}
|
||||
|
||||
def test_update_df_datetime(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [datetime(2019, 7, 12, 11, 0)]})
|
||||
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(datetime(2019, 7, 11, 11, 0)),
|
||||
util.to_timestamp(datetime(2019, 7, 12, 11, 0)),
|
||||
]
|
||||
}
|
||||
|
||||
def test_update_df_datetime_timestamp_seconds(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [datetime(2019, 7, 12, 11, 0)]})
|
||||
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(datetime(2019, 7, 11, 11, 0)),
|
||||
util.to_timestamp(datetime(2019, 7, 12, 11, 0)),
|
||||
]
|
||||
}
|
||||
|
||||
def test_update_df_datetime_timestamp_ms(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [datetime(2019, 7, 12, 11, 0)]})
|
||||
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [
|
||||
util.to_timestamp(datetime(2019, 7, 11, 11, 0)),
|
||||
util.to_timestamp(datetime(2019, 7, 12, 11, 0)),
|
||||
]
|
||||
}
|
||||
|
||||
def test_update_df_partial(self):
|
||||
tbl = Table({"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"]}, index="b")
|
||||
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7, 8], "b": ["a", "b", "c", "d"]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {"a": [5, 6, 7, 8], "b": ["a", "b", "c", "d"]}
|
||||
|
||||
def test_df_update_index(self):
|
||||
tbl = Table(pd.DataFrame({"a": [1, 2, 3, 4]}))
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7, 8], "__INDEX__": [0, 1, 2, 3]})
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {"a": [5, 6, 7, 8], "index": [0, 1, 2, 3]}
|
||||
|
||||
def test_update_df_partial_implicit(self):
|
||||
tbl = Table({"a": [1, 2, 3, 4]})
|
||||
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7, 8], "__INDEX__": [0, 1, 2, 3]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {"a": [5, 6, 7, 8]}
|
||||
|
||||
def test_update_df_datetime_partial(self, util):
|
||||
tbl = Table({"a": [datetime(2019, 7, 11, 11, 0)], "b": [1]}, index="b")
|
||||
|
||||
update_data = pd.DataFrame({"a": [datetime(2019, 7, 12, 11, 0)], "b": [1]})
|
||||
|
||||
tbl.update(update_data)
|
||||
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [util.to_timestamp(datetime(2019, 7, 12, 11, 0))],
|
||||
"b": [1],
|
||||
}
|
||||
|
||||
def test_update_df_one_col(self):
|
||||
tbl = Table({"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"]})
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7]})
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [1, 2, 3, 4, 5, 6, 7],
|
||||
"b": ["a", "b", "c", "d", None, None, None],
|
||||
}
|
||||
|
||||
def test_update_df_nonseq_partial(self):
|
||||
tbl = Table({"a": [1, 2, 3, 4], "b": ["a", "b", "c", "d"]}, index="b")
|
||||
update_data = pd.DataFrame({"a": [5, 6, 7], "b": ["a", "c", "d"]})
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {"a": [5, 2, 6, 7], "b": ["a", "b", "c", "d"]}
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_update_df_with_none_partial(self):
|
||||
tbl = Table({"a": [1, float("nan"), 3], "b": ["a", None, "d"]}, index="b")
|
||||
update_data = pd.DataFrame({"a": [4, 5], "b": ["a", "d"]})
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {
|
||||
"a": [None, 4, 5],
|
||||
"b": [None, "a", "d"],
|
||||
} # pkeys are ordered
|
||||
|
||||
def test_update_df_unset_partial(self):
|
||||
tbl = Table({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index="b")
|
||||
update_data = pd.DataFrame(
|
||||
{"a": pd.Series([None, None], dtype=pd.Int32Dtype()), "b": ["a", "c"]}
|
||||
)
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {"a": [None, 2, None], "b": ["a", "b", "c"]}
|
||||
|
||||
def test_update_df_nan_partial(self):
|
||||
tbl = Table({"a": [1, 2, 3], "b": ["a", "b", "c"]}, index="b")
|
||||
update_data = pd.DataFrame(
|
||||
{"a": pd.Series([None, None], dtype=pd.Int32Dtype()), "b": ["a", "c"]}
|
||||
)
|
||||
tbl.update(update_data)
|
||||
assert tbl.view().to_columns() == {"a": [None, 2, None], "b": ["a", "b", "c"]}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,53 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
# test that loading perspective and calling a common constructor code path does
|
||||
# not load various "expensive" modules. "Expensive" is in quotes because this is
|
||||
# utter nonsense.
|
||||
@pytest.mark.skip(reason="https://github.com/pyca/bcrypt/issues/694")
|
||||
def test_lazy_modules():
|
||||
import sys
|
||||
|
||||
cache = {}
|
||||
for key in list(sys.modules.keys()):
|
||||
if (
|
||||
key.startswith("perspective")
|
||||
or key.startswith("test")
|
||||
or key.startswith("pandas")
|
||||
or key.startswith("pyarrow")
|
||||
or key.startswith("tornado")
|
||||
):
|
||||
cache[key] = sys.modules[key]
|
||||
del sys.modules[key]
|
||||
|
||||
import perspective
|
||||
|
||||
t1 = perspective.table("x\n1")
|
||||
t1.delete()
|
||||
|
||||
assert "perspective" in sys.modules
|
||||
assert "pandas" not in sys.modules
|
||||
assert "pyarrow" not in sys.modules
|
||||
assert "tornado" not in sys.modules
|
||||
|
||||
for k, v in cache.items():
|
||||
sys.modules[k] = v
|
||||
|
||||
|
||||
def test_all():
|
||||
import perspective
|
||||
|
||||
for key in perspective.__all__:
|
||||
assert hasattr(perspective, key)
|
||||
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,246 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
from perspective.widget.viewer import PerspectiveViewer
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
class TestViewer:
|
||||
def test_viewer_get_table(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
assert viewer.table == table
|
||||
|
||||
# loading
|
||||
|
||||
def test_viewer_load_table(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
assert viewer.columns == ["a"]
|
||||
|
||||
def test_viewer_load_named_table(self):
|
||||
table = Table({"a": [1, 2, 3]}, name="data_1")
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
assert viewer.columns == ["a"]
|
||||
assert viewer.table_name == "data_1"
|
||||
assert viewer.table == table
|
||||
|
||||
def test_viewer_load_data(self):
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load({"a": [1, 2, 3]})
|
||||
assert viewer.columns == ["a"]
|
||||
|
||||
def test_viewer_load_named_data(self):
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load({"a": [1, 2, 3]}, name="data_1")
|
||||
assert viewer.columns == ["a"]
|
||||
assert viewer.table_name == "data_1"
|
||||
|
||||
def test_viewer_load_schema(self):
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load({"a": "string", "b": "integer", "c": "boolean", "d": "string"})
|
||||
for col in viewer.columns:
|
||||
assert col in ["a", "b", "c", "d"]
|
||||
|
||||
def test_viewer_load_table_with_options(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
# options should be disregarded when loading Table
|
||||
viewer.load(table, limit=1)
|
||||
assert viewer.columns == ["a"]
|
||||
assert viewer.table.size() == 3
|
||||
|
||||
def test_viewer_load_data_with_options(self):
|
||||
viewer = PerspectiveViewer()
|
||||
# options should be forwarded to the Table constructor
|
||||
viewer.load({"a": [1, 2, 3]}, limit=1)
|
||||
assert viewer.columns == ["a"]
|
||||
assert viewer.table.size() == 1
|
||||
|
||||
def test_viewer_load_clears_state(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer(theme="Pro Dark", group_by=["a"])
|
||||
viewer.load(table)
|
||||
assert viewer.group_by == ["a"]
|
||||
viewer.load({"b": [1, 2, 3]})
|
||||
assert viewer.group_by == []
|
||||
assert viewer.theme == "Pro Dark" # should not break UI
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_viewer_load_np(self):
|
||||
table = Table({"a": np.arange(1, 100)})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
assert viewer.columns == ["a"]
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_viewer_load_np_data(self):
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load({"a": np.arange(1, 100)})
|
||||
assert viewer.columns == ["a"]
|
||||
assert viewer.table.size() == 99
|
||||
|
||||
@pytest.mark.skip
|
||||
def test_viewer_load_df(self):
|
||||
table = Table(pd.DataFrame({"a": np.arange(1, 100)}))
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
for col in viewer.columns:
|
||||
assert col in ["index", "a"]
|
||||
assert viewer.table.size() == 99
|
||||
|
||||
def test_viewer_load_df_data(self):
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(pd.DataFrame({"a": np.arange(1, 100)}))
|
||||
for col in viewer.columns:
|
||||
assert col in ["index", "a"]
|
||||
|
||||
# update
|
||||
|
||||
def test_viewer_update_dict(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.update({"a": [4, 5, 6]})
|
||||
assert table.size() == 6
|
||||
assert viewer.table.size() == 6
|
||||
assert viewer.table.view().to_columns() == {"a": [1, 2, 3, 4, 5, 6]}
|
||||
|
||||
def test_viewer_update_list(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.update([{"a": 4}, {"a": 5}, {"a": 6}])
|
||||
assert table.size() == 6
|
||||
assert viewer.table.size() == 6
|
||||
assert viewer.table.view().to_columns() == {"a": [1, 2, 3, 4, 5, 6]}
|
||||
|
||||
def test_viewer_update_df(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.update(pd.DataFrame({"a": [4, 5, 6]}))
|
||||
assert table.size() == 6
|
||||
assert viewer.table.size() == 6
|
||||
assert viewer.table.view().to_columns() == {"a": [1, 2, 3, 4, 5, 6]}
|
||||
|
||||
def test_viewer_update_dict_partial(self):
|
||||
table = Table({"a": [1, 2, 3], "b": [5, 6, 7]}, index="a")
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.update({"a": [1, 2, 3], "b": [8, 9, 10]})
|
||||
assert table.size() == 3
|
||||
assert viewer.table.size() == 3
|
||||
assert viewer.table.view().to_columns() == {"a": [1, 2, 3], "b": [8, 9, 10]}
|
||||
|
||||
# clear
|
||||
|
||||
def test_viewer_clear(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.clear()
|
||||
assert viewer.table.size() == 0
|
||||
assert viewer.table.schema() == {"a": "integer"}
|
||||
|
||||
# replace
|
||||
|
||||
def test_viewer_replace(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer()
|
||||
viewer.load(table)
|
||||
viewer.replace({"a": [4, 5, 6]})
|
||||
assert viewer.table.size() == 3
|
||||
assert viewer.table.schema() == {"a": "integer"}
|
||||
assert viewer.table.view().to_columns() == {"a": [4, 5, 6]}
|
||||
|
||||
# reset
|
||||
|
||||
def test_viewer_reset(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer(plugin="X Bar", filter=[["a", "==", 2]])
|
||||
viewer.load(table)
|
||||
assert viewer.filter == [["a", "==", 2]]
|
||||
viewer.reset()
|
||||
assert viewer.plugin == "Datagrid"
|
||||
assert viewer.filter == []
|
||||
|
||||
# delete
|
||||
|
||||
def test_viewer_delete(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer(plugin="X Bar", filter=[["a", "==", 2]])
|
||||
viewer.load(table)
|
||||
assert viewer.filter == [["a", "==", 2]]
|
||||
viewer.delete()
|
||||
assert viewer.table_name is None
|
||||
assert viewer.table is None
|
||||
|
||||
def test_viewer_delete_without_table(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer(plugin="X Bar", filter=[["a", "==", 2]])
|
||||
viewer.load(table)
|
||||
assert viewer.filter == [["a", "==", 2]]
|
||||
viewer.delete(delete_table=False)
|
||||
assert viewer.table_name is not None
|
||||
assert viewer.table is not None
|
||||
assert viewer.filter == []
|
||||
|
||||
def test_save_restore(self):
|
||||
table = Table({"a": [1, 2, 3]})
|
||||
viewer = PerspectiveViewer(
|
||||
plugin="X Bar", filter=[["a", "==", 2]], expressions={'"a" * 2': '"a" * 2'}
|
||||
)
|
||||
viewer.load(table)
|
||||
|
||||
# Save config
|
||||
config = viewer.save()
|
||||
assert viewer.filter == [["a", "==", 2]]
|
||||
assert config["filter"] == [["a", "==", 2]]
|
||||
assert viewer.plugin == "X Bar"
|
||||
assert config["plugin"] == "X Bar"
|
||||
assert config["expressions"] == {'"a" * 2': '"a" * 2'}
|
||||
|
||||
# reset configuration
|
||||
viewer.reset()
|
||||
assert viewer.plugin == "Datagrid"
|
||||
assert viewer.filter == []
|
||||
assert viewer.expressions == {}
|
||||
|
||||
# restore configuration
|
||||
viewer.restore(**config)
|
||||
assert viewer.filter == [["a", "==", 2]]
|
||||
assert viewer.plugin == "X Bar"
|
||||
assert viewer.expressions == {'"a" * 2': '"a" * 2'}
|
||||
|
||||
def test_save_restore_plugin_config(self):
|
||||
viewer = PerspectiveViewer(
|
||||
plugin="Datagrid", plugin_config={"columns": {"a": {"fixed": 4}}}
|
||||
)
|
||||
config = viewer.save()
|
||||
|
||||
assert config["plugin_config"] == {"columns": {"a": {"fixed": 4}}}
|
||||
|
||||
viewer.reset()
|
||||
assert viewer.plugin_config == {}
|
||||
|
||||
viewer.restore(**config)
|
||||
assert viewer.plugin_config == config["plugin_config"]
|
||||
@@ -0,0 +1,12 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import io
|
||||
import json
|
||||
from decimal import Decimal
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.ipc as ipc
|
||||
|
||||
from perspective import VirtualDataSlice
|
||||
|
||||
|
||||
def round_trip(arrow_table):
|
||||
"""Serialize a PyArrow table to IPC, feed through VirtualDataSlice, read back as JSON."""
|
||||
buf = io.BytesIO()
|
||||
with ipc.new_stream(buf, arrow_table.schema) as writer:
|
||||
writer.write_table(arrow_table)
|
||||
ds = VirtualDataSlice()
|
||||
ds.from_arrow_ipc(buf.getvalue())
|
||||
return json.loads(ds.render_to_columns_json())
|
||||
|
||||
|
||||
class TestCoerceSmallIntegers:
|
||||
def test_coerce_int8(self):
|
||||
table = pa.table({"col": pa.array([-1, 127, None], type=pa.int8())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [-1, 127, None]
|
||||
|
||||
def test_coerce_int16(self):
|
||||
table = pa.table({"col": pa.array([-300, 32000, None], type=pa.int16())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [-300, 32000, None]
|
||||
|
||||
|
||||
class TestCoerceUnsignedIntegers:
|
||||
def test_coerce_uint8(self):
|
||||
table = pa.table({"col": pa.array([0, 255, None], type=pa.uint8())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [0, 255, None]
|
||||
|
||||
def test_coerce_uint16(self):
|
||||
table = pa.table({"col": pa.array([0, 65535, None], type=pa.uint16())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [0, 65535, None]
|
||||
|
||||
def test_coerce_uint32(self):
|
||||
table = pa.table({"col": pa.array([0, 4_294_967_295, None], type=pa.uint32())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [0.0, 4_294_967_295.0, None]
|
||||
|
||||
def test_coerce_uint64(self):
|
||||
table = pa.table({"col": pa.array([0, 1 << 53, None], type=pa.uint64())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [0.0, 9_007_199_254_740_992.0, None]
|
||||
|
||||
|
||||
class TestCoerceFloat32:
|
||||
def test_coerce_float32(self):
|
||||
table = pa.table({"col": pa.array([3.14, -0.0, None], type=pa.float32())})
|
||||
result = round_trip(table)
|
||||
assert abs(result["col"][0] - 3.14) < 0.001
|
||||
assert result["col"][1] == 0.0
|
||||
assert result["col"][2] is None
|
||||
|
||||
|
||||
class TestCoerceDate64:
|
||||
def test_coerce_date64(self):
|
||||
day = 19738
|
||||
table = pa.table({"col": pa.array([day * 86_400_000, None], type=pa.date64())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [day * 86_400_000, None]
|
||||
|
||||
|
||||
class TestCoerceTime:
|
||||
def test_coerce_time32_second(self):
|
||||
table = pa.table({"col": pa.array([49530, None], type=pa.time32("s"))})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [49_530_000, None]
|
||||
|
||||
def test_coerce_time32_millisecond(self):
|
||||
table = pa.table({"col": pa.array([49_530_000, None], type=pa.time32("ms"))})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [49_530_000, None]
|
||||
|
||||
def test_coerce_time64_microsecond(self):
|
||||
table = pa.table(
|
||||
{"col": pa.array([49_530_000_000, None], type=pa.time64("us"))}
|
||||
)
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [49_530_000, None]
|
||||
|
||||
def test_coerce_time64_nanosecond(self):
|
||||
table = pa.table(
|
||||
{"col": pa.array([49_530_000_000_000, None], type=pa.time64("ns"))}
|
||||
)
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [49_530_000, None]
|
||||
|
||||
|
||||
class TestCoerceLargeUtf8:
|
||||
def test_coerce_large_utf8(self):
|
||||
table = pa.table({"col": pa.array(["hello", "", None], type=pa.large_utf8())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == ["hello", "", None]
|
||||
|
||||
|
||||
class TestCoerceDecimal128:
|
||||
def test_coerce_decimal128(self):
|
||||
arr = pa.array([Decimal("1234.5678"), None], type=pa.decimal128(10, 4))
|
||||
table = pa.table({"col": arr})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1234.5678, None]
|
||||
|
||||
|
||||
class TestCoerceInt64:
|
||||
def test_coerce_int64(self):
|
||||
table = pa.table({"col": pa.array([1, -1, None], type=pa.int64())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1.0, -1.0, None]
|
||||
|
||||
|
||||
class TestCoerceTimestamp:
|
||||
def test_coerce_timestamp_second(self):
|
||||
table = pa.table({"col": pa.array([1000, None], type=pa.timestamp("s"))})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1_000_000, None]
|
||||
|
||||
def test_coerce_timestamp_microsecond(self):
|
||||
table = pa.table({"col": pa.array([1_000_000, None], type=pa.timestamp("us"))})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1000, None]
|
||||
|
||||
def test_coerce_timestamp_nanosecond(self):
|
||||
table = pa.table(
|
||||
{"col": pa.array([1_000_000_000, None], type=pa.timestamp("ns"))}
|
||||
)
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1000, None]
|
||||
|
||||
|
||||
class TestPassthrough:
|
||||
def test_passthrough_bool(self):
|
||||
table = pa.table({"col": pa.array([True, False, None], type=pa.bool_())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [True, False, None]
|
||||
|
||||
def test_passthrough_utf8(self):
|
||||
table = pa.table({"col": pa.array(["a", "b", None], type=pa.utf8())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == ["a", "b", None]
|
||||
|
||||
def test_passthrough_float64(self):
|
||||
table = pa.table({"col": pa.array([1.5, -2.5, None], type=pa.float64())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1.5, -2.5, None]
|
||||
|
||||
def test_passthrough_int32(self):
|
||||
table = pa.table({"col": pa.array([1, -1, None], type=pa.int32())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1, -1, None]
|
||||
|
||||
def test_passthrough_date32(self):
|
||||
table = pa.table({"col": pa.array([19738, None], type=pa.date32())})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [19738 * 86_400_000, None]
|
||||
|
||||
def test_passthrough_timestamp_millisecond(self):
|
||||
table = pa.table({"col": pa.array([1000, None], type=pa.timestamp("ms"))})
|
||||
result = round_trip(table)
|
||||
assert result["col"] == [1000, None]
|
||||
|
||||
|
||||
class TestFallback:
|
||||
def test_fallback_fixed_size_binary(self):
|
||||
table = pa.table(
|
||||
{"col": pa.array([b"\x01\x02", b"\x03\x04", None], type=pa.binary(2))}
|
||||
)
|
||||
result = round_trip(table)
|
||||
col = result["col"]
|
||||
assert isinstance(col[0], str)
|
||||
assert isinstance(col[1], str)
|
||||
assert col[0] != col[1]
|
||||
assert col[2] is None
|
||||
|
||||
|
||||
class TestEmpty:
|
||||
def test_empty_batch(self):
|
||||
# PyArrow's new_stream with an empty table writes no record batches,
|
||||
# so we use RecordBatchStreamWriter directly to produce a valid empty batch.
|
||||
schema = pa.schema([("col", pa.int8())])
|
||||
batch = pa.record_batch({"col": pa.array([], type=pa.int8())})
|
||||
buf = io.BytesIO()
|
||||
writer = ipc.RecordBatchStreamWriter(buf, schema)
|
||||
writer.write_batch(batch)
|
||||
writer.close()
|
||||
ds = VirtualDataSlice()
|
||||
ds.from_arrow_ipc(buf.getvalue())
|
||||
result = json.loads(ds.render_to_columns_json())
|
||||
assert result["col"] == []
|
||||
@@ -0,0 +1,908 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
import duckdb
|
||||
|
||||
from perspective import Client
|
||||
from perspective.virtual_servers.duckdb import DuckDBVirtualServer
|
||||
|
||||
_SUPERSTORE_LOCAL = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"node_modules",
|
||||
"superstore-arrow",
|
||||
"superstore.parquet",
|
||||
)
|
||||
|
||||
_SUPERSTORE_URL = (
|
||||
"https://cdn.jsdelivr.net/npm/superstore-arrow@3.2.0/superstore.parquet"
|
||||
)
|
||||
|
||||
|
||||
def _get_superstore_parquet():
|
||||
if os.path.exists(_SUPERSTORE_LOCAL):
|
||||
return _SUPERSTORE_LOCAL
|
||||
path = os.path.join(tempfile.gettempdir(), "superstore.parquet")
|
||||
if not os.path.exists(path):
|
||||
urllib.request.urlretrieve(_SUPERSTORE_URL, path)
|
||||
return path
|
||||
|
||||
|
||||
SUPERSTORE_PARQUET = _get_superstore_parquet()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def client():
|
||||
db = duckdb.connect()
|
||||
db.execute("SET default_null_order=NULLS_FIRST_ON_ASC_LAST_ON_DESC")
|
||||
db.execute(
|
||||
f"CREATE TABLE superstore AS SELECT * FROM read_parquet('{SUPERSTORE_PARQUET}')"
|
||||
)
|
||||
|
||||
server = DuckDBVirtualServer(db)
|
||||
|
||||
def handle_request(msg):
|
||||
session.handle_request(msg)
|
||||
|
||||
def handle_response(msg):
|
||||
c.handle_response(msg)
|
||||
|
||||
session = server.new_session(handle_response)
|
||||
c = Client(handle_request)
|
||||
return c
|
||||
|
||||
|
||||
class TestDuckDBClient:
|
||||
def test_get_hosted_table_names(self, client):
|
||||
tables = client.get_hosted_table_names()
|
||||
assert tables == ["memory.superstore"]
|
||||
|
||||
|
||||
class TestDuckDBTable:
|
||||
def test_schema(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
schema = table.schema()
|
||||
assert schema == {
|
||||
"Product Name": "string",
|
||||
"Ship Date": "date",
|
||||
"City": "string",
|
||||
"Row ID": "integer",
|
||||
"Customer Name": "string",
|
||||
"Quantity": "integer",
|
||||
"Discount": "float",
|
||||
"Sub-Category": "string",
|
||||
"Segment": "string",
|
||||
"Category": "string",
|
||||
"Order Date": "date",
|
||||
"Order ID": "string",
|
||||
"Sales": "float",
|
||||
"State": "string",
|
||||
"Postal Code": "float",
|
||||
"Country": "string",
|
||||
"Customer ID": "string",
|
||||
"Ship Mode": "string",
|
||||
"Region": "string",
|
||||
"Profit": "float",
|
||||
"Product ID": "string",
|
||||
}
|
||||
|
||||
def test_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
columns = table.columns()
|
||||
assert columns == [
|
||||
"Row ID",
|
||||
"Order ID",
|
||||
"Order Date",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"Customer ID",
|
||||
"Customer Name",
|
||||
"Segment",
|
||||
"Country",
|
||||
"City",
|
||||
"State",
|
||||
"Postal Code",
|
||||
"Region",
|
||||
"Product ID",
|
||||
"Category",
|
||||
"Sub-Category",
|
||||
"Product Name",
|
||||
"Sales",
|
||||
"Quantity",
|
||||
"Discount",
|
||||
"Profit",
|
||||
]
|
||||
|
||||
def test_size(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
size = table.size()
|
||||
assert size == 9994
|
||||
|
||||
|
||||
class TestDuckDBView:
|
||||
def test_num_rows(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit"])
|
||||
num_rows = view.num_rows()
|
||||
assert num_rows == 9994
|
||||
view.delete()
|
||||
|
||||
def test_num_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit", "State"])
|
||||
num_columns = view.num_columns()
|
||||
assert num_columns == 3
|
||||
view.delete()
|
||||
|
||||
def test_schema(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit", "State"])
|
||||
schema = view.schema()
|
||||
assert schema == {
|
||||
"Sales": "float",
|
||||
"Profit": "float",
|
||||
"State": "string",
|
||||
}
|
||||
view.delete()
|
||||
|
||||
def test_to_json(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Quantity"])
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Quantity": 2},
|
||||
{"Sales": 731.94, "Quantity": 3},
|
||||
{"Sales": 14.62, "Quantity": 2},
|
||||
{"Sales": 957.5775, "Quantity": 5},
|
||||
{"Sales": 22.368, "Quantity": 2},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_to_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Quantity"])
|
||||
columns = view.to_columns(start_row=0, end_row=5)
|
||||
assert columns == {
|
||||
"Sales": [261.96, 731.94, 14.62, 957.5775, 22.368],
|
||||
"Quantity": [2, 3, 2, 5, 2],
|
||||
}
|
||||
view.delete()
|
||||
|
||||
def test_column_paths(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit", "State"])
|
||||
paths = view.column_paths()
|
||||
assert paths == ["Sales", "Profit", "State"]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBGroupBy:
|
||||
def test_single_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Region"],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
num_rows = view.num_rows()
|
||||
assert num_rows == 5
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 2297200.860299955},
|
||||
{"__ROW_PATH__": ["Central"], "Sales": 501239.8908000005},
|
||||
{"__ROW_PATH__": ["East"], "Sales": 678781.2399999979},
|
||||
{"__ROW_PATH__": ["South"], "Sales": 391721.9050000003},
|
||||
{"__ROW_PATH__": ["West"], "Sales": 725457.8245000006},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_multi_level_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Region", "Category"],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 2297200.860299955},
|
||||
{"__ROW_PATH__": ["Central"], "Sales": 501239.8908000005},
|
||||
{"__ROW_PATH__": ["Central", "Furniture"], "Sales": 163797.16380000004},
|
||||
{
|
||||
"__ROW_PATH__": ["Central", "Office Supplies"],
|
||||
"Sales": 167026.41500000027,
|
||||
},
|
||||
{"__ROW_PATH__": ["Central", "Technology"], "Sales": 170416.3119999999},
|
||||
{"__ROW_PATH__": ["East"], "Sales": 678781.2399999979},
|
||||
{"__ROW_PATH__": ["East", "Furniture"], "Sales": 208291.20400000009},
|
||||
{"__ROW_PATH__": ["East", "Office Supplies"], "Sales": 205516.0549999999},
|
||||
{"__ROW_PATH__": ["East", "Technology"], "Sales": 264973.9810000003},
|
||||
{"__ROW_PATH__": ["South"], "Sales": 391721.9050000003},
|
||||
{"__ROW_PATH__": ["South", "Furniture"], "Sales": 117298.6840000001},
|
||||
{"__ROW_PATH__": ["South", "Office Supplies"], "Sales": 125651.31299999992},
|
||||
{"__ROW_PATH__": ["South", "Technology"], "Sales": 148771.9079999999},
|
||||
{"__ROW_PATH__": ["West"], "Sales": 725457.8245000006},
|
||||
{"__ROW_PATH__": ["West", "Furniture"], "Sales": 252612.7435000003},
|
||||
{"__ROW_PATH__": ["West", "Office Supplies"], "Sales": 220853.24900000007},
|
||||
{"__ROW_PATH__": ["West", "Technology"], "Sales": 251991.83199999997},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_group_by_with_count_aggregate(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Region"],
|
||||
aggregates={"Sales": "count"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 9994},
|
||||
{"__ROW_PATH__": ["Central"], "Sales": 2323},
|
||||
{"__ROW_PATH__": ["East"], "Sales": 2848},
|
||||
{"__ROW_PATH__": ["South"], "Sales": 1620},
|
||||
{"__ROW_PATH__": ["West"], "Sales": 3203},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_group_by_with_avg_aggregate(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Category"],
|
||||
aggregates={"Sales": "avg"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 229.8580008304938},
|
||||
{"__ROW_PATH__": ["Furniture"], "Sales": 349.83488698727007},
|
||||
{"__ROW_PATH__": ["Office Supplies"], "Sales": 119.32410089611732},
|
||||
{"__ROW_PATH__": ["Technology"], "Sales": 452.70927612344155},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_group_by_with_min_aggregate(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Quantity"],
|
||||
group_by=["Region"],
|
||||
aggregates={"Quantity": "min"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Quantity": 1},
|
||||
{"__ROW_PATH__": ["Central"], "Quantity": 1},
|
||||
{"__ROW_PATH__": ["East"], "Quantity": 1},
|
||||
{"__ROW_PATH__": ["South"], "Quantity": 1},
|
||||
{"__ROW_PATH__": ["West"], "Quantity": 1},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_group_by_with_max_aggregate(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Quantity"],
|
||||
group_by=["Region"],
|
||||
aggregates={"Quantity": "max"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Quantity": 14},
|
||||
{"__ROW_PATH__": ["Central"], "Quantity": 14},
|
||||
{"__ROW_PATH__": ["East"], "Quantity": 14},
|
||||
{"__ROW_PATH__": ["South"], "Quantity": 14},
|
||||
{"__ROW_PATH__": ["West"], "Quantity": 14},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBSplitBy:
|
||||
def test_single_split_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
split_by=["Region"],
|
||||
group_by=["Category"],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
|
||||
column_paths = view.column_paths()
|
||||
assert column_paths == [
|
||||
"Central_Sales",
|
||||
"East_Sales",
|
||||
"South_Sales",
|
||||
"West_Sales",
|
||||
]
|
||||
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{
|
||||
"__ROW_PATH__": [],
|
||||
"Central|Sales": 501239.8908000005,
|
||||
"East|Sales": 678781.2399999979,
|
||||
"South|Sales": 391721.9050000003,
|
||||
"West|Sales": 725457.8245000006,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Furniture"],
|
||||
"Central|Sales": 163797.16380000004,
|
||||
"East|Sales": 208291.20400000009,
|
||||
"South|Sales": 117298.6840000001,
|
||||
"West|Sales": 252612.7435000003,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Office Supplies"],
|
||||
"Central|Sales": 167026.41500000027,
|
||||
"East|Sales": 205516.0549999999,
|
||||
"South|Sales": 125651.31299999992,
|
||||
"West|Sales": 220853.24900000007,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Technology"],
|
||||
"Central|Sales": 170416.3119999999,
|
||||
"East|Sales": 264973.9810000003,
|
||||
"South|Sales": 148771.9079999999,
|
||||
"West|Sales": 251991.83199999997,
|
||||
},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_split_by_without_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
split_by=["Category"],
|
||||
)
|
||||
paths = view.column_paths()
|
||||
assert any("Furniture" in c for c in paths)
|
||||
assert any("Office Supplies" in c for c in paths)
|
||||
assert any("Technology" in c for c in paths)
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBFilter:
|
||||
def test_filter_with_equals(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Region"],
|
||||
filter=[["Region", "==", "West"]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 14.62, "Region": "West"},
|
||||
{"Sales": 48.86, "Region": "West"},
|
||||
{"Sales": 7.28, "Region": "West"},
|
||||
{"Sales": 907.152, "Region": "West"},
|
||||
{"Sales": 18.504, "Region": "West"},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_not_equals(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Region"],
|
||||
filter=[["Region", "!=", "West"]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Region": "South"},
|
||||
{"Sales": 731.94, "Region": "South"},
|
||||
{"Sales": 957.5775, "Region": "South"},
|
||||
{"Sales": 22.368, "Region": "South"},
|
||||
{"Sales": 15.552, "Region": "South"},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_greater_than(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
filter=[["Quantity", ">", 5]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 48.86, "Quantity": 7},
|
||||
{"Sales": 907.152, "Quantity": 6},
|
||||
{"Sales": 1706.184, "Quantity": 9},
|
||||
{"Sales": 665.88, "Quantity": 6},
|
||||
{"Sales": 19.46, "Quantity": 7},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_less_than(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
filter=[["Quantity", "<", 3]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Quantity": 2},
|
||||
{"Sales": 14.62, "Quantity": 2},
|
||||
{"Sales": 22.368, "Quantity": 2},
|
||||
{"Sales": 55.5, "Quantity": 2},
|
||||
{"Sales": 8.56, "Quantity": 2},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_greater_than_or_equal(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
filter=[["Quantity", ">=", 10]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 40.096, "Quantity": 14},
|
||||
{"Sales": 43.12, "Quantity": 14},
|
||||
{"Sales": 384.45, "Quantity": 11},
|
||||
{"Sales": 3347.37, "Quantity": 13},
|
||||
{"Sales": 100.24, "Quantity": 10},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_less_than_or_equal(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
filter=[["Quantity", "<=", 2]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Quantity": 2},
|
||||
{"Sales": 14.62, "Quantity": 2},
|
||||
{"Sales": 22.368, "Quantity": 2},
|
||||
{"Sales": 55.5, "Quantity": 2},
|
||||
{"Sales": 8.56, "Quantity": 2},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_like(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "State"],
|
||||
filter=[["State", "LIKE", "Cal%"]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 14.62, "State": "California"},
|
||||
{"Sales": 48.86, "State": "California"},
|
||||
{"Sales": 7.28, "State": "California"},
|
||||
{"Sales": 907.152, "State": "California"},
|
||||
{"Sales": 18.504, "State": "California"},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_multiple_filters(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Region", "Quantity"],
|
||||
filter=[
|
||||
["Region", "==", "West"],
|
||||
["Quantity", ">", 3],
|
||||
],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 48.86, "Region": "West", "Quantity": 7},
|
||||
{"Sales": 7.28, "Region": "West", "Quantity": 4},
|
||||
{"Sales": 907.152, "Region": "West", "Quantity": 6},
|
||||
{"Sales": 114.9, "Region": "West", "Quantity": 5},
|
||||
{"Sales": 1706.184, "Region": "West", "Quantity": 9},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_filter_with_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Category"],
|
||||
filter=[["Region", "==", "West"]],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
num_rows = view.num_rows()
|
||||
assert num_rows == 4
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 725457.8245000006},
|
||||
{"__ROW_PATH__": ["Furniture"], "Sales": 252612.7435000003},
|
||||
{"__ROW_PATH__": ["Office Supplies"], "Sales": 220853.24900000007},
|
||||
{"__ROW_PATH__": ["Technology"], "Sales": 251991.83199999997},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBSort:
|
||||
def test_sort_ascending(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
sort=[["Sales", "asc"]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 0.444, "Quantity": 1},
|
||||
{"Sales": 0.556, "Quantity": 1},
|
||||
{"Sales": 0.836, "Quantity": 1},
|
||||
{"Sales": 0.852, "Quantity": 1},
|
||||
{"Sales": 0.876, "Quantity": 1},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_sort_descending(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Quantity"],
|
||||
sort=[["Sales", "desc"]],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 22638.48, "Quantity": 6},
|
||||
{"Sales": 17499.95, "Quantity": 5},
|
||||
{"Sales": 13999.96, "Quantity": 4},
|
||||
{"Sales": 11199.968, "Quantity": 4},
|
||||
{"Sales": 10499.97, "Quantity": 3},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_sort_with_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Region"],
|
||||
sort=[["Sales", "desc"]],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 2297200.860299955},
|
||||
{"__ROW_PATH__": ["West"], "Sales": 725457.8245000006},
|
||||
{"__ROW_PATH__": ["East"], "Sales": 678781.2399999979},
|
||||
{"__ROW_PATH__": ["Central"], "Sales": 501239.8908000005},
|
||||
{"__ROW_PATH__": ["South"], "Sales": 391721.9050000003},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_multi_column_sort(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Region", "Sales", "Quantity"],
|
||||
sort=[
|
||||
["Region", "asc"],
|
||||
["Sales", "desc"],
|
||||
],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Region": "Central", "Sales": 17499.95, "Quantity": 5},
|
||||
{"Region": "Central", "Sales": 9892.74, "Quantity": 13},
|
||||
{"Region": "Central", "Sales": 9449.95, "Quantity": 5},
|
||||
{"Region": "Central", "Sales": 8159.952, "Quantity": 8},
|
||||
{"Region": "Central", "Sales": 5443.96, "Quantity": 4},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBExpressions:
|
||||
def test_simple_expression(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "doublesales"],
|
||||
expressions={"doublesales": '"Sales" * 2'},
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "doublesales": 523.92},
|
||||
{"Sales": 731.94, "doublesales": 1463.88},
|
||||
{"Sales": 14.62, "doublesales": 29.24},
|
||||
{"Sales": 957.5775, "doublesales": 1915.155},
|
||||
{"Sales": 22.368, "doublesales": 44.736},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_expression_with_multiple_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Profit", "margin"],
|
||||
expressions={"margin": '"Profit" / "Sales"'},
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Profit": 41.9136, "margin": 0.16000000000000003},
|
||||
{"Sales": 731.94, "Profit": 219.582, "margin": 0.3},
|
||||
{"Sales": 14.62, "Profit": 6.8714, "margin": 0.47000000000000003},
|
||||
{"Sales": 957.5775, "Profit": -383.031, "margin": -0.4},
|
||||
{"Sales": 22.368, "Profit": 2.5164, "margin": 0.1125},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_expression_with_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["total"],
|
||||
group_by=["Region"],
|
||||
expressions={"total": '"Sales" + "Profit"'},
|
||||
aggregates={"total": "sum"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "total": 2583597.882000014},
|
||||
{"__ROW_PATH__": ["Central"], "total": 540946.2532999996},
|
||||
{"__ROW_PATH__": ["East"], "total": 770304.0199999991},
|
||||
{"__ROW_PATH__": ["South"], "total": 438471.33530000027},
|
||||
{"__ROW_PATH__": ["West"], "total": 833876.2733999988},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBViewport:
|
||||
def test_start_row_and_end_row(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit"])
|
||||
json = view.to_json(start_row=10, end_row=15)
|
||||
assert json == [
|
||||
{"Sales": 1706.184, "Profit": 85.3092},
|
||||
{"Sales": 911.424, "Profit": 68.3568},
|
||||
{"Sales": 15.552, "Profit": 5.4432},
|
||||
{"Sales": 407.976, "Profit": 132.5922},
|
||||
{"Sales": 68.81, "Profit": -123.858},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_start_col_and_end_col(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales", "Profit", "Quantity", "Discount"],
|
||||
)
|
||||
json = view.to_json(start_row=0, end_row=5, start_col=1, end_col=3)
|
||||
assert json == [
|
||||
{"Profit": 41.9136, "Quantity": 2},
|
||||
{"Profit": 219.582, "Quantity": 3},
|
||||
{"Profit": 6.8714, "Quantity": 2},
|
||||
{"Profit": -383.031, "Quantity": 5},
|
||||
{"Profit": 2.5164, "Quantity": 2},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBDataTypes:
|
||||
def test_integer_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Quantity"])
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Quantity": 2},
|
||||
{"Quantity": 3},
|
||||
{"Quantity": 2},
|
||||
{"Quantity": 5},
|
||||
{"Quantity": 2},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_float_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales", "Profit"])
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Sales": 261.96, "Profit": 41.9136},
|
||||
{"Sales": 731.94, "Profit": 219.582},
|
||||
{"Sales": 14.62, "Profit": 6.8714},
|
||||
{"Sales": 957.5775, "Profit": -383.031},
|
||||
{"Sales": 22.368, "Profit": 2.5164},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_string_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Region", "State", "City"])
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Region": "South", "State": "Kentucky", "City": "Henderson"},
|
||||
{"Region": "South", "State": "Kentucky", "City": "Henderson"},
|
||||
{"Region": "West", "State": "California", "City": "Los Angeles"},
|
||||
{"Region": "South", "State": "Florida", "City": "Fort Lauderdale"},
|
||||
{"Region": "South", "State": "Florida", "City": "Fort Lauderdale"},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_date_columns(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Order Date"])
|
||||
json = view.to_json(start_row=0, end_row=5)
|
||||
assert json == [
|
||||
{"Order Date": 1478563200000},
|
||||
{"Order Date": 1478563200000},
|
||||
{"Order Date": 1465689600000},
|
||||
{"Order Date": 1444521600000},
|
||||
{"Order Date": 1444521600000},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBCombinedOperations:
|
||||
def test_group_by_filter_sort(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Category"],
|
||||
filter=[["Region", "==", "West"]],
|
||||
sort=[["Sales", "desc"]],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "Sales": 725457.8245000006},
|
||||
{"__ROW_PATH__": ["Furniture"], "Sales": 252612.7435000003},
|
||||
{"__ROW_PATH__": ["Technology"], "Sales": 251991.83199999997},
|
||||
{"__ROW_PATH__": ["Office Supplies"], "Sales": 220853.24900000007},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_split_by_group_by_filter(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Category"],
|
||||
split_by=["Region"],
|
||||
filter=[["Quantity", ">", 3]],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
|
||||
paths = view.column_paths()
|
||||
assert paths == [
|
||||
"Central_Sales",
|
||||
"East_Sales",
|
||||
"South_Sales",
|
||||
"West_Sales",
|
||||
]
|
||||
|
||||
num_rows = view.num_rows()
|
||||
assert num_rows == 4
|
||||
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{
|
||||
"__ROW_PATH__": [],
|
||||
"Central|Sales": 332883.0567999998,
|
||||
"East|Sales": 455143.735,
|
||||
"South|Sales": 274208.7699999999,
|
||||
"West|Sales": 470561.28350000136,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Furniture"],
|
||||
"Central|Sales": 111457.73279999988,
|
||||
"East|Sales": 140376.95899999997,
|
||||
"South|Sales": 80859.618,
|
||||
"West|Sales": 165219.5734999998,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Office Supplies"],
|
||||
"Central|Sales": 103937.78599999992,
|
||||
"East|Sales": 135823.893,
|
||||
"South|Sales": 84393.3579999999,
|
||||
"West|Sales": 140206.93099999975,
|
||||
},
|
||||
{
|
||||
"__ROW_PATH__": ["Technology"],
|
||||
"Central|Sales": 117487.53800000002,
|
||||
"East|Sales": 178942.883,
|
||||
"South|Sales": 108955.79400000005,
|
||||
"West|Sales": 165134.77900000007,
|
||||
},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_split_by_only(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
split_by=["Region"],
|
||||
filter=[["Quantity", ">", 3]],
|
||||
)
|
||||
|
||||
paths = view.column_paths()
|
||||
assert paths == [
|
||||
"Central_Sales",
|
||||
"East_Sales",
|
||||
"South_Sales",
|
||||
"West_Sales",
|
||||
]
|
||||
|
||||
num_rows = view.num_rows()
|
||||
assert num_rows == 4284
|
||||
|
||||
json = view.to_json(start_row=0, end_row=1)
|
||||
assert json == [
|
||||
{
|
||||
"Central|Sales": None,
|
||||
"East|Sales": None,
|
||||
"South|Sales": 957.5775,
|
||||
"West|Sales": None,
|
||||
},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
def test_expressions_group_by_sort(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["profitmargin"],
|
||||
group_by=["Region"],
|
||||
expressions={"profitmargin": '"Profit" / "Sales" * 100'},
|
||||
sort=[["profitmargin", "desc"]],
|
||||
aggregates={"profitmargin": "avg"},
|
||||
)
|
||||
json = view.to_json()
|
||||
assert json == [
|
||||
{"__ROW_PATH__": [], "profitmargin": 12.031392972104467},
|
||||
{"__ROW_PATH__": ["West"], "profitmargin": 21.948661793784012},
|
||||
{"__ROW_PATH__": ["East"], "profitmargin": 16.722695960406636},
|
||||
{"__ROW_PATH__": ["South"], "profitmargin": 16.35190329218107},
|
||||
{"__ROW_PATH__": ["Central"], "profitmargin": -10.407293926323575},
|
||||
]
|
||||
view.delete()
|
||||
|
||||
|
||||
class TestDuckDBMinMax:
|
||||
def test_min_max_integer(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Quantity"])
|
||||
min_val, max_val = view.get_min_max("Quantity")
|
||||
assert min_val == 1
|
||||
assert max_val == 14
|
||||
view.delete()
|
||||
|
||||
def test_min_max_float(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Sales"])
|
||||
min_val, max_val = view.get_min_max("Sales")
|
||||
assert min_val == 0.444
|
||||
assert max_val == 22638.48
|
||||
view.delete()
|
||||
|
||||
def test_min_max_string(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(columns=["Category"])
|
||||
min_val, max_val = view.get_min_max("Category")
|
||||
assert min_val == "Furniture"
|
||||
assert max_val == "Technology"
|
||||
view.delete()
|
||||
|
||||
def test_min_max_with_group_by(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Sales"],
|
||||
group_by=["Region"],
|
||||
aggregates={"Sales": "sum"},
|
||||
)
|
||||
min_val, max_val = view.get_min_max("Sales")
|
||||
assert min_val > 0
|
||||
assert max_val > 0
|
||||
assert max_val >= min_val
|
||||
view.delete()
|
||||
|
||||
def test_min_max_with_filter(self, client):
|
||||
table = client.open_table("memory.superstore")
|
||||
view = table.view(
|
||||
columns=["Quantity"],
|
||||
filter=[["Quantity", ">", 10]],
|
||||
)
|
||||
min_val, max_val = view.get_min_max("Quantity")
|
||||
assert min_val >= 11
|
||||
assert max_val == 14
|
||||
view.delete()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
@@ -0,0 +1,278 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from functools import partial
|
||||
from types import MethodType
|
||||
import numpy as np
|
||||
from perspective.widget import PerspectiveWidget
|
||||
from pytest import raises
|
||||
import pytest
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
pytest.skip(allow_module_level=True)
|
||||
|
||||
|
||||
def mock_post(self, msg, msg_id=None, assert_msg=None):
|
||||
"""Mock the widget's `post()` method so we can introspect the contents."""
|
||||
assert msg == assert_msg
|
||||
|
||||
|
||||
class TestWidget:
|
||||
def test_widget(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {}},
|
||||
}
|
||||
|
||||
def test_widget_indexed(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data, plugin="X Bar", index="a")
|
||||
assert widget.plugin == "X Bar"
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {"index": "a"}},
|
||||
}
|
||||
|
||||
def test_widget_no_data(self):
|
||||
widget = PerspectiveWidget(None, plugin="X Bar", group_by=["a"])
|
||||
assert widget.plugin == "X Bar"
|
||||
assert widget.group_by == ["a"]
|
||||
|
||||
def test_widget_schema(self):
|
||||
schema = {
|
||||
"a": "integer",
|
||||
"b": "float",
|
||||
"c": "boolean",
|
||||
"d": "date",
|
||||
"e": "datetime",
|
||||
"f": "string",
|
||||
}
|
||||
widget = PerspectiveWidget(schema)
|
||||
assert widget.table.schema() == schema
|
||||
|
||||
def test_widget_schema_with_index(self):
|
||||
widget = PerspectiveWidget({"a": "integer"}, index="a")
|
||||
assert widget.table.get_index() == "a"
|
||||
|
||||
def test_widget_schema_with_limit(self):
|
||||
widget = PerspectiveWidget({"a": "integer"}, limit=5)
|
||||
assert widget.table.get_limit() == 5
|
||||
|
||||
def test_widget_no_data_with_index(self):
|
||||
# should fail
|
||||
with raises(TypeError):
|
||||
PerspectiveWidget(None, index="a")
|
||||
|
||||
def test_widget_no_data_with_limit(self):
|
||||
# should fail
|
||||
with raises(TypeError):
|
||||
PerspectiveWidget(None, limit=5)
|
||||
|
||||
def test_widget_eventual_data(self):
|
||||
table = Table({"a": np.arange(0, 50)})
|
||||
widget = PerspectiveWidget(None, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
|
||||
with raises(TypeError):
|
||||
widget._make_load_message()
|
||||
|
||||
widget.load(table)
|
||||
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {}},
|
||||
}
|
||||
|
||||
def test_widget_eventual_data_server(self):
|
||||
widget = PerspectiveWidget(None, plugin="X Bar", server=True)
|
||||
assert widget.plugin == "X Bar"
|
||||
widget.load({"a": np.arange(0, 50)}, index="a")
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {
|
||||
"table_name": widget.table_name,
|
||||
},
|
||||
}
|
||||
|
||||
def test_widget_eventual_data_indexed(self):
|
||||
widget = PerspectiveWidget(None, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
widget.load({"a": np.arange(0, 50)}, index="a")
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {"index": "a"}},
|
||||
}
|
||||
|
||||
def test_widget_eventual_table_indexed(self):
|
||||
table = Table({"a": np.arange(0, 50)}, index="a")
|
||||
widget = PerspectiveWidget(None, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
widget.load(table)
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {"index": "a"}},
|
||||
}
|
||||
|
||||
def test_widget_load_table(self):
|
||||
table = Table({"a": np.arange(0, 50)})
|
||||
widget = PerspectiveWidget(table, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {}},
|
||||
}
|
||||
|
||||
def test_widget_load_table_indexed(self):
|
||||
table = Table({"a": np.arange(0, 50)}, index="a")
|
||||
widget = PerspectiveWidget(table, plugin="X Bar")
|
||||
assert widget.plugin == "X Bar"
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name, "options": {"index": "a"}},
|
||||
}
|
||||
|
||||
def test_widget_load_table_ignore_limit(self):
|
||||
table = Table({"a": np.arange(0, 50)})
|
||||
widget = PerspectiveWidget(table, limit=1)
|
||||
assert widget.table.size() == 50
|
||||
|
||||
def test_widget_pass_index(self):
|
||||
data = {"a": [1, 2, 3, 1]}
|
||||
widget = PerspectiveWidget(data, index="a")
|
||||
assert widget.table.size() == 3
|
||||
|
||||
def test_widget_pass_limit(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data, limit=1)
|
||||
assert widget.table.size() == 1
|
||||
|
||||
def test_widget_pass_options_invalid(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
with raises(TypeError):
|
||||
PerspectiveWidget(data, index="index", limit=1)
|
||||
|
||||
# server mode
|
||||
def test_widget_load_table_server(self):
|
||||
table = Table({"a": np.arange(0, 50)})
|
||||
widget = PerspectiveWidget(table, server=True)
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name},
|
||||
}
|
||||
|
||||
def test_widget_no_data_with_server(self):
|
||||
# should fail
|
||||
widget = PerspectiveWidget(None, server=True)
|
||||
with raises(TypeError):
|
||||
widget._make_load_message()
|
||||
|
||||
def test_widget_eventual_data_with_server(self):
|
||||
# should fail
|
||||
widget = PerspectiveWidget(None, server=True)
|
||||
|
||||
with raises(TypeError):
|
||||
widget._make_load_message()
|
||||
|
||||
# then succeed
|
||||
widget.load(Table({"a": np.arange(0, 50)}))
|
||||
load_msg = widget._make_load_message()
|
||||
assert load_msg.to_columns() == {
|
||||
"id": -2,
|
||||
"type": "table",
|
||||
"data": {"table_name": widget.table_name},
|
||||
}
|
||||
|
||||
# clear
|
||||
|
||||
def test_widget_clear(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data)
|
||||
widget.clear()
|
||||
assert widget.table.size() == 0
|
||||
|
||||
def test_widget_clear_server(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data, server=True)
|
||||
widget.clear()
|
||||
assert widget.table.size() == 0
|
||||
|
||||
# replace
|
||||
|
||||
def test_widget_replace(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data)
|
||||
widget.replace({"a": [1]})
|
||||
assert widget.table.size() == 1
|
||||
|
||||
def test_widget_replace_server(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data, server=True)
|
||||
widget.replace({"a": [1]})
|
||||
assert widget.table.size() == 1
|
||||
|
||||
# delete
|
||||
|
||||
def test_widget_delete(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data)
|
||||
mocked_post = partial(mock_post, assert_msg={"cmd": "delete"})
|
||||
widget.post = MethodType(mocked_post, widget)
|
||||
widget.delete()
|
||||
assert widget.table is None
|
||||
|
||||
def test_widget_delete_with_view(self):
|
||||
data = {"a": np.arange(0, 50)}
|
||||
widget = PerspectiveWidget(data)
|
||||
|
||||
# create a view on the manager
|
||||
table_name, table = list(widget.manager._tables.items())[0]
|
||||
make_view_message = {
|
||||
"id": 1,
|
||||
"table_name": table_name,
|
||||
"view_name": "view1",
|
||||
"cmd": "view",
|
||||
"config": {"group_by": ["a"]},
|
||||
}
|
||||
widget.manager._process(make_view_message, lambda x: True)
|
||||
|
||||
assert len(widget.manager._views) == 1
|
||||
|
||||
mocked_post = partial(mock_post, assert_msg={"cmd": "delete"})
|
||||
|
||||
widget.post = MethodType(mocked_post, widget)
|
||||
widget.delete()
|
||||
|
||||
assert widget.table is None
|
||||
@@ -0,0 +1,453 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
import pytest
|
||||
from perspective.widget import PerspectiveWidget
|
||||
import perspective as psp
|
||||
|
||||
client = psp.Server().new_local_client()
|
||||
Table = client.table
|
||||
|
||||
|
||||
pytest.skip(allow_module_level=True)
|
||||
|
||||
|
||||
class TestWidgetPandas:
|
||||
def test_widget_load_table_df(self, superstore):
|
||||
table = Table(superstore)
|
||||
widget = PerspectiveWidget(table)
|
||||
assert widget.table.schema() == {
|
||||
"index": "integer",
|
||||
"Country": "string",
|
||||
"Region": "string",
|
||||
"Category": "string",
|
||||
"City": "string",
|
||||
"Customer ID": "string",
|
||||
"Discount": "float",
|
||||
"Order Date": "date",
|
||||
"Order ID": "string",
|
||||
"Postal Code": "string",
|
||||
"Product ID": "string",
|
||||
"Profit": "float",
|
||||
"Quantity": "integer",
|
||||
"Row ID": "integer",
|
||||
"Sales": "integer",
|
||||
"Segment": "string",
|
||||
"Ship Date": "date",
|
||||
"Ship Mode": "string",
|
||||
"State": "string",
|
||||
"Sub-Category": "string",
|
||||
}
|
||||
|
||||
assert sorted(widget.columns) == sorted(
|
||||
[
|
||||
"index",
|
||||
"Category",
|
||||
"City",
|
||||
"Country",
|
||||
"Customer ID",
|
||||
"Discount",
|
||||
"Order Date",
|
||||
"Order ID",
|
||||
"Postal Code",
|
||||
"Product ID",
|
||||
"Profit",
|
||||
"Quantity",
|
||||
"Region",
|
||||
"Row ID",
|
||||
"Sales",
|
||||
"Segment",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"State",
|
||||
"Sub-Category",
|
||||
]
|
||||
)
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == len(superstore)
|
||||
assert view.num_columns() == len(superstore.columns) + 1 # index
|
||||
|
||||
def test_widget_load_data_df(self, superstore):
|
||||
widget = PerspectiveWidget(superstore)
|
||||
assert sorted(widget.columns) == sorted(
|
||||
[
|
||||
"index",
|
||||
"Category",
|
||||
"City",
|
||||
"Country",
|
||||
"Customer ID",
|
||||
"Discount",
|
||||
"Order Date",
|
||||
"Order ID",
|
||||
"Postal Code",
|
||||
"Product ID",
|
||||
"Profit",
|
||||
"Quantity",
|
||||
"Region",
|
||||
"Row ID",
|
||||
"Sales",
|
||||
"Segment",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"State",
|
||||
"Sub-Category",
|
||||
]
|
||||
)
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == len(superstore)
|
||||
assert view.num_columns() == 20
|
||||
|
||||
def test_widget_load_series(self, superstore):
|
||||
series = pd.Series(superstore["Profit"].values, name="profit")
|
||||
widget = PerspectiveWidget(series)
|
||||
assert widget.table.schema() == {"index": "integer", "profit": "float"}
|
||||
|
||||
assert sorted(widget.columns) == sorted(["index", "profit"])
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == len(superstore)
|
||||
assert view.num_columns() == 2
|
||||
|
||||
def test_widget_load_pivot_table(self, superstore):
|
||||
pivot_table = pd.pivot_table(
|
||||
superstore,
|
||||
values="Discount",
|
||||
index=["Country", "Region"],
|
||||
columns=["Category", "Segment"],
|
||||
)
|
||||
widget = PerspectiveWidget(pivot_table)
|
||||
assert widget.group_by == ["Country", "Region"]
|
||||
assert widget.split_by == ["Category", "Segment"]
|
||||
assert widget.columns == ["value"]
|
||||
# table should host flattened data
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == 60
|
||||
assert view.num_columns() == 6
|
||||
|
||||
def test_widget_load_pivot_table_with_user_pivots(self, superstore):
|
||||
pivot_table = pd.pivot_table(
|
||||
superstore,
|
||||
values="Discount",
|
||||
index=["Country", "Region"],
|
||||
columns="Category",
|
||||
)
|
||||
widget = PerspectiveWidget(pivot_table, group_by=["Category", "Segment"])
|
||||
assert widget.group_by == ["Category", "Segment"]
|
||||
assert widget.split_by == []
|
||||
assert widget.columns == [
|
||||
"index",
|
||||
"Country",
|
||||
"Region",
|
||||
"Financials",
|
||||
"Industrials",
|
||||
"Technology",
|
||||
]
|
||||
# table should host flattened data
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == 5
|
||||
assert view.num_columns() == 6
|
||||
|
||||
def test_widget_load_group_by(self, superstore):
|
||||
df_pivoted = superstore.set_index(["Country", "Region"])
|
||||
widget = PerspectiveWidget(df_pivoted)
|
||||
assert widget.group_by == ["Country", "Region"]
|
||||
assert widget.split_by == []
|
||||
assert sorted(widget.columns) == sorted(
|
||||
[
|
||||
"index",
|
||||
"Category",
|
||||
"Country",
|
||||
"City",
|
||||
"Customer ID",
|
||||
"Discount",
|
||||
"Order Date",
|
||||
"Order ID",
|
||||
"Postal Code",
|
||||
"Product ID",
|
||||
"Profit",
|
||||
"Quantity",
|
||||
"Region",
|
||||
"Row ID",
|
||||
"Sales",
|
||||
"Segment",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"State",
|
||||
"Sub-Category",
|
||||
]
|
||||
)
|
||||
assert widget.table.size() == 100
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == len(superstore)
|
||||
assert view.num_columns() == len(superstore.columns) + 1 # index
|
||||
|
||||
def test_widget_load_group_by_with_user_pivots(self, superstore):
|
||||
df_pivoted = superstore.set_index(["Country", "Region"])
|
||||
widget = PerspectiveWidget(df_pivoted, group_by=["Category", "Segment"])
|
||||
assert widget.group_by == ["Category", "Segment"]
|
||||
assert widget.split_by == []
|
||||
assert sorted(widget.columns) == sorted(
|
||||
[
|
||||
"index",
|
||||
"Category",
|
||||
"Country",
|
||||
"City",
|
||||
"Customer ID",
|
||||
"Discount",
|
||||
"Order Date",
|
||||
"Order ID",
|
||||
"Postal Code",
|
||||
"Product ID",
|
||||
"Profit",
|
||||
"Quantity",
|
||||
"Region",
|
||||
"Row ID",
|
||||
"Sales",
|
||||
"Segment",
|
||||
"Ship Date",
|
||||
"Ship Mode",
|
||||
"State",
|
||||
"Sub-Category",
|
||||
]
|
||||
)
|
||||
assert widget.table.size() == 100
|
||||
view = widget.table.view()
|
||||
assert view.num_rows() == len(superstore)
|
||||
assert view.num_columns() == len(superstore.columns) + 1 # index
|
||||
|
||||
def test_widget_load_split_by(self, superstore):
|
||||
arrays = [
|
||||
np.array(
|
||||
[
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
]
|
||||
),
|
||||
np.array(
|
||||
[
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
]
|
||||
),
|
||||
np.array(
|
||||
[
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
]
|
||||
),
|
||||
]
|
||||
tuples = list(zip(*arrays))
|
||||
index = pd.MultiIndex.from_tuples(tuples, names=["first", "second", "third"])
|
||||
df_both = pd.DataFrame(
|
||||
np.random.randn(3, 16), index=["A", "B", "C"], columns=index
|
||||
)
|
||||
widget = PerspectiveWidget(df_both)
|
||||
assert widget.columns == ["value"]
|
||||
assert widget.split_by == ["first", "second", "third"]
|
||||
assert widget.group_by == ["index"]
|
||||
|
||||
def test_widget_load_split_by_preserve_user_settings(self, superstore):
|
||||
arrays = [
|
||||
np.array(
|
||||
[
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
]
|
||||
),
|
||||
np.array(
|
||||
[
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
]
|
||||
),
|
||||
np.array(
|
||||
[
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
]
|
||||
),
|
||||
]
|
||||
tuples = list(zip(*arrays))
|
||||
index = pd.MultiIndex.from_tuples(tuples, names=["first", "second", "third"])
|
||||
df_both = pd.DataFrame(
|
||||
np.random.randn(3, 16), index=["A", "B", "C"], columns=index
|
||||
)
|
||||
widget = PerspectiveWidget(df_both, columns=["first", "third"])
|
||||
assert widget.columns == ["first", "third"]
|
||||
assert widget.split_by == ["first", "second", "third"]
|
||||
assert widget.group_by == ["index"]
|
||||
|
||||
def test_pivottable_values_index(self, superstore):
|
||||
arrays = {
|
||||
"A": [
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"bar",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"baz",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"foo",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
"qux",
|
||||
],
|
||||
"B": [
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
"one",
|
||||
"one",
|
||||
"two",
|
||||
"two",
|
||||
],
|
||||
"C": [
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
"X",
|
||||
"Y",
|
||||
],
|
||||
"D": np.arange(16),
|
||||
}
|
||||
|
||||
df = pd.DataFrame(arrays)
|
||||
df_pivot = df.pivot_table(
|
||||
values=["D"], index=["A"], columns=["B", "C"], aggfunc={"D": "count"}
|
||||
)
|
||||
widget = PerspectiveWidget(df_pivot)
|
||||
assert widget.columns == ["value"]
|
||||
assert widget.split_by == ["B", "C"]
|
||||
assert widget.group_by == ["A"]
|
||||
|
||||
def test_pivottable_multi_values(self, superstore):
|
||||
pt = pd.pivot_table(
|
||||
superstore,
|
||||
values=["Discount", "Sales"],
|
||||
index=["Country", "Region"],
|
||||
aggfunc={"Discount": "count", "Sales": "sum"},
|
||||
columns=["State", "Quantity"],
|
||||
)
|
||||
widget = PerspectiveWidget(pt)
|
||||
assert widget.columns == ["Discount", "Sales"]
|
||||
assert widget.split_by == ["State", "Quantity"]
|
||||
assert widget.group_by == ["Country", "Region"]
|
||||
@@ -0,0 +1,142 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
|
||||
class VirtualServerHandler:
|
||||
"""
|
||||
An interface for implementing a Perspective `VirtualServer`. It operates
|
||||
thusly:
|
||||
|
||||
- A table is selected by name (validated via `get_hosted_tables`).
|
||||
|
||||
- The UI will ask the model to create a temporary table with the results
|
||||
of querying this table with a specific query `config`, a simple struct
|
||||
which reflects the UI configurable fields (see `get_features`).
|
||||
|
||||
- The UI will query slices of the temporary table as it needs them to
|
||||
render. This may be a rectangular slice, a whole column or the entire
|
||||
set, and it is returned from teh model via a custom push-only
|
||||
struct `PerspectiveColumn` for now, though in the future we will support
|
||||
e.g. Polars and other arrow-native formats directly.
|
||||
|
||||
- The UI will delete its own temporary tables via `view_delete` but it is
|
||||
ok for them to die intermittently, the UI will recover automatically.
|
||||
"""
|
||||
|
||||
def get_features(self):
|
||||
"""
|
||||
[OPTIONAL] Toggle UI features through data model support. For example,
|
||||
setting `"group_by": False` would hide the "Group By" UI control, as
|
||||
well as prevent this field from appearing in `config` dicts later
|
||||
provided to `table_make_view`.
|
||||
|
||||
This API defaults to just "columns", e.g. a simple flat datagrid in
|
||||
which you can just scroll, select and format columns.
|
||||
|
||||
# Example
|
||||
|
||||
```python
|
||||
return {
|
||||
"group_by": True,
|
||||
"split_by": True,
|
||||
"sort": True,
|
||||
"expressions": True,
|
||||
"filter_ops": {
|
||||
"integer": ["==", "<"],
|
||||
},
|
||||
"aggregates": {
|
||||
"string": ["count"],
|
||||
"float": ["count", "sum"],
|
||||
},
|
||||
}
|
||||
```
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def get_hosted_tables(self) -> list[str]:
|
||||
"""
|
||||
List of `Table` names available to query from.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def table_schema(self, table_name):
|
||||
"""
|
||||
Get the _Perspective Schema_ for a `Table`, a mapping of column name to
|
||||
Perspective column types, a simplified set of six visually-relevant
|
||||
types mapped from DuckDB's much richer type system. Optionally,
|
||||
a model may also implement `view_schema` which describes temporary
|
||||
tables, but for DuckDB this method is identical.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def table_size(self, table_name):
|
||||
"""
|
||||
Get a table's row count. Optionally, a model may also implement the
|
||||
`view_size` method to get the row count for temporary tables, but for
|
||||
DuckDB this method is identical.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def view_schema(self, view_name, config):
|
||||
return self.table_schema(view_name)
|
||||
|
||||
def view_size(self, view_name):
|
||||
return self.table_size(view_name)
|
||||
|
||||
def table_make_view(self, table_name, view_name, config):
|
||||
"""
|
||||
Create a temporary table `view_name` from the results of querying
|
||||
`table_name` with a query configuration `config`.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def table_validate_expression(self, view_name, expression):
|
||||
"""
|
||||
[OPTIONAL] Given a temporary table `view_name`, validate the type of
|
||||
a column expression string `expression`, or raise an error if the
|
||||
expression is invalid. This is enabeld by `"expressions"` via
|
||||
`get_features` and defaults to allow all expressions.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def view_delete(self, view_name):
|
||||
"""
|
||||
Delete a temporary table. The UI will do this automatically, and it
|
||||
can recover.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def view_get_min_max(self, view_name, column_name, config):
|
||||
"""
|
||||
[OPTIONAL] Get the min and max values of a column in a view.
|
||||
Returns a tuple of (min, max) as native Python values.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
def view_get_data(self, view_name, config, viewport, data):
|
||||
"""
|
||||
Serialize a rectangular slice `viewport` from temporary table
|
||||
`view_name`, into the `PerspectiveColumn` serialization API injected
|
||||
via `data`. The push-only `PerspectiveColumn` type can handle casting
|
||||
Python types as input, but once a type is pushed to a column name it
|
||||
must not be changed.
|
||||
"""
|
||||
|
||||
pass
|
||||
@@ -0,0 +1,240 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import perspective
|
||||
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
from perspective.virtual_servers import VirtualServerHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NUMBER_AGGS = [
|
||||
"sum",
|
||||
"count",
|
||||
"any_value",
|
||||
"arbitrary",
|
||||
"array_agg",
|
||||
"avg",
|
||||
"bit_and",
|
||||
"bit_or",
|
||||
"bit_xor",
|
||||
"bitstring_agg",
|
||||
"bool_and",
|
||||
"bool_or",
|
||||
"countif",
|
||||
"favg",
|
||||
"fsum",
|
||||
"geomean",
|
||||
"kahan_sum",
|
||||
"last",
|
||||
"max",
|
||||
"min",
|
||||
"product",
|
||||
"string_agg",
|
||||
"sumkahan",
|
||||
]
|
||||
|
||||
STRING_AGGS = [
|
||||
"count",
|
||||
"any_value",
|
||||
"arbitrary",
|
||||
"first",
|
||||
"countif",
|
||||
"last",
|
||||
"string_agg",
|
||||
]
|
||||
|
||||
FILTER_OPS = [
|
||||
"==",
|
||||
"!=",
|
||||
"LIKE",
|
||||
"IS DISTINCT FROM",
|
||||
"IS NOT DISTINCT FROM",
|
||||
">=",
|
||||
"<=",
|
||||
">",
|
||||
"<",
|
||||
]
|
||||
|
||||
|
||||
class ClickhouseVirtualSession:
|
||||
def __init__(self, callback, db):
|
||||
self.session = perspective.VirtualServer(ClickhouseVirtualServerHandler(db))
|
||||
self.callback = callback
|
||||
|
||||
def handle_request(self, msg):
|
||||
self.callback(self.session.handle_request(msg))
|
||||
|
||||
|
||||
class ClickhouseVirtualServer:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
def new_session(self, callback):
|
||||
return ClickhouseVirtualSession(callback, self.db)
|
||||
|
||||
|
||||
class ClickhouseVirtualServerHandler(VirtualServerHandler):
|
||||
"""
|
||||
An implementation of a `perspective.VirtualServerHandler` for ClickHouse.
|
||||
"""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
self.sql_builder = perspective.GenericSQLVirtualServerModel(
|
||||
{"create_entity": "VIEW", "grouping_fn": "GROUPING"}
|
||||
)
|
||||
|
||||
def get_features(self):
|
||||
return {
|
||||
"group_by": True,
|
||||
"split_by": False,
|
||||
"sort": True,
|
||||
"expressions": True,
|
||||
"group_rollup_mode": ["rollup", "flat", "total"],
|
||||
"filter_ops": {
|
||||
"integer": FILTER_OPS,
|
||||
"float": FILTER_OPS,
|
||||
"string": FILTER_OPS,
|
||||
"boolean": FILTER_OPS,
|
||||
"date": FILTER_OPS,
|
||||
"datetime": FILTER_OPS,
|
||||
},
|
||||
"aggregates": {
|
||||
"integer": NUMBER_AGGS,
|
||||
"float": NUMBER_AGGS,
|
||||
"string": STRING_AGGS,
|
||||
"boolean": STRING_AGGS,
|
||||
"date": STRING_AGGS,
|
||||
"datetime": STRING_AGGS,
|
||||
},
|
||||
}
|
||||
|
||||
def get_hosted_tables(self):
|
||||
query = "SHOW TABLES"
|
||||
results = run_query(self.db, query)
|
||||
return [result[0] for result in results]
|
||||
|
||||
def table_schema(self, table_name, config=None):
|
||||
query = self.sql_builder.table_schema(table_name)
|
||||
results = run_query(self.db, query)
|
||||
schema = {}
|
||||
for result in results:
|
||||
col_name = result[0]
|
||||
if not col_name.startswith("__"):
|
||||
schema[col_name] = clickhouse_type_to_psp(result[1])
|
||||
|
||||
return schema
|
||||
|
||||
def view_column_size(self, view_name, config):
|
||||
query = f"SELECT COUNT() FROM system.columns WHERE table = '{view_name}'"
|
||||
results = run_query(self.db, query)
|
||||
gs = len(config["group_by"])
|
||||
return results[0][0] - (
|
||||
0 if gs == 0 else gs + (1 if len(config["split_by"]) == 0 else 0)
|
||||
)
|
||||
|
||||
def table_size(self, table_name):
|
||||
query = self.sql_builder.table_size(table_name)
|
||||
results = run_query(self.db, query)
|
||||
return results[0][0]
|
||||
|
||||
def table_make_view(self, table_name, view_name, config):
|
||||
query = self.sql_builder.table_make_view(table_name, view_name, config)
|
||||
run_query(self.db, query, execute=True)
|
||||
|
||||
def table_validate_expression(self, view_name, expression):
|
||||
query = self.sql_builder.table_validate_expression(view_name, expression)
|
||||
results = run_query(self.db, query)
|
||||
return clickhouse_type_to_psp(results[0][1])
|
||||
|
||||
def view_delete(self, view_name):
|
||||
query = self.sql_builder.view_delete(view_name)
|
||||
run_query(self.db, query, execute=True)
|
||||
|
||||
def view_get_min_max(self, view_name, column_name, config):
|
||||
query = self.sql_builder.view_get_min_max(view_name, column_name, config)
|
||||
results = run_query(self.db, query)
|
||||
row = results[0]
|
||||
return (row[0], row[1])
|
||||
|
||||
def view_get_data(self, view_name, config, schema, viewport, data):
|
||||
group_by = config["group_by"]
|
||||
query = self.sql_builder.view_get_data(view_name, config, viewport, schema)
|
||||
results, columns, dtypes = run_query(self.db, query, columns=True)
|
||||
for cidx, col in enumerate(columns):
|
||||
dtype = clickhouse_type_to_psp(str(dtypes[cidx]))
|
||||
for ridx, row in enumerate(results):
|
||||
grouping_id = row[0] if len(group_by) > 0 else None
|
||||
|
||||
value = row[cidx]
|
||||
if dtype == "string" and not isinstance(value, str):
|
||||
value = str(value)
|
||||
|
||||
data.set_col(dtype, col, ridx, value, grouping_id)
|
||||
|
||||
|
||||
################################################################################
|
||||
#
|
||||
# ClickHouse Utils
|
||||
|
||||
|
||||
def clickhouse_type_to_psp(name):
|
||||
"""Convert a ClickHouse `dtype` to a Perspective `ColumnType`."""
|
||||
if name.startswith("Nullable(") and name.endswith(")"):
|
||||
name = name[9:-1]
|
||||
|
||||
if name.startswith("Array"):
|
||||
return "string"
|
||||
|
||||
if name in ("Int64", "UInt64", "Float64"):
|
||||
return "float"
|
||||
|
||||
if name == "String":
|
||||
return "string"
|
||||
|
||||
if name == "DateTime":
|
||||
return "datetime"
|
||||
|
||||
if name == "Date":
|
||||
return "date"
|
||||
|
||||
msg = f"Unknown type '{name}'"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def run_query(db, query, execute=False, columns=False):
|
||||
query = " ".join(query.split())
|
||||
start = datetime.now()
|
||||
result = None
|
||||
try:
|
||||
if execute:
|
||||
db.command(query)
|
||||
else:
|
||||
req = db.query(query)
|
||||
result = req.result_rows
|
||||
except Exception as e:
|
||||
logger.error(e)
|
||||
logger.error(f"{query}")
|
||||
raise e
|
||||
else:
|
||||
logger.debug(f"{datetime.now() - start} {query}")
|
||||
if columns:
|
||||
return (
|
||||
result,
|
||||
req.column_names,
|
||||
[(x.name if hasattr(x, "name") else str(x)) for x in req.column_types],
|
||||
)
|
||||
else:
|
||||
return result
|
||||
@@ -0,0 +1,236 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import io
|
||||
import duckdb
|
||||
import perspective
|
||||
import pyarrow.ipc as ipc
|
||||
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
from perspective.virtual_servers import VirtualServerHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NUMBER_AGGS = [
|
||||
"sum",
|
||||
"count",
|
||||
"any_value",
|
||||
"arbitrary",
|
||||
# "arg_max",
|
||||
# "arg_max_null",
|
||||
# "arg_min",
|
||||
# "arg_min_null",
|
||||
"array_agg",
|
||||
"avg",
|
||||
"bit_and",
|
||||
"bit_or",
|
||||
"bit_xor",
|
||||
"bitstring_agg",
|
||||
"bool_and",
|
||||
"bool_or",
|
||||
"countif",
|
||||
"favg",
|
||||
"fsum",
|
||||
"geomean",
|
||||
# "histogram",
|
||||
# "histogram_values",
|
||||
"kahan_sum",
|
||||
"last",
|
||||
# "list"
|
||||
"max",
|
||||
# "max_by"
|
||||
"min",
|
||||
# "min_by"
|
||||
"product",
|
||||
"string_agg",
|
||||
"sumkahan",
|
||||
# "weighted_avg",
|
||||
]
|
||||
|
||||
STRING_AGGS = [
|
||||
"count",
|
||||
"any_value",
|
||||
"arbitrary",
|
||||
"first",
|
||||
"countif",
|
||||
"last",
|
||||
"string_agg",
|
||||
]
|
||||
|
||||
FILTER_OPS = [
|
||||
"==",
|
||||
"!=",
|
||||
"LIKE",
|
||||
"IS DISTINCT FROM",
|
||||
"IS NOT DISTINCT FROM",
|
||||
">=",
|
||||
"<=",
|
||||
">",
|
||||
"<",
|
||||
]
|
||||
|
||||
|
||||
class DuckDBVirtualSession:
|
||||
def __init__(self, callback, db):
|
||||
self.session = perspective.VirtualServer(DuckDBVirtualServerHandler(db))
|
||||
self.callback = callback
|
||||
|
||||
def handle_request(self, msg):
|
||||
self.callback(self.session.handle_request(msg))
|
||||
|
||||
|
||||
class DuckDBVirtualServer:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
def new_session(self, callback):
|
||||
return DuckDBVirtualSession(callback, self.db)
|
||||
|
||||
|
||||
class DuckDBVirtualServerHandler(VirtualServerHandler):
|
||||
"""
|
||||
An implementation of a `perspective.VirtualServerHandler` for DuckDB.
|
||||
"""
|
||||
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
self.sql_builder = perspective.GenericSQLVirtualServerModel({})
|
||||
|
||||
def get_features(self):
|
||||
return {
|
||||
"group_by": True,
|
||||
"split_by": True,
|
||||
"sort": True,
|
||||
"expressions": True,
|
||||
"group_rollup_mode": ["rollup", "flat", "total"],
|
||||
"filter_ops": {
|
||||
"integer": FILTER_OPS,
|
||||
"float": FILTER_OPS,
|
||||
"string": FILTER_OPS,
|
||||
"boolean": FILTER_OPS,
|
||||
"date": FILTER_OPS,
|
||||
"datetime": FILTER_OPS,
|
||||
},
|
||||
"aggregates": {
|
||||
"integer": NUMBER_AGGS,
|
||||
"float": NUMBER_AGGS,
|
||||
"string": STRING_AGGS,
|
||||
"boolean": STRING_AGGS,
|
||||
"date": STRING_AGGS,
|
||||
"datetime": STRING_AGGS,
|
||||
},
|
||||
}
|
||||
|
||||
def get_hosted_tables(self):
|
||||
query = self.sql_builder.get_hosted_tables()
|
||||
results = run_query(self.db, query)
|
||||
return [f"{result[0]}.{result[2]}" for result in results]
|
||||
|
||||
def table_schema(self, table_name, config=None):
|
||||
query = self.sql_builder.table_schema(table_name)
|
||||
results = run_query(self.db, query)
|
||||
schema = {}
|
||||
for result in results:
|
||||
col_name = result[0]
|
||||
if not col_name.startswith("__"):
|
||||
schema[col_name] = duckdb_type_to_psp(result[1])
|
||||
|
||||
return schema
|
||||
|
||||
def view_column_size(self, table_name, config):
|
||||
query = self.sql_builder.view_column_size(table_name)
|
||||
results = run_query(self.db, query)
|
||||
gs = len(config["group_by"])
|
||||
return results[0][0] - (
|
||||
0 if gs == 0 else gs + (1 if len(config["split_by"]) == 0 else 0)
|
||||
)
|
||||
|
||||
def table_size(self, table_name):
|
||||
query = self.sql_builder.table_size(table_name)
|
||||
results = run_query(self.db, query)
|
||||
return results[0][0]
|
||||
|
||||
def table_make_view(self, table_name, view_name, config):
|
||||
query = self.sql_builder.table_make_view(table_name, view_name, config)
|
||||
run_query(self.db, query, execute=True)
|
||||
|
||||
def table_validate_expression(self, view_name, expression):
|
||||
query = self.sql_builder.table_validate_expression(view_name, expression)
|
||||
results = run_query(self.db, query)
|
||||
return duckdb_type_to_psp(results[0][1])
|
||||
|
||||
def view_delete(self, view_name):
|
||||
query = self.sql_builder.view_delete(view_name)
|
||||
run_query(self.db, query, execute=True)
|
||||
|
||||
def view_get_min_max(self, view_name, column_name, config):
|
||||
query = self.sql_builder.view_get_min_max(view_name, column_name, config)
|
||||
results = run_query(self.db, query)
|
||||
row = results[0]
|
||||
return (row[0], row[1])
|
||||
|
||||
def view_get_data(self, view_name, config, schema, viewport, data):
|
||||
query = self.sql_builder.view_get_data(view_name, config, viewport, schema)
|
||||
result = self.db.sql(query)
|
||||
arrow_table = result.fetch_arrow_table()
|
||||
buf = io.BytesIO()
|
||||
with ipc.new_stream(buf, arrow_table.schema) as writer:
|
||||
writer.write_table(arrow_table)
|
||||
data.from_arrow_ipc(buf.getvalue())
|
||||
|
||||
|
||||
################################################################################
|
||||
#
|
||||
# DuckDB Utils
|
||||
|
||||
|
||||
def duckdb_type_to_psp(name):
|
||||
"""Convert a DuckDB `dtype` to a Perspective `ColumnType`."""
|
||||
if name == "VARCHAR":
|
||||
return "string"
|
||||
if name in ("DOUBLE", "BIGINT", "HUGEINT"):
|
||||
return "float"
|
||||
if name == "INTEGER":
|
||||
return "integer"
|
||||
if name == "DATE":
|
||||
return "date"
|
||||
if name == "BOOLEAN":
|
||||
return "boolean"
|
||||
if name == "TIMESTAMP":
|
||||
return "datetime"
|
||||
|
||||
msg = f"Unknown type '{name}'"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def run_query(db, query, execute=False, columns=False):
|
||||
query = " ".join(query.split())
|
||||
start = datetime.now()
|
||||
result = None
|
||||
try:
|
||||
if execute:
|
||||
db.execute(query)
|
||||
else:
|
||||
req = db.sql(query)
|
||||
result = req.fetchall()
|
||||
except (duckdb.ParserException, duckdb.BinderException) as e:
|
||||
logger.error(e)
|
||||
logger.error(f"{query}")
|
||||
raise e
|
||||
else:
|
||||
logger.debug(f"{datetime.now() - start} {query}")
|
||||
if columns:
|
||||
return (result, req.columns, req.dtypes)
|
||||
else:
|
||||
return result
|
||||
@@ -0,0 +1,710 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import logging
|
||||
import polars as pl
|
||||
import perspective
|
||||
|
||||
from datetime import datetime
|
||||
import re
|
||||
|
||||
from perspective.virtual_servers import VirtualServerHandler
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NUMBER_AGGS = [
|
||||
"sum",
|
||||
"count",
|
||||
"any_value",
|
||||
"avg",
|
||||
"mean",
|
||||
"max",
|
||||
"min",
|
||||
"first",
|
||||
"last",
|
||||
]
|
||||
|
||||
STRING_AGGS = [
|
||||
"count",
|
||||
"any_value",
|
||||
"first",
|
||||
"last",
|
||||
]
|
||||
|
||||
FILTER_OPS = [
|
||||
"==",
|
||||
"!=",
|
||||
">=",
|
||||
"<=",
|
||||
">",
|
||||
"<",
|
||||
]
|
||||
|
||||
AGG_MAP = {
|
||||
"sum": lambda e: e.sum(),
|
||||
"count": lambda e: e.count(),
|
||||
"avg": lambda e: e.mean(),
|
||||
"mean": lambda e: e.mean(),
|
||||
"min": lambda e: e.min(),
|
||||
"max": lambda e: e.max(),
|
||||
"first": lambda e: e.first(),
|
||||
"last": lambda e: e.last(),
|
||||
"any_value": lambda e: e.first(),
|
||||
"arbitrary": lambda e: e.first(),
|
||||
}
|
||||
|
||||
|
||||
class PolarsVirtualSession:
|
||||
def __init__(self, callback, tables):
|
||||
self.session = perspective.VirtualServer(PolarsVirtualServerHandler(tables))
|
||||
self.callback = callback
|
||||
|
||||
def handle_request(self, msg):
|
||||
self.callback(self.session.handle_request(msg))
|
||||
|
||||
|
||||
class PolarsVirtualServer:
|
||||
def __init__(self, tables):
|
||||
self.tables = tables
|
||||
|
||||
def new_session(self, callback):
|
||||
return PolarsVirtualSession(callback, self.tables)
|
||||
|
||||
|
||||
class PolarsVirtualServerHandler(VirtualServerHandler):
|
||||
"""
|
||||
An implementation of a `perspective.VirtualServerHandler` for Polars.
|
||||
"""
|
||||
|
||||
def __init__(self, tables):
|
||||
self.tables = tables
|
||||
self.views = {}
|
||||
self.view_schemas = {}
|
||||
|
||||
def get_features(self):
|
||||
return {
|
||||
"group_by": True,
|
||||
"split_by": True,
|
||||
"sort": True,
|
||||
"expressions": True,
|
||||
"group_rollup_mode": ["rollup", "flat", "total"],
|
||||
"filter_ops": {
|
||||
"integer": FILTER_OPS,
|
||||
"float": FILTER_OPS,
|
||||
"string": FILTER_OPS,
|
||||
"boolean": ["==", "!="],
|
||||
"date": FILTER_OPS,
|
||||
"datetime": FILTER_OPS,
|
||||
},
|
||||
"aggregates": {
|
||||
"integer": NUMBER_AGGS,
|
||||
"float": NUMBER_AGGS,
|
||||
"string": STRING_AGGS,
|
||||
"boolean": STRING_AGGS,
|
||||
"date": STRING_AGGS,
|
||||
"datetime": STRING_AGGS,
|
||||
},
|
||||
}
|
||||
|
||||
def get_hosted_tables(self):
|
||||
return list(self.tables.keys())
|
||||
|
||||
def table_schema(self, table_name, config=None):
|
||||
df = self.tables[table_name]
|
||||
schema = {}
|
||||
for col_name, dtype in df.schema.items():
|
||||
if not col_name.startswith("__"):
|
||||
schema[col_name] = polars_type_to_psp(dtype)
|
||||
return schema
|
||||
|
||||
def table_size(self, table_name):
|
||||
return self.tables[table_name].height
|
||||
|
||||
def view_schema(self, view_name, config):
|
||||
if view_name in self.view_schemas:
|
||||
return self.view_schemas[view_name]
|
||||
return self.table_schema(view_name)
|
||||
|
||||
def view_size(self, view_name):
|
||||
if view_name in self.views:
|
||||
return self.views[view_name].height
|
||||
return self.table_size(view_name)
|
||||
|
||||
def table_validate_expression(self, table_name, expression):
|
||||
df = self.tables.get(table_name)
|
||||
if df is None:
|
||||
return None
|
||||
expr = parse_expression(expression)
|
||||
result = df.select(expr.alias("__expr__"))
|
||||
return polars_type_to_psp(result["__expr__"].dtype)
|
||||
|
||||
def table_make_view(self, table_name, view_name, config):
|
||||
start = datetime.now()
|
||||
df = self.tables[table_name]
|
||||
group_by = config.get("group_by", [])
|
||||
columns = [c for c in config.get("columns", []) if c is not None]
|
||||
aggregates = config.get("aggregates", {})
|
||||
sort = config.get("sort", [])
|
||||
filters = config.get("filter", [])
|
||||
split_by = config.get("split_by", [])
|
||||
expressions = config.get("expressions", {})
|
||||
group_rollup_mode = config.get("group_rollup_mode", "rollup")
|
||||
|
||||
if expressions:
|
||||
for expr_name, expr_str in expressions.items():
|
||||
expr = parse_expression(expr_str)
|
||||
df = df.with_columns(expr.alias(expr_name))
|
||||
|
||||
df = apply_filters(df, filters)
|
||||
|
||||
col_alias = lambda c: c.replace("_", "-")
|
||||
is_flat = group_rollup_mode == "flat"
|
||||
is_total = group_rollup_mode == "total"
|
||||
|
||||
if is_total:
|
||||
if split_by:
|
||||
result = build_split_by_total(
|
||||
df, split_by, columns, aggregates, col_alias
|
||||
)
|
||||
else:
|
||||
result = build_total(df, columns, aggregates, col_alias)
|
||||
elif split_by and group_by:
|
||||
if is_flat:
|
||||
result = build_split_by_grouped_flat(
|
||||
df, group_by, split_by, columns, aggregates, col_alias
|
||||
)
|
||||
result = apply_sort_split_by_flat(
|
||||
result, sort, columns, group_by, split_by
|
||||
)
|
||||
else:
|
||||
result = build_split_by_grouped(
|
||||
df, group_by, split_by, columns, aggregates, col_alias
|
||||
)
|
||||
result = apply_sort_grouped(result, sort, group_by, col_alias)
|
||||
elif split_by:
|
||||
result = build_split_by_flat(df, split_by, columns, col_alias)
|
||||
result = apply_sort_flat(result, sort, col_alias)
|
||||
elif group_by:
|
||||
if is_flat:
|
||||
result = build_flat_group_by(
|
||||
df, group_by, columns, aggregates, col_alias
|
||||
)
|
||||
result = apply_sort_flat(result, sort, col_alias)
|
||||
else:
|
||||
result = build_rollup(df, group_by, columns, aggregates, col_alias)
|
||||
result = apply_sort_grouped(result, sort, group_by, col_alias)
|
||||
else:
|
||||
select_exprs = [pl.col(c).alias(col_alias(c)) for c in columns]
|
||||
result = df.select(select_exprs)
|
||||
result = apply_sort_flat(result, sort, col_alias)
|
||||
|
||||
self.views[view_name] = result
|
||||
self.view_schemas[view_name] = compute_view_schema(result)
|
||||
logger.debug(
|
||||
f"{datetime.now() - start} table_make_view {table_name} -> {view_name}"
|
||||
)
|
||||
|
||||
def view_delete(self, view_name):
|
||||
self.views.pop(view_name, None)
|
||||
self.view_schemas.pop(view_name, None)
|
||||
|
||||
def view_get_min_max(self, view_name, column_name, config):
|
||||
df = self.views[view_name]
|
||||
col = df[column_name]
|
||||
min_val = col.min()
|
||||
max_val = col.max()
|
||||
return (min_val, max_val)
|
||||
|
||||
def view_get_data(self, view_name, config, schema, viewport, data):
|
||||
df = self.views.get(view_name)
|
||||
if df is None:
|
||||
return
|
||||
|
||||
group_by = config.get("group_by", [])
|
||||
split_by = config.get("split_by", [])
|
||||
group_rollup_mode = config.get("group_rollup_mode", "rollup")
|
||||
is_split_by = len(split_by) > 0
|
||||
is_flat = group_rollup_mode == "flat"
|
||||
|
||||
start_row = viewport.get("start_row", 0) or 0
|
||||
end_row = viewport.get("end_row") or df.height
|
||||
start_col = viewport.get("start_col", 0) or 0
|
||||
end_col = viewport.get("end_col")
|
||||
|
||||
length = min(end_row, df.height) - start_row
|
||||
if length <= 0:
|
||||
return
|
||||
df_slice = df.slice(start_row, length)
|
||||
|
||||
data_columns = [c for c in schema.keys() if not c.startswith("__")]
|
||||
if end_col is not None:
|
||||
data_columns = data_columns[start_col:end_col]
|
||||
else:
|
||||
data_columns = data_columns[start_col:]
|
||||
|
||||
has_group_by = len(group_by) > 0
|
||||
has_grouping_id = has_group_by and not is_flat
|
||||
|
||||
all_cols = []
|
||||
if has_grouping_id:
|
||||
all_cols.append("__GROUPING_ID__")
|
||||
for idx in range(len(group_by)):
|
||||
all_cols.append(f"__ROW_PATH_{idx}__")
|
||||
all_cols.extend(data_columns)
|
||||
|
||||
grouping_ids = None
|
||||
if has_grouping_id:
|
||||
grouping_ids = df_slice["__GROUPING_ID__"].to_list()
|
||||
|
||||
for cidx, col in enumerate(all_cols):
|
||||
if cidx == 0 and has_grouping_id:
|
||||
continue
|
||||
|
||||
series = df_slice[col]
|
||||
dtype = polars_type_to_psp(series.dtype)
|
||||
values = series.to_list()
|
||||
|
||||
push_col = col
|
||||
if is_split_by and not col.startswith("__"):
|
||||
push_col = col.replace("_", "|")
|
||||
|
||||
for ridx, value in enumerate(values):
|
||||
if grouping_ids:
|
||||
grouping_id = grouping_ids[ridx]
|
||||
elif has_group_by:
|
||||
grouping_id = 0
|
||||
else:
|
||||
grouping_id = None
|
||||
|
||||
if value is not None and isinstance(value, float) and value != value:
|
||||
value = None
|
||||
|
||||
data.set_col(dtype, push_col, ridx, value, grouping_id)
|
||||
|
||||
|
||||
################################################################################
|
||||
#
|
||||
# Polars Utils
|
||||
|
||||
|
||||
def polars_type_to_psp(dtype):
|
||||
"""Convert a Polars `dtype` to a Perspective `ColumnType`."""
|
||||
if dtype in (pl.Utf8, pl.String):
|
||||
return "string"
|
||||
if dtype == pl.Categorical:
|
||||
return "string"
|
||||
if dtype in (pl.Int8, pl.Int16, pl.Int32, pl.UInt8, pl.UInt16):
|
||||
return "integer"
|
||||
if dtype in (pl.Int64, pl.UInt64, pl.UInt32, pl.Float32, pl.Float64):
|
||||
return "float"
|
||||
if dtype == pl.Date:
|
||||
return "date"
|
||||
if dtype == pl.Boolean:
|
||||
return "boolean"
|
||||
if isinstance(dtype, pl.Datetime) or dtype == pl.Datetime:
|
||||
return "datetime"
|
||||
|
||||
msg = f"Unknown Polars type '{dtype}'"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def apply_filters(df, filters):
|
||||
"""Apply a list of filter configs to a DataFrame."""
|
||||
if not filters:
|
||||
return df
|
||||
|
||||
mask = pl.lit(True)
|
||||
for filt in filters:
|
||||
col_name = filt[0]
|
||||
op = filt[1]
|
||||
value = filt[2] if len(filt) > 2 else None
|
||||
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
col_expr = pl.col(col_name)
|
||||
if op == "==":
|
||||
mask = mask & (col_expr == value)
|
||||
elif op == "!=":
|
||||
mask = mask & (col_expr != value)
|
||||
elif op == ">":
|
||||
mask = mask & (col_expr > value)
|
||||
elif op == "<":
|
||||
mask = mask & (col_expr < value)
|
||||
elif op == ">=":
|
||||
mask = mask & (col_expr >= value)
|
||||
elif op == "<=":
|
||||
mask = mask & (col_expr <= value)
|
||||
|
||||
return df.filter(mask)
|
||||
|
||||
|
||||
def get_polars_agg_expr(col, agg_name, filter_expr=None):
|
||||
"""Convert an aggregate name to a Polars expression."""
|
||||
if isinstance(agg_name, list):
|
||||
agg_name = agg_name[0]
|
||||
if isinstance(agg_name, dict):
|
||||
agg_name = "first"
|
||||
expr = pl.col(col)
|
||||
if filter_expr is not None:
|
||||
expr = expr.filter(filter_expr)
|
||||
if agg_name in AGG_MAP:
|
||||
return AGG_MAP[agg_name](expr)
|
||||
|
||||
msg = f"Unknown aggregate '{agg_name}'"
|
||||
raise ValueError(msg)
|
||||
|
||||
|
||||
def default_aggregate(col_name, df):
|
||||
"""Return the default aggregate for a column based on its type."""
|
||||
dtype = df[col_name].dtype
|
||||
psp_type = polars_type_to_psp(dtype)
|
||||
if psp_type in ("integer", "float"):
|
||||
return "sum"
|
||||
return "count"
|
||||
|
||||
|
||||
def build_rollup(df, group_by, columns, aggregates, col_alias):
|
||||
"""Emulate GROUP BY ROLLUP using multiple group_by operations."""
|
||||
n = len(group_by)
|
||||
frames = []
|
||||
data_columns = [c for c in columns if c not in group_by]
|
||||
|
||||
for level in range(n + 1):
|
||||
num_groups = n - level
|
||||
active_groups = group_by[:num_groups]
|
||||
|
||||
agg_exprs = []
|
||||
for col in data_columns:
|
||||
agg_name = aggregates.get(col, default_aggregate(col, df))
|
||||
agg_exprs.append(get_polars_agg_expr(col, agg_name).alias(col_alias(col)))
|
||||
|
||||
if active_groups:
|
||||
grouped = df.group_by(active_groups, maintain_order=True).agg(agg_exprs)
|
||||
else:
|
||||
grouped = df.select(agg_exprs)
|
||||
|
||||
for idx in range(n):
|
||||
if idx < num_groups:
|
||||
grouped = grouped.with_columns(
|
||||
pl.col(group_by[idx]).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
else:
|
||||
src_dtype = df[group_by[idx]].dtype
|
||||
grouped = grouped.with_columns(
|
||||
pl.lit(None).cast(src_dtype).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
|
||||
grouping_id = sum(1 << i for i in range(num_groups, n))
|
||||
grouped = grouped.with_columns(
|
||||
pl.lit(grouping_id).cast(pl.Int64).alias("__GROUPING_ID__")
|
||||
)
|
||||
|
||||
for gb_col in active_groups:
|
||||
if gb_col in grouped.columns:
|
||||
grouped = grouped.drop(gb_col)
|
||||
|
||||
frames.append(grouped)
|
||||
|
||||
result = pl.concat(frames, how="diagonal")
|
||||
path_cols = [f"__ROW_PATH_{i}__" for i in range(n)]
|
||||
data_col_aliases = [col_alias(c) for c in data_columns]
|
||||
final_order = ["__GROUPING_ID__"] + path_cols + data_col_aliases
|
||||
result = result.select([c for c in final_order if c in result.columns])
|
||||
return result
|
||||
|
||||
|
||||
def build_flat_group_by(df, group_by, columns, aggregates, col_alias):
|
||||
"""Build a simple GROUP BY (no rollup) - only leaf-level rows."""
|
||||
n = len(group_by)
|
||||
data_columns = [c for c in columns if c not in group_by]
|
||||
|
||||
agg_exprs = []
|
||||
for col in data_columns:
|
||||
agg_name = aggregates.get(col, default_aggregate(col, df))
|
||||
agg_exprs.append(get_polars_agg_expr(col, agg_name).alias(col_alias(col)))
|
||||
|
||||
grouped = df.group_by(group_by, maintain_order=True).agg(agg_exprs)
|
||||
|
||||
for idx in range(n):
|
||||
grouped = grouped.with_columns(
|
||||
pl.col(group_by[idx]).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
|
||||
for gb_col in group_by:
|
||||
if gb_col in grouped.columns:
|
||||
grouped = grouped.drop(gb_col)
|
||||
|
||||
path_cols = [f"__ROW_PATH_{i}__" for i in range(n)]
|
||||
data_col_aliases = [col_alias(c) for c in data_columns]
|
||||
final_order = path_cols + data_col_aliases
|
||||
result = grouped.select([c for c in final_order if c in grouped.columns])
|
||||
return result.sort(path_cols)
|
||||
|
||||
|
||||
def build_total(df, columns, aggregates, col_alias):
|
||||
"""Build a single total row aggregating the entire dataset."""
|
||||
agg_exprs = []
|
||||
for col in columns:
|
||||
agg_name = aggregates.get(col, default_aggregate(col, df))
|
||||
agg_exprs.append(get_polars_agg_expr(col, agg_name).alias(col_alias(col)))
|
||||
return df.select(agg_exprs)
|
||||
|
||||
|
||||
def build_split_by_total(df, split_by, columns, aggregates, col_alias):
|
||||
"""Build a single total row with split_by (pivot) columns."""
|
||||
split_col = split_by[0]
|
||||
data_columns = [c for c in columns if c not in split_by]
|
||||
split_values = sorted(df[split_col].unique().to_list())
|
||||
|
||||
agg_exprs = []
|
||||
for sv in split_values:
|
||||
filter_expr = pl.col(split_col) == sv
|
||||
for dc in data_columns:
|
||||
agg_name = aggregates.get(dc, default_aggregate(dc, df))
|
||||
col_name = f"{sv}_{col_alias(dc)}"
|
||||
agg_exprs.append(
|
||||
get_polars_agg_expr(dc, agg_name, filter_expr=filter_expr).alias(
|
||||
col_name
|
||||
)
|
||||
)
|
||||
|
||||
return df.select(agg_exprs)
|
||||
|
||||
|
||||
def build_split_by_grouped_flat(df, group_by, split_by, columns, aggregates, col_alias):
|
||||
"""Build a flat grouped view with split_by (pivot) columns - no rollup rows."""
|
||||
n = len(group_by)
|
||||
split_col = split_by[0]
|
||||
data_columns = [c for c in columns if c not in group_by and c not in split_by]
|
||||
split_values = sorted(df[split_col].unique().to_list())
|
||||
|
||||
agg_exprs = []
|
||||
for sv in split_values:
|
||||
filter_expr = pl.col(split_col) == sv
|
||||
for dc in data_columns:
|
||||
agg_name = aggregates.get(dc, default_aggregate(dc, df))
|
||||
col_name = f"{sv}_{col_alias(dc)}"
|
||||
agg_exprs.append(
|
||||
get_polars_agg_expr(dc, agg_name, filter_expr=filter_expr).alias(
|
||||
col_name
|
||||
)
|
||||
)
|
||||
|
||||
for dc in data_columns:
|
||||
agg_name = aggregates.get(dc, default_aggregate(dc, df))
|
||||
agg_exprs.append(get_polars_agg_expr(dc, agg_name).alias(f"__SORT_{dc}__"))
|
||||
|
||||
grouped = df.group_by(group_by, maintain_order=True).agg(agg_exprs)
|
||||
|
||||
for idx in range(n):
|
||||
grouped = grouped.with_columns(
|
||||
pl.col(group_by[idx]).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
|
||||
for gb_col in group_by:
|
||||
if gb_col in grouped.columns:
|
||||
grouped = grouped.drop(gb_col)
|
||||
|
||||
path_cols = [f"__ROW_PATH_{i}__" for i in range(n)]
|
||||
data_col_names = []
|
||||
for sv in split_values:
|
||||
for dc in data_columns:
|
||||
data_col_names.append(f"{sv}_{col_alias(dc)}")
|
||||
sort_col_names = [f"__SORT_{dc}__" for dc in data_columns]
|
||||
final_order = path_cols + data_col_names + sort_col_names
|
||||
result = grouped.select([c for c in final_order if c in grouped.columns])
|
||||
return result.sort(path_cols)
|
||||
|
||||
|
||||
def apply_sort_grouped(df, sort_config, group_by, col_alias):
|
||||
"""Apply sort to a ROLLUP result DataFrame."""
|
||||
n = len(group_by)
|
||||
|
||||
sort_cols = []
|
||||
sort_desc = []
|
||||
for entry in sort_config:
|
||||
col = entry[0]
|
||||
direction = entry[1]
|
||||
if direction != "none":
|
||||
aliased = col_alias(col)
|
||||
if aliased in df.columns:
|
||||
sort_cols.append(aliased)
|
||||
sort_desc.append(direction in ("desc", "col desc"))
|
||||
|
||||
if not sort_cols:
|
||||
# Default: tree order by row path, nulls first
|
||||
path_cols = [f"__ROW_PATH_{i}__" for i in range(n)]
|
||||
return df.sort(path_cols, descending=[False] * n, nulls_last=False)
|
||||
|
||||
# With explicit sort: grand total first, then rest sorted
|
||||
is_total = pl.lit(True)
|
||||
for i in range(n):
|
||||
is_total = is_total & pl.col(f"__ROW_PATH_{i}__").is_null()
|
||||
|
||||
total_row = df.filter(is_total)
|
||||
rest = df.filter(~is_total)
|
||||
rest = rest.sort(sort_cols, descending=sort_desc)
|
||||
return pl.concat([total_row, rest])
|
||||
|
||||
|
||||
def apply_sort_split_by_flat(df, sort_config, columns, group_by, split_by):
|
||||
"""Apply sort to a flat split_by grouped DataFrame using __SORT__ columns."""
|
||||
data_columns = [c for c in columns if c not in group_by and c not in split_by]
|
||||
sort_cols = []
|
||||
sort_desc = []
|
||||
for entry in sort_config:
|
||||
col = entry[0]
|
||||
direction = entry[1]
|
||||
if direction != "none":
|
||||
sort_name = f"__SORT_{col}__"
|
||||
if sort_name in df.columns:
|
||||
sort_cols.append(sort_name)
|
||||
sort_desc.append(direction in ("desc", "col desc"))
|
||||
|
||||
if sort_cols:
|
||||
df = df.sort(sort_cols, descending=sort_desc)
|
||||
|
||||
drop_cols = [
|
||||
f"__SORT_{dc}__" for dc in data_columns if f"__SORT_{dc}__" in df.columns
|
||||
]
|
||||
if drop_cols:
|
||||
df = df.drop(drop_cols)
|
||||
return df
|
||||
|
||||
|
||||
def apply_sort_flat(df, sort_config, col_alias):
|
||||
"""Apply sort to a flat (non-grouped) DataFrame."""
|
||||
if not sort_config:
|
||||
return df
|
||||
|
||||
sort_cols = []
|
||||
sort_descending = []
|
||||
for sort_entry in sort_config:
|
||||
col = sort_entry[0]
|
||||
direction = sort_entry[1]
|
||||
if direction != "none":
|
||||
aliased = col_alias(col)
|
||||
if aliased in df.columns:
|
||||
sort_cols.append(aliased)
|
||||
sort_descending.append(direction in ("desc", "col desc"))
|
||||
|
||||
if sort_cols:
|
||||
return df.sort(sort_cols, descending=sort_descending)
|
||||
return df
|
||||
|
||||
|
||||
def compute_view_schema(df):
|
||||
"""Compute the Perspective schema for a view DataFrame."""
|
||||
schema = {}
|
||||
for col_name, dtype in df.schema.items():
|
||||
if col_name.startswith("__"):
|
||||
continue
|
||||
schema[col_name] = polars_type_to_psp(dtype)
|
||||
return schema
|
||||
|
||||
|
||||
def build_split_by_grouped(df, group_by, split_by, columns, aggregates, col_alias):
|
||||
"""Build a grouped rollup with split_by (pivot) columns."""
|
||||
n = len(group_by)
|
||||
split_col = split_by[0]
|
||||
data_columns = [c for c in columns if c not in group_by and c not in split_by]
|
||||
split_values = sorted(df[split_col].unique().to_list())
|
||||
|
||||
frames = []
|
||||
for level in range(n + 1):
|
||||
num_groups = n - level
|
||||
active_groups = group_by[:num_groups]
|
||||
|
||||
agg_exprs = []
|
||||
for sv in split_values:
|
||||
filter_expr = pl.col(split_col) == sv
|
||||
for dc in data_columns:
|
||||
agg_name = aggregates.get(dc, default_aggregate(dc, df))
|
||||
col_name = f"{sv}_{col_alias(dc)}"
|
||||
agg_exprs.append(
|
||||
get_polars_agg_expr(dc, agg_name, filter_expr=filter_expr).alias(
|
||||
col_name
|
||||
)
|
||||
)
|
||||
|
||||
if active_groups:
|
||||
grouped = df.group_by(active_groups, maintain_order=True).agg(agg_exprs)
|
||||
else:
|
||||
grouped = df.select(agg_exprs)
|
||||
|
||||
for idx in range(n):
|
||||
if idx < num_groups:
|
||||
grouped = grouped.with_columns(
|
||||
pl.col(group_by[idx]).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
else:
|
||||
src_dtype = df[group_by[idx]].dtype
|
||||
grouped = grouped.with_columns(
|
||||
pl.lit(None).cast(src_dtype).alias(f"__ROW_PATH_{idx}__")
|
||||
)
|
||||
|
||||
grouping_id = sum(1 << i for i in range(num_groups, n))
|
||||
grouped = grouped.with_columns(
|
||||
pl.lit(grouping_id).cast(pl.Int64).alias("__GROUPING_ID__")
|
||||
)
|
||||
|
||||
for gb_col in active_groups:
|
||||
if gb_col in grouped.columns:
|
||||
grouped = grouped.drop(gb_col)
|
||||
|
||||
frames.append(grouped)
|
||||
|
||||
result = pl.concat(frames, how="diagonal")
|
||||
path_cols = [f"__ROW_PATH_{i}__" for i in range(n)]
|
||||
data_col_names = []
|
||||
for sv in split_values:
|
||||
for dc in data_columns:
|
||||
data_col_names.append(f"{sv}_{col_alias(dc)}")
|
||||
final_order = ["__GROUPING_ID__"] + path_cols + data_col_names
|
||||
result = result.select([c for c in final_order if c in result.columns])
|
||||
return result
|
||||
|
||||
|
||||
def build_split_by_flat(df, split_by, columns, col_alias):
|
||||
"""Build a flat (non-grouped) split_by view."""
|
||||
split_col = split_by[0]
|
||||
data_columns = [c for c in columns if c not in split_by]
|
||||
split_values = sorted(df[split_col].unique().to_list())
|
||||
|
||||
exprs = []
|
||||
for sv in split_values:
|
||||
for dc in data_columns:
|
||||
col_name = f"{sv}_{col_alias(dc)}"
|
||||
exprs.append(
|
||||
pl.when(pl.col(split_col) == sv)
|
||||
.then(pl.col(dc))
|
||||
.otherwise(None)
|
||||
.alias(col_name)
|
||||
)
|
||||
|
||||
return df.select(exprs)
|
||||
|
||||
|
||||
def parse_expression(expr_str):
|
||||
"""Parse a Perspective expression string into a Polars expression."""
|
||||
pattern = r'"([^"]*)"'
|
||||
parts = []
|
||||
last_end = 0
|
||||
for match in re.finditer(pattern, expr_str):
|
||||
parts.append(expr_str[last_end : match.start()])
|
||||
col_name = match.group(1)
|
||||
parts.append(f'pl.col("{col_name}")')
|
||||
last_end = match.end()
|
||||
parts.append(expr_str[last_end:])
|
||||
polars_expr_str = "".join(parts)
|
||||
return eval(polars_expr_str, {"pl": pl, "__builtins__": {}})
|
||||
@@ -0,0 +1,349 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
|
||||
from string import Template
|
||||
from ipywidgets import DOMWidget
|
||||
from traitlets import Unicode, observe
|
||||
from .viewer import PerspectiveViewer
|
||||
|
||||
__version__ = re.sub(".dev[0-9]+", "", importlib.metadata.version("perspective-python"))
|
||||
|
||||
__all__ = ["PerspectiveWidget"]
|
||||
|
||||
__doc__ = """
|
||||
`PerspectiveWidget` is a JupyterLab widget that implements the same API as
|
||||
`<perspective-viewer>`, allowing for fast, intuitive
|
||||
transformations/visualizations of various data formats within JupyterLab.
|
||||
|
||||
`PerspectiveWidget` is compatible with Jupyterlab 3 and Jupyter Notebook 6 via a
|
||||
[prebuilt extension](https://jupyterlab.readthedocs.io/en/stable/extension/extension_dev.html#prebuilt-extensions).
|
||||
To use it, simply install `perspective-python` and the extensions should be
|
||||
available.
|
||||
|
||||
`perspective-python`'s JupyterLab extension also provides convenient builtin
|
||||
viewers for `csv`, `json`, or `arrow` files. Simply right-click on a file with
|
||||
this extension and choose the appropriate `Perpective` option from the context
|
||||
menu.
|
||||
|
||||
## `PerspectiveWidget`
|
||||
|
||||
Building on top of the API provided by `perspective.Table`, the
|
||||
`PerspectiveWidget` is a JupyterLab plugin that offers the entire functionality
|
||||
of Perspective within the Jupyter environment. It supports the same API
|
||||
semantics of `<perspective-viewer>`, along with the additional data types
|
||||
supported by `perspective.Table`. `PerspectiveWidget` takes keyword arguments
|
||||
for the managed `View`:
|
||||
|
||||
```python
|
||||
from perspective.widget import PerspectiveWidget
|
||||
w = perspective.PerspectiveWidget(
|
||||
data,
|
||||
plugin="X Bar",
|
||||
aggregates={"datetime": "any"},
|
||||
sort=[["date", "desc"]]
|
||||
)
|
||||
```
|
||||
|
||||
### Creating a widget
|
||||
|
||||
A widget is created through the `PerspectiveWidget` constructor, which takes as
|
||||
its first, required parameter a `perspective.Table`, a dataset, a schema, or
|
||||
`None`, which serves as a special value that tells the Widget to defer loading
|
||||
any data until later. In maintaining consistency with the Javascript API,
|
||||
Widgets cannot be created with empty dictionaries or lists—`None` should be used
|
||||
if the intention is to await data for loading later on. A widget can be
|
||||
constructed from a dataset:
|
||||
|
||||
```python
|
||||
from perspective.widget import PerspectiveWidget
|
||||
PerspectiveWidget(data, group_by=["date"])
|
||||
```
|
||||
|
||||
.. or a schema:
|
||||
|
||||
```python
|
||||
PerspectiveWidget({"a": int, "b": str})
|
||||
```
|
||||
|
||||
.. or an instance of a `perspective.Table`:
|
||||
|
||||
```python
|
||||
table = perspective.table(data)
|
||||
PerspectiveWidget(table)
|
||||
```
|
||||
"""
|
||||
|
||||
|
||||
class PerspectiveWidget(DOMWidget, PerspectiveViewer):
|
||||
"""`PerspectiveWidget` allows for Perspective to be used as a Jupyter
|
||||
widget.
|
||||
|
||||
Using `perspective.Table`, you can create a widget that extends the full
|
||||
functionality of `perspective-viewer`. Changes on the viewer can be
|
||||
programatically set on the `PerspectiveWidget` instance.
|
||||
|
||||
# Examples
|
||||
|
||||
>>> from perspective.widget import PerspectiveWidget
|
||||
>>> data = {
|
||||
... "a": [1, 2, 3],
|
||||
... "b": [
|
||||
... "2019/07/11 7:30PM",
|
||||
... "2019/07/11 8:30PM",
|
||||
... "2019/07/11 9:30PM"
|
||||
... ]
|
||||
... }
|
||||
>>> widget = PerspectiveWidget(
|
||||
... data,
|
||||
... group_by=["a"],
|
||||
... sort=[["b", "desc"]],
|
||||
... filter=[["a", ">", 1]]
|
||||
... )
|
||||
>>> widget.sort
|
||||
[["b", "desc"]]
|
||||
>>> widget.sort.append(["a", "asc"])
|
||||
>>> widget.sort
|
||||
[["b", "desc"], ["a", "asc"]]
|
||||
>>> widget.table.update({"a": [4, 5]}) # Browser UI updates
|
||||
"""
|
||||
|
||||
# Required by ipywidgets for proper registration of the backend
|
||||
_model_name = Unicode("PerspectiveModel").tag(sync=True)
|
||||
_model_module = Unicode("@perspective-dev/jupyterlab").tag(sync=True)
|
||||
_model_module_version = Unicode("~{}".format(__version__)).tag(sync=True)
|
||||
_view_name = Unicode("PerspectiveView").tag(sync=True)
|
||||
_view_module = Unicode("@perspective-dev/jupyterlab").tag(sync=True)
|
||||
_view_module_version = Unicode("~{}".format(__version__)).tag(sync=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
data,
|
||||
index=None,
|
||||
limit=None,
|
||||
binding_mode="server",
|
||||
**kwargs,
|
||||
):
|
||||
"""Initialize an instance of `PerspectiveWidget`
|
||||
with the given table/data and viewer configuration.
|
||||
|
||||
If an `AsyncTable` is passed in, then certain widget methods like
|
||||
`update()` and `delete()` return coroutines which must be awaited.
|
||||
|
||||
# Arguments
|
||||
|
||||
- `data` (`Table`|`AsyncTable`|`dict`|`list`|`pandas.DataFrame`|`bytes`|`str`): a
|
||||
`perspective.Table` instance, a `perspective.AsyncTable` instance, or
|
||||
a dataset to be loaded in the widget.
|
||||
|
||||
# Keyword Arguments
|
||||
|
||||
- `index` (`str`): A column name to be used as the primary key.
|
||||
Ignored if `server` is True.
|
||||
- `binding_mode` (`str`): "client-server" or "server"
|
||||
- `limit` (`int`): A upper limit on the number of rows in the Table.
|
||||
Cannot be set at the same time as `index`, ignored if `server`
|
||||
is True.
|
||||
- `kwargs` (`dict`): configuration options for the `PerspectiveViewer`,
|
||||
and `Table` constructor if `data` is a dataset.
|
||||
|
||||
# Examples
|
||||
|
||||
>>> widget = PerspectiveWidget(
|
||||
... {"a": [1, 2, 3]},
|
||||
... aggregates={"a": "avg"},
|
||||
... group_by=["a"],
|
||||
... sort=[["b", "desc"]],
|
||||
... filter=[["a", ">", 1]],
|
||||
... expressions=["\"a\" + 100"])
|
||||
"""
|
||||
|
||||
self.binding_mode = binding_mode
|
||||
|
||||
# Pass table load options to the front-end, unless in server mode
|
||||
self._options = {}
|
||||
|
||||
if index is not None and limit is not None:
|
||||
raise TypeError("Index and Limit cannot be set at the same time!")
|
||||
|
||||
# Parse the dataset we pass in - if it's Pandas, preserve pivots
|
||||
# if isinstance(data, pandas.DataFrame) or isinstance(data, pandas.Series):
|
||||
# data, config = deconstruct_pandas(data)
|
||||
|
||||
# if config.get("group_by", None) and "group_by" not in kwargs:
|
||||
# kwargs.update({"group_by": config["group_by"]})
|
||||
|
||||
# if config.get("split_by", None) and "split_by" not in kwargs:
|
||||
# kwargs.update({"split_by": config["split_by"]})
|
||||
|
||||
# if config.get("columns", None) and "columns" not in kwargs:
|
||||
# kwargs.update({"columns": config["columns"]})
|
||||
|
||||
# Initialize the viewer
|
||||
super(PerspectiveWidget, self).__init__(**kwargs)
|
||||
|
||||
# Handle messages from the the front end
|
||||
self.on_msg(self.handle_message)
|
||||
self._sessions = {}
|
||||
|
||||
# If an empty dataset is provided, don't call `load()` and wait
|
||||
# for the user to call `load()`.
|
||||
if data is None:
|
||||
if index is not None or limit is not None:
|
||||
raise TypeError(
|
||||
"Cannot initialize PerspectiveWidget `index` or `limit` without a Table, data, or schema!"
|
||||
)
|
||||
else:
|
||||
if index is not None:
|
||||
self._options.update({"index": index})
|
||||
|
||||
if limit is not None:
|
||||
self._options.update({"limit": limit})
|
||||
|
||||
loading = self.load(data, **self._options)
|
||||
if inspect.isawaitable(loading):
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(loading)
|
||||
|
||||
def load(self, data, **options):
|
||||
"""Load the widget with data."""
|
||||
# Viewer will ignore **options if `data` is a Table or View.
|
||||
return super(PerspectiveWidget, self).load(data, **options)
|
||||
|
||||
def update(self, data):
|
||||
"""Update the widget with new data."""
|
||||
return super(PerspectiveWidget, self).update(data)
|
||||
|
||||
def clear(self):
|
||||
"""Clears the widget's underlying `Table`."""
|
||||
return super(PerspectiveWidget, self).clear()
|
||||
|
||||
def replace(self, data):
|
||||
"""Replaces the widget's `Table` with new data conforming to the same
|
||||
schema. Does not clear user-set state. If in client mode, serializes
|
||||
the data and sends it to the browser.
|
||||
"""
|
||||
return super(PerspectiveWidget, self).replace(data)
|
||||
|
||||
def delete(self, delete_table=True):
|
||||
"""Delete the Widget's data and clears its internal state.
|
||||
|
||||
# Arguments
|
||||
|
||||
- `delete_table` (`bool`): whether the underlying `Table` will be
|
||||
deleted. Defaults to True.
|
||||
"""
|
||||
ret = super(PerspectiveWidget, self).delete(delete_table)
|
||||
|
||||
# Close the underlying comm and remove widget from the front-end
|
||||
self.close()
|
||||
return ret
|
||||
|
||||
@observe("value")
|
||||
def handle_message(self, widget, content, buffers):
|
||||
"""Given a message from `PerspectiveJupyterClient.send`, process the
|
||||
message and return the result to `self.post`.
|
||||
|
||||
# Arguments
|
||||
|
||||
- `widget`: a reference to the `Widget` instance that received the
|
||||
message.
|
||||
- `content` (dict): - the message from the front-end. Automatically
|
||||
de-serialized by ipywidgets.
|
||||
- `buffers`: optional arraybuffers from the front-end, if any.
|
||||
"""
|
||||
if content["type"] == "connect":
|
||||
client_id = content["client_id"]
|
||||
logging.debug("view {} connected", client_id)
|
||||
|
||||
def send_response(msg):
|
||||
self.send({"type": "binary_msg", "client_id": client_id}, [msg])
|
||||
|
||||
self._sessions[client_id] = self.new_proxy_session(send_response)
|
||||
elif content["type"] == "binary_msg":
|
||||
[binary_msg] = buffers
|
||||
client_id = content["client_id"]
|
||||
session = self._sessions[client_id]
|
||||
if session is not None:
|
||||
import asyncio
|
||||
|
||||
asyncio.create_task(session.handle_request_async(binary_msg))
|
||||
else:
|
||||
logging.error("No session for client_id {}".format(client_id))
|
||||
elif content["type"] == "hangup":
|
||||
# XXX(tom): client won't reliably send this so shouldn't rely on it
|
||||
# to clean up; does jupyter notify us when the client on the
|
||||
# websocket, i.e. the view, disconnects?
|
||||
client_id = content["client_id"]
|
||||
logging.debug("view {} hangup", client_id)
|
||||
session = self._sessions.pop(client_id, None)
|
||||
if session:
|
||||
session.close()
|
||||
|
||||
def _repr_mimebundle_(self, **kwargs):
|
||||
super_bundle = super(DOMWidget, self)._repr_mimebundle_(**kwargs)
|
||||
if not _jupyter_html_export_enabled():
|
||||
return super_bundle
|
||||
|
||||
# Serialize viewer attrs + view data to be rendered in the template
|
||||
viewer_attrs = self.save()
|
||||
data = self.table.view().to_arrow()
|
||||
b64_data = base64.encodebytes(data)
|
||||
template_path = os.path.join(
|
||||
os.path.dirname(__file__), "../templates/exported_widget.html.template"
|
||||
)
|
||||
with open(template_path, "r") as template_data:
|
||||
template = Template(template_data.read())
|
||||
|
||||
def psp_cdn(module, path=None):
|
||||
if path is None:
|
||||
path = f"cdn/{module}.js"
|
||||
|
||||
# perspective developer affordance: works with your local `pnpm run start blocks`
|
||||
# return f"http://localhost:8080/node_modules/@perspective-dev/{module}/dist/{path}"
|
||||
return f"https://cdn.jsdelivr.net/npm/@perspective-dev/{module}@{__version__}/dist/{path}"
|
||||
|
||||
return super(DOMWidget, self)._repr_mimebundle_(**kwargs) | {
|
||||
"text/html": template.substitute(
|
||||
psp_cdn_perspective=psp_cdn("perspective"),
|
||||
psp_cdn_perspective_viewer=psp_cdn("perspective-viewer"),
|
||||
psp_cdn_perspective_viewer_datagrid=psp_cdn(
|
||||
"perspective-viewer-datagrid"
|
||||
),
|
||||
psp_cdn_perspective_viewer_charts=psp_cdn("perspective-viewer-charts"),
|
||||
psp_cdn_perspective_viewer_themes=psp_cdn(
|
||||
"perspective-viewer-themes", "css/themes.css"
|
||||
),
|
||||
viewer_id=self.model_id,
|
||||
viewer_attrs=viewer_attrs,
|
||||
b64_data=b64_data.decode("utf-8"),
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
def _jupyter_html_export_enabled():
|
||||
return os.environ.get("PSP_JUPYTER_HTML_EXPORT", None) == "1"
|
||||
|
||||
|
||||
def set_jupyter_html_export(val):
|
||||
"""Enables HTML export for Jupyter widgets, when set to True.
|
||||
HTML export can also be enabled by setting the environment variable
|
||||
`PSP_JUPYTER_HTML_EXPORT` to the string `1`.
|
||||
"""
|
||||
os.environ["PSP_JUPYTER_HTML_EXPORT"] = "1" if val else "0"
|
||||
@@ -0,0 +1,15 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from .viewer import PerspectiveViewer
|
||||
|
||||
__all__ = ["PerspectiveViewer"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
|
||||
def validate_version(version):
|
||||
# basic semver of form \d+\.\d+\.\d+(\+.+)?
|
||||
spl = version.split(".", 2)
|
||||
return (
|
||||
len(spl) == 3
|
||||
and spl[0].isdigit()
|
||||
and spl[1].isdigit()
|
||||
and (spl[2].split("+")[0]).isdigit()
|
||||
)
|
||||
@@ -0,0 +1,343 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from random import random
|
||||
from .viewer_traitlets import PerspectiveTraitlets
|
||||
import perspective
|
||||
|
||||
# from .. import Server
|
||||
|
||||
# ─ │ ┌ ┬ ┐
|
||||
# ┄ ┆ ├ ┼ ┤ ╲ ╱
|
||||
# ┈ ┊ └ ┴ ┘
|
||||
# ━ ┃ ┏ ┳ ┓ ┏ ┯ ┓ ┏ ┳ ┓ ┏ ┯ ┓
|
||||
# ┅ ┇ ┣ ╋ ┫ ┣ ┿ ┫ ┠ ╂ ┨ ┠ ┼ ┨
|
||||
# ┉ ┋ ┗ ┻ ┛ ┗ ┷ ┛ ┗ ┻ ┛ ┗ ┷ ┛
|
||||
|
||||
# global_server = PySyncServer()
|
||||
|
||||
|
||||
class PerspectiveViewer(PerspectiveTraitlets, object):
|
||||
"""PerspectiveViewer wraps the `perspective.Table` API and exposes an API
|
||||
around creating views, loading data, and updating data.
|
||||
"""
|
||||
|
||||
# Viewer attributes that should be saved in `save()` and restored using
|
||||
# `restore()`. Symmetric to `PERSISTENT_ATTRIBUTES` in `perspective-viewer`.
|
||||
PERSISTENT_ATTRIBUTES = (
|
||||
"group_by",
|
||||
"split_by",
|
||||
"filter",
|
||||
"sort",
|
||||
"aggregates",
|
||||
"columns",
|
||||
"expressions",
|
||||
"plugin",
|
||||
"plugin_config",
|
||||
"theme",
|
||||
"settings",
|
||||
"title",
|
||||
"version",
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
plugin="Datagrid",
|
||||
columns=None,
|
||||
group_by=None,
|
||||
split_by=None,
|
||||
aggregates=None,
|
||||
sort=None,
|
||||
filter=None,
|
||||
expressions=None,
|
||||
plugin_config=None,
|
||||
settings=True,
|
||||
theme=None,
|
||||
title=None,
|
||||
# ignored, here for restore compatibility
|
||||
version=None,
|
||||
):
|
||||
"""Initialize an instance of `PerspectiveViewer` with the given viewer
|
||||
configuration. Do not pass a `Table` or data into the constructor -
|
||||
use the :func:`load()` method to provide the viewer with data.
|
||||
|
||||
Keyword Arguments:
|
||||
columns (:obj:`list` of :obj:`str`): A list of column names to be
|
||||
visible to the user.
|
||||
group_by (:obj:`list` of :obj:`str`): A list of column names to
|
||||
use as group by.
|
||||
split_by (:obj:`list` of :obj:`str`): A list of column names
|
||||
to use as split by.
|
||||
aggregates (:obj:`dict` of :obj:`str` to :obj:`str`): A dictionary
|
||||
of column names to aggregate types, which specify aggregates
|
||||
for individual columns.
|
||||
sort (:obj:`list` of :obj:`list` of :obj:`str`): A list of lists,
|
||||
each list containing a column name and a sort direction
|
||||
(``asc``, ``desc``, ``asc abs``, ``desc abs``, ``col asc``,
|
||||
``col desc``, ``col asc abs``, ``col desc abs``).
|
||||
filter (:obj:`list` of :obj:`list` of :obj:`str`): A list of lists,
|
||||
each list containing a column name, a filter comparator, and a
|
||||
value to filter by.
|
||||
expressions (:obj:`list` of :obj:`str`): A list of string
|
||||
expressions which are applied to the view.
|
||||
plugin (:obj:`str`/:obj:`perspective.Plugin`): Which plugin to
|
||||
select by default.
|
||||
plugin_config (:obj:`dict`): A configuration for the plugin, i.e.
|
||||
the datagrid plugin or a chart plugin.
|
||||
settings(:obj:`bool`): Whether the perspective query settings
|
||||
panel should be open.
|
||||
theme (:obj:`str`): The color theme to use.
|
||||
version (:obj:`str`): The version this configuration is restored from.
|
||||
This should only be used when restoring a configuration,
|
||||
and should not be set manually.
|
||||
|
||||
Examples:
|
||||
>>> viewer = PerspectiveViewer(
|
||||
... aggregates={"a": "avg"},
|
||||
... group_by=["a"],
|
||||
... sort=[["b", "desc"]],
|
||||
... filter=[["a", ">", 1]],
|
||||
... expressions=["\"a\" + 100"]
|
||||
... )
|
||||
"""
|
||||
|
||||
# The Table under management by this viewer and its
|
||||
# attached PerspectiveManager
|
||||
self._table = None
|
||||
self._client = None
|
||||
|
||||
# Viewer configuration
|
||||
self.plugin = plugin # validate_plugin(plugin)
|
||||
self.columns = columns or [] # validate_columns(columns) or []
|
||||
self.group_by = group_by or [] # validate_group_by(group_by) or []
|
||||
self.split_by = split_by or [] # validate_split_by(split_by) or []
|
||||
self.aggregates = aggregates or {} # validate_aggregates(aggregates) or {}
|
||||
self.sort = sort or [] # validate_sort(sort) or []
|
||||
self.filter = filter or [] # validate_filter(filter) or []
|
||||
self.expressions = expressions or {} # validate_expressions(expressions) or {}
|
||||
self.plugin_config = (
|
||||
plugin_config or {}
|
||||
) # validate_plugin_config(plugin_config) or {}
|
||||
self.settings = settings
|
||||
self.theme = theme
|
||||
self.title = title
|
||||
|
||||
def new_proxy_session(self, cb):
|
||||
return perspective.ProxySession(self._client, cb)
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
"""Returns the ``perspective.Client`` or ``perspective.AsyncClient`` under management by the viewer."""
|
||||
return self._client
|
||||
|
||||
@property
|
||||
def table(self):
|
||||
"""Returns the ``perspective.Table`` or ``perspective.AsyncTable`` under management by the viewer."""
|
||||
return self._table
|
||||
|
||||
def is_async(self):
|
||||
"""Returns whether this widget has an async interface or synchronous"""
|
||||
return isinstance(self._table, perspective.perspective.AsyncTable)
|
||||
|
||||
def load(self, data, **options):
|
||||
"""Given a ``perspective.Table``, a ``perspective.AsyncTable``,
|
||||
or data that can be handled by ``perspective.Table``, pass it to the
|
||||
viewer. Like `__init__`, load accepts a `perspective.Table`, a dataset,
|
||||
or a schema.
|
||||
|
||||
``load()`` resets the state of the viewer: if a ``perspective.Table``
|
||||
has already been loaded, ``**options`` is ignored as the options
|
||||
already set on the ``Table`` take precedence.
|
||||
|
||||
If data is passed in, a ``perspective.Table`` is automatically created
|
||||
by this method, and the options passed to ``**config`` are extended to
|
||||
the new Table. If the widget already has a dataset, and the new data
|
||||
has different columns to the old one, then the widget state (pivots,
|
||||
sort, etc.) is cleared to prevent applying settings on columns that
|
||||
don't exist.
|
||||
|
||||
When a ``perspective.AsyncTable`` is loaded, the widget's interface
|
||||
becomes async. Methods which operate on the underlying Perspective
|
||||
view, inclusive of the ``load()`` call itself, return coroutine values
|
||||
which must be awaited.
|
||||
|
||||
Loading a ``perspective.Table`` or plain data will make the interface
|
||||
synchronous again.
|
||||
|
||||
Args:
|
||||
data (:obj:`Table`|:obj:`AsyncTable`|:obj:`dict`|:obj:`list`|:obj:`pandas.DataFrame`|:obj:`bytes`|:obj:`str`): a
|
||||
`perspective.Table` instance, a `perspective.AsyncTable`
|
||||
instance, or a dataset to be loaded in the viewer.
|
||||
|
||||
Keyword Arguments:
|
||||
name (:obj:`str`): An optional name to reference the table by so it can
|
||||
be accessed from the front-end. If not provided, a name will
|
||||
be generated.
|
||||
index (:obj:`str`): A column name to be used as the primary key.
|
||||
Ignored if a ``Table`` or ``AsyncTable`` is supplied.
|
||||
limit (:obj:`int`): A upper limit on the number of rows in the Table.
|
||||
Cannot be set at the same time as `index`. Ignored if a
|
||||
``Table`` or ``AsyncTable`` is supplied.
|
||||
|
||||
Returns:
|
||||
coro (:obj:`coroutine`): when `AsyncTable` is passed, the `load()` call must be awaited
|
||||
"""
|
||||
name = options.pop("name", str(random()))
|
||||
|
||||
# Reset the viewer when `load()` is called multiple times.
|
||||
if self.table is not None:
|
||||
self.reset()
|
||||
|
||||
if isinstance(data, perspective.perspective.AsyncTable):
|
||||
self._table = data
|
||||
async def load_table():
|
||||
self._client = await self.table.get_client()
|
||||
self.table_name = self.table.get_name()
|
||||
# If the user does not set columns to show, synchronize viewer state
|
||||
# with dataset.
|
||||
if len(self.columns) == 0:
|
||||
self.columns = await self.table.columns()
|
||||
|
||||
return load_table()
|
||||
elif isinstance(data, perspective.perspective.Table):
|
||||
self._table = data
|
||||
self._client = data.get_client()
|
||||
elif isinstance(data, perspective.perspective.View) or isinstance(data, perspective.perspective.AsyncView):
|
||||
raise TypeError(
|
||||
"Views cannot be loaded directly, load a table or raw data instead"
|
||||
)
|
||||
else:
|
||||
self._table = perspective.table(data, name=name, **options)
|
||||
self._client = perspective.GLOBAL_CLIENT
|
||||
|
||||
# If the user does not set columns to show, synchronize viewer state
|
||||
# with dataset.
|
||||
if len(self.columns) == 0:
|
||||
self.columns = self.table.columns()
|
||||
|
||||
self.table_name = self.table.get_name()
|
||||
|
||||
def update(self, data):
|
||||
"""Update the table under management by the viewer with new data.
|
||||
This function follows the semantics of `Table.update()`, and will be
|
||||
affected by whether an index is set on the underlying table.
|
||||
|
||||
When this widget has loaded an ``AsyncTable``, returns a coroutine
|
||||
which must be awaited.
|
||||
|
||||
Args:
|
||||
data (:obj:`dict`|:obj:`list`|:obj:`pandas.DataFrame`): the
|
||||
update data for the table.
|
||||
|
||||
Returns:
|
||||
coro (:obj:`coroutine`): when async, must be awaited
|
||||
"""
|
||||
return self.table.update(data)
|
||||
|
||||
def clear(self):
|
||||
"""Clears the rows of this viewer's ``Table``."""
|
||||
if self.table is not None:
|
||||
return self.table.clear()
|
||||
|
||||
def replace(self, data):
|
||||
"""Replaces the rows of this viewer's `Table` with new data.
|
||||
|
||||
Args:
|
||||
data (:obj:`dict`|:obj:`list`|:obj:`pandas.DataFrame`): new data
|
||||
to set into the table - must conform to the table's schema.
|
||||
|
||||
Returns:
|
||||
coro (:obj:`coroutine`): when async, must be awaited
|
||||
"""
|
||||
if self.table is not None:
|
||||
return self.table.replace(data)
|
||||
|
||||
def save(self):
|
||||
"""Get the viewer's attributes as a dictionary, symmetric with `restore`
|
||||
so that a viewer's configuration can be reproduced."""
|
||||
return {
|
||||
attr: getattr(self, attr)
|
||||
for attr in PerspectiveViewer.PERSISTENT_ATTRIBUTES
|
||||
}
|
||||
|
||||
def restore(self, **kwargs):
|
||||
"""Restore a given set of attributes, passed as kwargs
|
||||
(e.g. dictionary). Symmetric with `save` so that a given viewer's
|
||||
configuration can be reproduced."""
|
||||
for k, v in kwargs.items():
|
||||
if k in PerspectiveViewer.PERSISTENT_ATTRIBUTES:
|
||||
setattr(self, k, v)
|
||||
|
||||
def to_kwargs(self):
|
||||
"""Get the viewer's attributes as a list of kwargs, which can be passed to
|
||||
the viewer constructor"""
|
||||
attrs = self.save()
|
||||
defaults = {
|
||||
"columns": [],
|
||||
"group_by": [],
|
||||
"split_by": [],
|
||||
"aggregates": {},
|
||||
"sort": [],
|
||||
"filter": [],
|
||||
"expressions": {},
|
||||
"plugin_config": {},
|
||||
"version": "2.10.0",
|
||||
}
|
||||
kwargs = {}
|
||||
for key, default in defaults.items():
|
||||
if attrs.get(key) != default:
|
||||
kwargs[key] = attrs[key]
|
||||
return ", ".join(
|
||||
["{}={}".format(attr, repr(val)) for attr, val in kwargs.items()]
|
||||
)
|
||||
|
||||
def reset(self):
|
||||
"""Resets the viewer's attributes and state, but does not delete or
|
||||
modify the underlying `Table`.
|
||||
|
||||
Example:
|
||||
widget = PerspectiveWidget(data, group_by=["date"], plugin=Plugin.XBAR)
|
||||
widget.reset()
|
||||
widget.plugin #
|
||||
"""
|
||||
self.group_by = []
|
||||
self.split_by = []
|
||||
self.filter = []
|
||||
self.sort = []
|
||||
self.expressions = {}
|
||||
self.aggregates = {}
|
||||
self.columns = []
|
||||
self.plugin = "Datagrid"
|
||||
self.plugin_config = {}
|
||||
|
||||
def delete(self, delete_table=True):
|
||||
"""Delete the Viewer's data and clears its internal state. If
|
||||
``delete_table`` is True, the underlying `perspective.Table` and the
|
||||
internal `View` object will be deleted.
|
||||
|
||||
Args:
|
||||
delete_table (:obj:`bool`) : whether the underlying `Table` will be
|
||||
deleted. Defaults to True.
|
||||
|
||||
Returns:
|
||||
coro (:obj:`coroutine`): when async and `delete_table` is `True`,
|
||||
must be awaited
|
||||
"""
|
||||
ret = None
|
||||
if delete_table:
|
||||
# Delete table
|
||||
ret = self.table.delete()
|
||||
self.table_name = None
|
||||
self._table = None
|
||||
|
||||
self.reset()
|
||||
return ret
|
||||
@@ -0,0 +1,101 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
from traitlets import HasTraits, Unicode, List, Bool, Dict, validate, Enum
|
||||
|
||||
import importlib.metadata
|
||||
|
||||
__version__ = importlib.metadata.version("perspective-python")
|
||||
|
||||
from .validate import (
|
||||
validate_version,
|
||||
)
|
||||
|
||||
|
||||
class PerspectiveTraitlets(HasTraits):
|
||||
"""Define the traitlet interface with `PerspectiveJupyterWidget` on the
|
||||
front end. Attributes which are set here are synchronized between the
|
||||
front-end and back-end.
|
||||
|
||||
Examples:
|
||||
>>> widget = perspective.PerspectiveWidget(
|
||||
... data, group_by=["a", "b", "c"])
|
||||
PerspectiveWidget(group_by=["a", "b", "c"])
|
||||
>>> widget.split_by=["b"]
|
||||
>>> widget
|
||||
PerspectiveWidget(group_by=["a", "b", "c"], split_by=["b"])
|
||||
"""
|
||||
|
||||
# `perspective-viewer` options
|
||||
plugin = Unicode("Datagrid").tag(sync=True)
|
||||
columns = List(default_value=[]).tag(sync=True)
|
||||
group_by = List(trait=Unicode(), default_value=[]).tag(sync=True, o=True)
|
||||
split_by = List(trait=Unicode(), default_value=[]).tag(sync=True)
|
||||
aggregates = Dict(default_value={}).tag(sync=True)
|
||||
sort = List(default_value=[]).tag(sync=True)
|
||||
filter = List(default_value=[]).tag(sync=True)
|
||||
expressions = Dict(default_value=[]).tag(sync=True)
|
||||
plugin_config = Dict(default_value={}).tag(sync=True)
|
||||
settings = Bool(True).tag(sync=True)
|
||||
theme = Unicode("Pro Light", allow_none=True).tag(sync=True)
|
||||
|
||||
# used to tell the frontend which table to connect to
|
||||
table_name = Unicode(None, allow_none=True).tag(sync=True)
|
||||
|
||||
server = Bool(False).tag(sync=True)
|
||||
binding_mode = Enum(("server", "client-server")).tag(default="server", sync=True)
|
||||
title = Unicode(None, allow_none=True).tag(sync=True)
|
||||
version = Unicode(__version__).tag(sync=True)
|
||||
|
||||
# @validate("plugin")
|
||||
# def _validate_plugin(self, proposal):
|
||||
# return validate_plugin(proposal.value)
|
||||
|
||||
# @validate("columns")
|
||||
# def _validate_columns(self, proposal):
|
||||
# return validate_columns(proposal.value)
|
||||
|
||||
# @validate("group_by")
|
||||
# def _validate_group_by(self, proposal):
|
||||
# return validate_group_by(proposal.value)
|
||||
|
||||
# @validate("split_by")
|
||||
# def _validate_split_by(self, proposal):
|
||||
# return validate_split_by(proposal.value)
|
||||
|
||||
# @validate("aggregates")
|
||||
# def _validate_aggregates(self, proposal):
|
||||
# return validate_aggregates(proposal.value)
|
||||
|
||||
# @validate("sort")
|
||||
# def _validate_sort(self, proposal):
|
||||
# return validate_sort(proposal.value)
|
||||
|
||||
# @validate("filter")
|
||||
# def _validate_filter(self, proposal):
|
||||
# return validate_filter(proposal.value)
|
||||
|
||||
# @validate("expressions")
|
||||
# def _validate_expressions(self, proposal):
|
||||
# return validate_expressions(proposal.value)
|
||||
|
||||
# @validate("plugin_config")
|
||||
# def _validate_plugin_config(self, proposal):
|
||||
# return validate_plugin_config(proposal.value)
|
||||
|
||||
# @validate("title")
|
||||
# def _validate_title(self, proposal):
|
||||
# return validate_title(proposal.value)
|
||||
|
||||
@validate("version")
|
||||
def _validate_version(self, proposal):
|
||||
return validate_version(proposal.value)
|
||||
Reference in New Issue
Block a user