chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,71 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
[package]
|
||||
name = "perspective-server"
|
||||
version = "4.5.2"
|
||||
authors = ["Andrew Stein <steinlink@gmail.com>"]
|
||||
edition = "2024"
|
||||
description = "A data visualization and analytics component, especially well-suited for large and/or streaming datasets."
|
||||
repository = "https://github.com/perspective-dev/perspective"
|
||||
license = "Apache-2.0"
|
||||
homepage = "https://perspective-dev.github.io"
|
||||
keywords = []
|
||||
build = "build.rs"
|
||||
include = [
|
||||
"src/**/*",
|
||||
"build.rs",
|
||||
"include/**/*",
|
||||
"cpp/**/*",
|
||||
"docs/**/*",
|
||||
"cmake/**/*",
|
||||
"Cargo.toml",
|
||||
]
|
||||
|
||||
[features]
|
||||
default = ["python"]
|
||||
external-cpp = []
|
||||
wasm-exceptions = []
|
||||
python = []
|
||||
disable-cpp = []
|
||||
|
||||
[build-dependencies]
|
||||
cmake = "0.1.58"
|
||||
num_cpus = "^1.15.0"
|
||||
shlex = "1.3.0"
|
||||
protobuf-src = { version = "2.1.1" }
|
||||
|
||||
[dependencies]
|
||||
perspective-client = { version = "4.5.2" }
|
||||
|
||||
# Key order is frequently implicitly relied upon in dynamic languages, so for
|
||||
# convenience we try to provide this (as well as explicit metadata calls).
|
||||
indexmap = { version = "2.2.6", features = ["serde"] }
|
||||
|
||||
# Convenient way to crawl the C++ static archive path
|
||||
link-cplusplus = "1.0.12"
|
||||
|
||||
async-lock = "2.5.0"
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
serde_json = { version = "1.0.107", features = ["raw_value"] }
|
||||
tracing = { version = ">=0.1.36" }
|
||||
thiserror = { version = "1.0.55" }
|
||||
futures = "0.3"
|
||||
|
||||
[dependencies.prost]
|
||||
version = "0.12.3"
|
||||
default-features = false
|
||||
features = ["prost-derive", "std"]
|
||||
|
||||
[lib]
|
||||
crate-type = ["rlib"]
|
||||
path = "src/lib.rs"
|
||||
@@ -0,0 +1,94 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import * as url from "node:url";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
import "zx/globals";
|
||||
|
||||
const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1);
|
||||
|
||||
const env = process.env.PSP_DEBUG ? "debug" : "release";
|
||||
const cwd = path.join(process.cwd(), "build", env);
|
||||
const cmake_dir = path.join(__dirname, "cpp", "perspective");
|
||||
const { compress } = await import("pro_self_extracting_wasm");
|
||||
|
||||
delete process.env.NODE;
|
||||
|
||||
async function run_emsdk(cmd) {
|
||||
const emsdkdir = `${__dirname}/../../.emsdk`;
|
||||
const { emscripten } = JSON.parse(
|
||||
fs.readFileSync(`${__dirname}/../../package.json`),
|
||||
);
|
||||
|
||||
if (!emscripten) {
|
||||
throw new Error("Emscripten version not specified in package.json");
|
||||
}
|
||||
|
||||
await $({ env: { FORCE_COLOR: "1", ...process.env } })`cd ${emsdkdir} \
|
||||
&& source ./emsdk_env.sh >/dev/null 2>&1 \
|
||||
&& emsdk activate ${emscripten} >/dev/null \
|
||||
&& cd ${cwd} \
|
||||
&& ${cmd}`.verbose();
|
||||
}
|
||||
|
||||
let cmake_flags = [];
|
||||
let make_flags = [];
|
||||
|
||||
if (!!process.env.PSP_BUILD_VERBOSE) {
|
||||
cmake_flags.push("-Wdev");
|
||||
cmake_flags.push("--debug-output");
|
||||
make_flags.push("VERBOSE=1");
|
||||
} else {
|
||||
cmake_flags.push("-Wno-dev"); // suppress developer warnings
|
||||
}
|
||||
|
||||
try {
|
||||
fs.mkdirSync(cwd, { recursive: true });
|
||||
process.env.CLICOLOR_FORCE = 1;
|
||||
const out = await $`cargo metadata`;
|
||||
const p = JSON.parse(out.stdout).packages.find(
|
||||
(x) => x.name === "perspective-client",
|
||||
).manifest_path;
|
||||
|
||||
const pp = path.parse(p);
|
||||
await run_emsdk([
|
||||
`emcmake`,
|
||||
`cmake`,
|
||||
cmake_dir,
|
||||
...cmake_flags,
|
||||
`-DCMAKE_BUILD_TYPE=${env}`,
|
||||
`-DRAPIDJSON_BUILD_EXAMPLES=OFF`,
|
||||
`-DPSP_PROTO_PATH=${pp.dir}`,
|
||||
`-DCMAKE_COLOR_DIAGNOSTICS=ON`,
|
||||
]);
|
||||
|
||||
await run_emsdk([
|
||||
`emmake`,
|
||||
`make`,
|
||||
`-j${process.env.PSP_NUM_CPUS || os.cpus().length}`,
|
||||
...make_flags,
|
||||
]);
|
||||
|
||||
fs.cpSync(`build/${env}/web`, "dist/wasm", { recursive: true });
|
||||
if (!process.env.PSP_HEAP_INSTRUMENTS) {
|
||||
compress(
|
||||
`./dist/wasm/perspective-server.wasm`,
|
||||
`./dist/wasm/perspective-server.wasm`,
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use cmake::Config;
|
||||
use protobuf_src::protoc;
|
||||
use shlex::Shlex;
|
||||
|
||||
fn main() -> Result<(), std::io::Error> {
|
||||
if std::env::var("DOCS_RS").is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if std::option_env!("PSP_DISABLE_CPP").is_none()
|
||||
&& std::env::var("CARGO_FEATURE_DISABLE_CPP").is_err()
|
||||
&& let Some(artifact_dir) = cmake_build()?
|
||||
{
|
||||
cmake_link_deps(&artifact_dir)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn cmake_build() -> Result<Option<PathBuf>, std::io::Error> {
|
||||
let mut dst = Config::new("cpp/perspective");
|
||||
if let Some(cpp_build_dir) = std::option_env!("PSP_CPP_BUILD_DIR") {
|
||||
std::fs::create_dir_all(cpp_build_dir)?;
|
||||
dst.out_dir(cpp_build_dir);
|
||||
}
|
||||
|
||||
let profile = std::env::var("PROFILE").unwrap();
|
||||
dst.always_configure(true);
|
||||
dst.define("CMAKE_BUILD_TYPE", profile.as_str());
|
||||
dst.define("ARROW_BUILD_EXAMPLES", "OFF");
|
||||
dst.define("RAPIDJSON_BUILD_EXAMPLES", "OFF");
|
||||
dst.define("ARROW_CXX_FLAGS_DEBUG", "-Wno-error");
|
||||
dst.define(
|
||||
"PSP_PROTOC_PATH",
|
||||
protoc()
|
||||
.parent()
|
||||
.expect("protoc() returned root path or empty string"),
|
||||
);
|
||||
dst.define("CMAKE_COLOR_DIAGNOSTICS", "ON");
|
||||
dst.define(
|
||||
"PSP_PROTO_PATH",
|
||||
std::env::var("DEP_PERSPECTIVE_CLIENT_PROTO_PATH").unwrap(),
|
||||
);
|
||||
|
||||
dst.env(
|
||||
"DEP_PERSPECTIVE_CLIENT_PROTO_PATH",
|
||||
std::env::var("DEP_PERSPECTIVE_CLIENT_PROTO_PATH").unwrap(),
|
||||
);
|
||||
|
||||
if cfg!(target_os = "macos") {
|
||||
// Set CMAKE_OSX_ARCHITECTURES et al. for Mac builds. Arrow does not forward on
|
||||
// CMAKE_OSX_ARCHITECTURES but it does forward on a CMAKE_TOOLCHAIN_FILE. In
|
||||
// Conda builds, the environment sets `CMAKE_ARGS` up with various
|
||||
// toolchain arguments. This block may need to be patched out or
|
||||
// adjusted for Conda.
|
||||
let toolchain_file = match std::env::var("PSP_ARCH").as_deref() {
|
||||
Ok("x86_64") => Some("./cmake/toolchains/darwin-x86_64.cmake"),
|
||||
Ok("aarch64") => Some("./cmake/toolchains/darwin-arm64.cmake"),
|
||||
Err(std::env::VarError::NotPresent) => None,
|
||||
arch @ Ok(_) | arch @ Err(_) => {
|
||||
panic!("Unknown PSP_ARCH value: {arch:?}")
|
||||
},
|
||||
};
|
||||
|
||||
if let Some(path) = toolchain_file {
|
||||
dst.define(
|
||||
"CMAKE_TOOLCHAIN_FILE",
|
||||
std::fs::canonicalize(path).expect("Failed to canonicalize toolchain file."),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if std::env::var("TARGET")
|
||||
.unwrap_or_default()
|
||||
.contains("wasm32")
|
||||
{
|
||||
dst.define("PSP_WASM_BUILD", "1");
|
||||
} else {
|
||||
dst.define("PSP_WASM_BUILD", "0");
|
||||
}
|
||||
|
||||
if std::env::var("CARGO_FEATURE_PYTHON").is_ok() {
|
||||
dst.define("CMAKE_POSITION_INDEPENDENT_CODE", "ON");
|
||||
dst.define("PSP_PYTHON_BUILD", "1");
|
||||
}
|
||||
|
||||
if std::env::var("CARGO_FEATURE_EXTERNAL_CPP").is_err() {
|
||||
dst.env("PSP_DISABLE_CLANGD", "1");
|
||||
}
|
||||
|
||||
// WASM Exceptions don't work with the prebuilt Pyodide distribution.
|
||||
// It must be rebuilt with WASM exceptions enabled
|
||||
if std::env::var("CARGO_FEATURE_WASM_EXCEPTIONS").is_ok() {
|
||||
dst.define("PSP_WASM_EXCEPTIONS", "1");
|
||||
} else {
|
||||
dst.define("PSP_WASM_EXCEPTIONS", "0");
|
||||
}
|
||||
|
||||
// Propagates to nested `cmake --build` invocations (e.g. the per-dep
|
||||
// fetch builds in FindInstallDependency.cmake), unlike `-j` via `build_arg`
|
||||
// which only reaches the top-level native tool.
|
||||
dst.env("CMAKE_BUILD_PARALLEL_LEVEL", num_cpus::get().to_string());
|
||||
|
||||
// Conda sets CMAKE_ARGS for e.g. cross-compiling toolchain in the environment -
|
||||
// normally they are passed directly to a cmake invocation in the recipe,
|
||||
// but our conda recipe doesn't directly invoke cmake
|
||||
if let Ok(cmake_args) = std::env::var("CMAKE_ARGS") {
|
||||
println!("cargo:warning=Setting CMAKE_ARGS from environment {cmake_args:?}");
|
||||
for arg in Shlex::new(&cmake_args) {
|
||||
dst.configure_arg(arg);
|
||||
}
|
||||
}
|
||||
|
||||
println!("cargo:warning=Building cmake {profile}");
|
||||
if std::env::var("PSP_BUILD_VERBOSE").unwrap_or_default() != "" {
|
||||
// checks non-empty env var
|
||||
dst.very_verbose(true);
|
||||
}
|
||||
|
||||
let artifact_dir = dst.build();
|
||||
Ok(Some(artifact_dir))
|
||||
}
|
||||
|
||||
fn cmake_link_deps(cmake_build_dir: &Path) -> Result<(), std::io::Error> {
|
||||
println!(
|
||||
"cargo:rustc-link-search=native={}/build",
|
||||
cmake_build_dir.display()
|
||||
);
|
||||
|
||||
// println!("cargo:warning=MESSAGE {}/build", cmake_build_dir.display());
|
||||
println!("cargo:rustc-link-lib=static=psp");
|
||||
link_cmake_static_archives(cmake_build_dir)?;
|
||||
println!("cargo:rerun-if-changed=cpp/perspective");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Walk the cmake output path and emit link instructions for all archives.
|
||||
/// TODO Can this be faster pls?
|
||||
/// TODO update apparently not https://github.com/rust-lang/cmake-rs/issues/149
|
||||
fn link_cmake_static_archives(dir: &Path) -> Result<(), std::io::Error> {
|
||||
if dir.is_dir() {
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let path = entry?.path();
|
||||
if path.is_dir() {
|
||||
link_cmake_static_archives(&path)?;
|
||||
} else {
|
||||
let ext = path.extension().as_ref().map(|x| x.to_string_lossy());
|
||||
let stem = path.file_stem().as_ref().map(|x| x.to_string_lossy());
|
||||
let is_archive = (cfg!(windows)
|
||||
&& ext.as_deref() == Some("lib")
|
||||
&& stem.as_deref() != Some("perspective"))
|
||||
|| (!cfg!(windows) && ext.as_deref() == Some("a"));
|
||||
if is_archive {
|
||||
let a = if cfg!(windows) {
|
||||
stem.unwrap().to_string()
|
||||
} else {
|
||||
stem.expect("bad")[3..].to_string()
|
||||
};
|
||||
|
||||
// println!("cargo:warning=static link {} {}", a, dir.display());
|
||||
println!("cargo:rustc-link-search=native={}", dir.display());
|
||||
println!("cargo:rustc-link-lib=static={a}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
import * as fs from "node:fs";
|
||||
|
||||
fs.rmSync("dist", { recursive: true, force: true });
|
||||
@@ -0,0 +1,18 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(Boost-download NONE)
|
||||
|
||||
# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
|
||||
cmake_policy(SET CMP0097 NEW)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(Boost
|
||||
URL "https://sourceforge.net/projects/boost/files/boost/1.82.0/boost_1_82_0.tar.gz"
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/Boost-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/Boost-build"
|
||||
SOURCE_SUBDIR ""
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(pybind11-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(pybind11
|
||||
GIT_REPOSITORY https://github.com/pybind/pybind11.git
|
||||
GIT_TAG v2.10.4
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/pybind11-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/pybind11-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,24 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(arrow-download NONE)
|
||||
|
||||
# Makes GIT_SUBMODULES "" in ExternalProject_Add skip initializing submodules
|
||||
cmake_policy(SET CMP0097 NEW)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(apachearrow
|
||||
GIT_REPOSITORY https://github.com/apache/arrow.git
|
||||
GIT_TAG apache-arrow-18.1.0
|
||||
GIT_SUBMODULES ""
|
||||
GIT_SHALLOW TRUE
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/arrow-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/arrow-build"
|
||||
SOURCE_SUBDIR "cpp"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
# This patch is to work around https://github.com/apache/arrow/issues/44384
|
||||
# It can be removed when a version of Arrow is released with https://github.com/apache/arrow/pull/44385
|
||||
PATCH_COMMAND "${CMAKE_COMMAND}" -E chdir <SOURCE_DIR> git apply "${CMAKE_SOURCE_DIR}/patches/fix_arrow_libtool.patch" && "${CMAKE_COMMAND}" -E chdir <SOURCE_DIR> git apply "${CMAKE_SOURCE_DIR}/patches/arrow_strptime.patch"
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(date-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(date
|
||||
GIT_REPOSITORY https://github.com/HowardHinnant/date.git
|
||||
GIT_TAG v3.0.0
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/date-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/date-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(exprtk-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(exprtk
|
||||
GIT_REPOSITORY https://github.com/ArashPartow/exprtk.git
|
||||
GIT_TAG 0.0.3
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/exprtk-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/exprtk-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(hopscotch-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(hopscotch
|
||||
GIT_REPOSITORY https://github.com/Tessil/hopscotch-map.git
|
||||
GIT_TAG v2.3.1
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/hopscotch-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/hopscotch-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,3 @@
|
||||
CompileFlags:
|
||||
@CLANGD_ADD_FLAGS@
|
||||
CompilationDatabase: @CMAKE_BINARY_DIR@
|
||||
@@ -0,0 +1,13 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
if(NOT WIN32)
|
||||
string(ASCII 27 Esc)
|
||||
set(ColorReset "${Esc}[m")
|
||||
set(Red "${Esc}[1;31m")
|
||||
set(Green "${Esc}[1;32m")
|
||||
set(Yellow "${Esc}[1;33m")
|
||||
set(Blue "${Esc}[1;34m")
|
||||
set(Magenta "${Esc}[1;35m")
|
||||
set(Cyan "${Esc}[1;36m")
|
||||
set(White "${Esc}[1;37m")
|
||||
endif()
|
||||
@@ -0,0 +1,76 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
# Program: Visualization Toolkit
|
||||
# Module: Copyright.txt
|
||||
|
||||
# Copyright (c) 1993-2015 Ken Martin, Will Schroeder, Bill Lorensen
|
||||
# All rights reserved.
|
||||
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
|
||||
# * Redistributions of source code must retain the above copyright notice,
|
||||
# this list of conditions and the following disclaimer.
|
||||
|
||||
# * Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
|
||||
# * Neither name of Ken Martin, Will Schroeder, or Bill Lorensen nor the names
|
||||
# of any contributors may be used to endorse or promote products derived
|
||||
# from this software without specific prior written permission.
|
||||
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS''
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR
|
||||
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
# - Try to find ExprTk lib
|
||||
#
|
||||
# This module supports requiring a minimum version, e.g. you can do
|
||||
# find_package(ExprTk 2.71828)
|
||||
# to require version 2.71828 or newer of ExprTk.
|
||||
#
|
||||
# Once done this will define
|
||||
#
|
||||
# ExprTk_FOUND - system has eigen lib with correct version
|
||||
# ExprTk_INCLUDE_DIRS - the eigen include directory
|
||||
# ExprTk_VERSION - eigen version
|
||||
#
|
||||
# And the following imported target:
|
||||
#
|
||||
# ExprTk::ExprTk
|
||||
|
||||
find_path(ExprTk_INCLUDE_DIR
|
||||
NAMES exprtk.hpp
|
||||
DOC "Path to ExprTk header")
|
||||
mark_as_advanced(ExprTk_INCLUDE_DIR)
|
||||
|
||||
if (ExprTk_INCLUDE_DIR)
|
||||
file(STRINGS "${ExprTk_INCLUDE_DIR}/exprtk.hpp" _exprtk_version_header
|
||||
REGEX "static \\* version")
|
||||
string(REGEX MATCH "static \\* version = \"([0-9.]+)\"" _exprtk_version_match "${_exprtk_version_header}")
|
||||
set(ExprTk_VERSION "${CMAKE_MATCH_1}")
|
||||
unset(_exprtk_version_header)
|
||||
unset(_exprtk_version_match)
|
||||
endif ()
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(ExprTk
|
||||
REQUIRED_VARS ExprTk_INCLUDE_DIR
|
||||
VERSION_VAR ExprTk_VERSION)
|
||||
|
||||
if (ExprTk_FOUND)
|
||||
set(ExprTk_INCLUDE_DIRS "${ExprTk_INCLUDE_DIR}")
|
||||
if (NOT TARGET ExprTk::ExprTk)
|
||||
add_library(ExprTk::ExprTk INTERFACE IMPORTED)
|
||||
set_target_properties(ExprTk::ExprTk PROPERTIES
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ExprTk_INCLUDE_DIR}")
|
||||
endif ()
|
||||
endif ()
|
||||
@@ -0,0 +1,100 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
# Workaround for: https://gitlab.kitware.com/cmake/cmake/-/issues/22740
|
||||
set(D "$")
|
||||
|
||||
# ##################################################
|
||||
# Helper to grab dependencies from remote sources #
|
||||
# ##################################################
|
||||
function(psp_build_dep name cmake_file)
|
||||
if(EXISTS ${CMAKE_BINARY_DIR}/${name}-build)
|
||||
psp_build_message("${Cyan}Dependency found - not rebuilding - ${CMAKE_BINARY_DIR}/${name}-build${ColorReset}")
|
||||
elseif(NAME STREQUAL "Boost")
|
||||
psp_build_message("${Cyan}Not building boost${ColorReset}")
|
||||
else()
|
||||
configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt)
|
||||
set(_cwd ${CMAKE_BINARY_DIR}/${name}-download)
|
||||
|
||||
message(STATUS "Configuring ${name} in ${_cwd}")
|
||||
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
|
||||
RESULT_VARIABLE result
|
||||
OUTPUT_VARIABLE cmd_output
|
||||
ERROR_VARIABLE cmd_error
|
||||
WORKING_DIRECTORY ${_cwd})
|
||||
|
||||
if(result)
|
||||
message(FATAL_ERROR "CMake step for ${name} failed:\nSTDOUT:${cmd_output}\nSTDERR: ${cmd_error}")
|
||||
endif()
|
||||
|
||||
message("${cmd_output}")
|
||||
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} --build .
|
||||
RESULT_VARIABLE result
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/${name}-download)
|
||||
|
||||
if(result)
|
||||
message(FATAL_ERROR "Build step for ${name} failed: ${result}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(${name} STREQUAL arrow)
|
||||
set(ARROW_DEFINE_OPTIONS ON)
|
||||
set(ARROW_BUILD_SHARED OFF)
|
||||
set(ARROW_BUILD_STATIC ON)
|
||||
set(ARROW_BUILD_INTEGRATION OFF)
|
||||
set(ARROW_JEMALLOC OFF)
|
||||
set(ARROW_CSV ON)
|
||||
set(ARROW_LZ4 ON)
|
||||
set(ARROW_WITH_ZSTD ON)
|
||||
set(ARROW_WITH_LZ4 ON)
|
||||
set(ARROW_NO_EXPORT ON)
|
||||
set(ARROW_CXXFLAGS " -Wno-documentation ")
|
||||
set(ARROW_DEPENDENCY_SOURCE "BUNDLED" CACHE STRING "override arrow's dependency source" FORCE)
|
||||
# if (PSP_ENABLE_WASM)
|
||||
# set(ARROW_CXX_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE)
|
||||
# set(ARROW_C_FLAGS_RELEASE " -msimd128 -mbulk-memory -mrelaxed-simd -s MEMORY64=1 " CACHE STRING "override arrow's dependency source" FORCE )
|
||||
# endif()
|
||||
if(PSP_WASM_BUILD)
|
||||
set(ARROW_ENABLE_THREADING OFF)
|
||||
else()
|
||||
set(ARROW_ENABLE_THREADING ON)
|
||||
endif()
|
||||
|
||||
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src/cpp/
|
||||
${CMAKE_BINARY_DIR}/${name}-build
|
||||
EXCLUDE_FROM_ALL
|
||||
)
|
||||
set(${name}_INCLUDE_DIRS
|
||||
"${CMAKE_BINARY_DIR}/${name}-build/src" # needed for generated headers, e.g. <arrow/util/config.h>
|
||||
"${CMAKE_BINARY_DIR}/${name}-src/cpp/src"
|
||||
PARENT_SCOPE)
|
||||
elseif(${name} STREQUAL re2)
|
||||
# Overwrite re2's CMakeLists with our custom CMakeLists.
|
||||
configure_file(${PSP_CMAKE_MODULE_PATH}/${name}/CMakeLists.txt ${CMAKE_BINARY_DIR}/${name}-src/ COPYONLY)
|
||||
|
||||
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
|
||||
${CMAKE_BINARY_DIR}/${name}-build
|
||||
EXCLUDE_FROM_ALL)
|
||||
set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src PARENT_SCOPE)
|
||||
elseif(${name} STREQUAL protobuf)
|
||||
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
|
||||
${CMAKE_BINARY_DIR}/${name}-build
|
||||
EXCLUDE_FROM_ALL)
|
||||
set(${name}_INCLUDE_DIRS ${CMAKE_BINARY_DIR}/${name}-src/src PARENT_SCOPE)
|
||||
elseif(${name} STREQUAL rapidjson)
|
||||
set(RAPIDJSON_SSE42 ON CACHE BOOL "Enable RapidJSON SIMD")
|
||||
set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
|
||||
add_subdirectory(${CMAKE_BINARY_DIR}/${name}-src
|
||||
${CMAKE_BINARY_DIR}/${name}-build
|
||||
EXCLUDE_FROM_ALL)
|
||||
set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
|
||||
# Header-only dependencies without a build step - no add_subdirectory()
|
||||
elseif(${name} MATCHES "^(Boost|exprtk)")
|
||||
set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src" PARENT_SCOPE)
|
||||
elseif(${name} MATCHES "^(hopscotch|date|ordered-map)$")
|
||||
set(${name}_INCLUDE_DIRS "${CMAKE_BINARY_DIR}/${name}-src/include" PARENT_SCOPE)
|
||||
else()
|
||||
message(FATAL_ERROR "Unknown dependency `${name}`")
|
||||
endif()
|
||||
endfunction()
|
||||
@@ -0,0 +1,126 @@
|
||||
set(DEFAULT_PROTOBUF_VERSION "33.2")
|
||||
|
||||
option(PSP_PROTOC_PATH "Path to the protoc binary" "")
|
||||
|
||||
# Function to download and extract protoc
|
||||
function(download_protoc VERSION DESTINATION)
|
||||
if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
|
||||
set(PROTOC_ZIP "protoc-${VERSION}-win64.zip")
|
||||
elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
|
||||
set(PROTOC_ZIP "protoc-${VERSION}-osx-x86_64.zip")
|
||||
elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
|
||||
if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64")
|
||||
set(PROTOC_ZIP "protoc-${VERSION}-linux-aarch_64.zip")
|
||||
else()
|
||||
set(PROTOC_ZIP "protoc-${VERSION}-linux-x86_64.zip")
|
||||
endif()
|
||||
else()
|
||||
message(FATAL_ERROR "Unsupported host system: ${CMAKE_HOST_SYSTEM_NAME}")
|
||||
endif()
|
||||
|
||||
set(PROTOC_URL "https://github.com/protocolbuffers/protobuf/releases/download/v${VERSION}/${PROTOC_ZIP}")
|
||||
set(DOWNLOAD_PATH "${CMAKE_BINARY_DIR}/downloaded/${PROTOC_ZIP}")
|
||||
|
||||
file(DOWNLOAD ${PROTOC_URL} ${DOWNLOAD_PATH}
|
||||
STATUS DOWNLOAD_STATUS
|
||||
SHOW_PROGRESS)
|
||||
|
||||
list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
|
||||
if(STATUS_CODE EQUAL 0)
|
||||
message(STATUS "Downloaded protoc: ${DOWNLOAD_PATH}")
|
||||
file(ARCHIVE_EXTRACT INPUT ${DOWNLOAD_PATH} DESTINATION ${DESTINATION})
|
||||
file(REMOVE ${DOWNLOAD_PATH})
|
||||
else()
|
||||
message(FATAL_ERROR "Failed to download protoc: Status code ${STATUS_CODE}")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
find_program(Protobuf_EXECUTABLE NAMES protoc PATHS ${PSP_PROTOC_PATH} ENV PATH ${CMAKE_BINARY_DIR}/protoc-release/bin/)
|
||||
|
||||
if(NOT Protobuf_EXECUTABLE)
|
||||
message(STATUS "Protobuf_EXECUTABLE not found, searching for protoc")
|
||||
download_protoc(${DEFAULT_PROTOBUF_VERSION} ${CMAKE_BINARY_DIR}/protoc-release)
|
||||
set(PROTOC_EXECUTABLE ${CMAKE_BINARY_DIR}/protoc-release/bin/protoc)
|
||||
# find_program(PROTOC_EXECUTABLE NAMES protoc)
|
||||
else()
|
||||
message(STATUS "Using Protobuf_EXECUTABLE from parent scope")
|
||||
set(PROTOC_EXECUTABLE ${Protobuf_EXECUTABLE})
|
||||
endif()
|
||||
|
||||
if(NOT PROTOC_EXECUTABLE)
|
||||
message(FATAL_ERROR "protoc not found")
|
||||
set(Protoc_FOUND FALSE)
|
||||
else()
|
||||
set(Protoc_FOUND TRUE)
|
||||
message(STATUS "Found protoc: ${PROTOC_EXECUTABLE}")
|
||||
|
||||
function(CHECK_PROTOC_VERSION)
|
||||
execute_process(
|
||||
COMMAND ${PROTOC_EXECUTABLE} --version
|
||||
OUTPUT_VARIABLE PROTOC_VERSION_OUTPUT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE
|
||||
)
|
||||
|
||||
if(NOT PROTOC_VERSION_OUTPUT MATCHES "^libprotoc ([0-9]+)\\.([0-9]+)(:?\\.([0-9]+))?")
|
||||
message(WARNING "Unable to determine protoc version")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(PROTOC_VERSION_MAJOR ${CMAKE_MATCH_1})
|
||||
set(PROTOC_VERSION_MINOR ${CMAKE_MATCH_2})
|
||||
set(PROTOC_VERSION_PATCH ${CMAKE_MATCH_4})
|
||||
|
||||
if (NOT PROTOC_VERSION_PATCH)
|
||||
set(FOUND_VERSION "${PROTOC_VERSION_MAJOR}.${PROTOC_VERSION_MINOR}")
|
||||
else()
|
||||
set(FOUND_VERSION "${PROTOC_VERSION_MAJOR}.${PROTOC_VERSION_MINOR}.${PROTOC_VERSION_PATCH}")
|
||||
endif()
|
||||
|
||||
# Force the external project to use the same version as our installed protoc CLI
|
||||
set(LIBPROTOBUF_VERSION "v${FOUND_VERSION}" PARENT_SCOPE)
|
||||
|
||||
set(MIN_PROTOC_VERSION_MAJOR 3)
|
||||
|
||||
if(PROTOC_VERSION_MAJOR LESS MIN_PROTOC_VERSION_MAJOR)
|
||||
message(FATAL_ERROR "protoc version ${FOUND_VERSION} is too old, at least ${MIN_PROTOC_VERSION_MAJOR}.x.x is required")
|
||||
else()
|
||||
message(STATUS "Found protoc version ${FOUND_VERSION}")
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
# Check the version of protoc
|
||||
CHECK_PROTOC_VERSION()
|
||||
endif()
|
||||
|
||||
function(protobuf_generate_cpp SRCS HDRS)
|
||||
if(NOT ARGN)
|
||||
message(SEND_ERROR "Error: protobuf_generate_cpp() called without any proto files")
|
||||
return()
|
||||
endif()
|
||||
|
||||
set(_PROTOBUF_GENERATE_CPP_SRCS)
|
||||
set(_PROTOBUF_GENERATE_CPP_HDRS)
|
||||
|
||||
foreach(PROTO_FILE ${ARGN})
|
||||
get_filename_component(PROTO_NAME ${PROTO_FILE} NAME_WE)
|
||||
get_filename_component(PROTO_PATH ${PROTO_FILE} PATH)
|
||||
|
||||
set(GENERATED_SRC "${CMAKE_CURRENT_BINARY_DIR}/${PROTO_NAME}.pb.cc")
|
||||
set(GENERATED_HDR "${CMAKE_CURRENT_BINARY_DIR}/${PROTO_NAME}.pb.h")
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${GENERATED_SRC} ${GENERATED_HDR}
|
||||
COMMAND ${PROTOC_EXECUTABLE} --cpp_out ${CMAKE_CURRENT_BINARY_DIR} --proto_path ${PROTO_PATH} ${PROTO_FILE}
|
||||
DEPENDS ${PROTO_FILE} ${PROTOC_EXECUTABLE}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
COMMENT "Generating ${GENERATED_SRC} and ${GENERATED_HDR} from ${PROTO_FILE}"
|
||||
)
|
||||
|
||||
set_source_files_properties(${GENERATED_SRC} ${GENERATED_HDR} PROPERTIES GENERATED TRUE)
|
||||
|
||||
list(APPEND _PROTOBUF_GENERATE_CPP_SRCS ${GENERATED_SRC})
|
||||
list(APPEND _PROTOBUF_GENERATE_CPP_HDRS ${GENERATED_HDR})
|
||||
endforeach()
|
||||
set(${SRCS} ${_PROTOBUF_GENERATE_CPP_SRCS} PARENT_SCOPE)
|
||||
set(${HDRS} ${_PROTOBUF_GENERATE_CPP_HDRS} PARENT_SCOPE)
|
||||
endfunction()
|
||||
@@ -0,0 +1,70 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
# Copyright 2017 gRPC authors.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
find_package(re2 QUIET CONFIG)
|
||||
if(re2_FOUND)
|
||||
message(STATUS "Found RE2 via CMake.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
# As per https://github.com/grpc/grpc/issues/25434, idempotence is necessary
|
||||
# because CMake fails when another target with the same name already exists.
|
||||
if(TARGET re2::re2)
|
||||
message(STATUS "Found RE2 via pkg-config already?")
|
||||
return()
|
||||
endif()
|
||||
|
||||
find_package(PkgConfig QUIET)
|
||||
if (PKG_CONFIG_FOUND)
|
||||
# TODO(junyer): Use the IMPORTED_TARGET option whenever CMake 3.6 (or newer)
|
||||
# becomes the minimum required: that will take care of the add_library() and
|
||||
# set_property() calls; then we can simply alias PkgConfig::RE2 as re2::re2.
|
||||
# For now, we can only set INTERFACE_* properties that existed in CMake 3.5.
|
||||
pkg_check_modules(RE2 QUIET re2)
|
||||
endif()
|
||||
|
||||
if(RE2_FOUND)
|
||||
set(re2_FOUND true)
|
||||
add_library(re2::re2 INTERFACE IMPORTED)
|
||||
if(RE2_INCLUDE_DIRS)
|
||||
set_property(TARGET re2::re2 PROPERTY
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${RE2_INCLUDE_DIRS}")
|
||||
endif()
|
||||
if(RE2_CFLAGS_OTHER)
|
||||
# Filter out the -std flag, which is handled by CMAKE_CXX_STANDARD.
|
||||
# TODO(junyer): Use the FILTER option whenever CMake 3.6 (or newer)
|
||||
# becomes the minimum required: that will allow this to be concise.
|
||||
foreach(flag IN LISTS RE2_CFLAGS_OTHER)
|
||||
if("${flag}" MATCHES "^-std=")
|
||||
list(REMOVE_ITEM RE2_CFLAGS_OTHER "${flag}")
|
||||
endif()
|
||||
endforeach()
|
||||
set_property(TARGET re2::re2 PROPERTY
|
||||
INTERFACE_COMPILE_OPTIONS "${RE2_CFLAGS_OTHER}")
|
||||
endif()
|
||||
|
||||
if(RE2_LDFLAGS)
|
||||
set_property(TARGET re2::re2 PROPERTY
|
||||
INTERFACE_LINK_LIBRARIES "${RE2_LDFLAGS}")
|
||||
endif()
|
||||
|
||||
message(STATUS "Found RE2 via pkg-config.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(re2_FIND_REQUIRED)
|
||||
message(WARNING "Failed to find RE2.")
|
||||
endif()
|
||||
@@ -0,0 +1,9 @@
|
||||
set(LLVM_ROOT $ENV{LLVM_ROOT})
|
||||
|
||||
set(CMAKE_C_COMPILER "${LLVM_ROOT}/bin/clang")
|
||||
set(CMAKE_CXX_COMPILER "${LLVM_ROOT}/bin/clang++")
|
||||
set(CMAKE_AR "${LLVM_ROOT}/bin/llvm-ar")
|
||||
set(CMAKE_RANLIB "${LLVM_ROOT}/bin/llvm-ranlib")
|
||||
|
||||
# include_directories(BEFORE SYSTEM "${LLVM_ROOT}/include")
|
||||
# link_directories(BEFORE "${LLVM_ROOT}/lib")
|
||||
@@ -0,0 +1,38 @@
|
||||
if(NOT WORKSPACE_ROOT)
|
||||
message(STATUS "${Yellow}WORKSPACE_ROOT not set, assuming ${CMAKE_CURRENT_SOURCE_DIR}/../../../..${ColorReset}")
|
||||
set(WORKSPACE_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../../../..")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
|
||||
set(CLANGD_ADD_FLAGS "Add:
|
||||
[
|
||||
-target,
|
||||
wasm32-unknown-emscripten,
|
||||
--sysroot=${WORKSPACE_ROOT}/.emsdk/upstream/emscripten/cache/sysroot,
|
||||
]")
|
||||
else()
|
||||
set(CLANGD_ADD_FLAGS "")
|
||||
endif()
|
||||
|
||||
|
||||
# Check for existence of .clangd file
|
||||
if(EXISTS ${CMAKE_CURRENT_LIST_DIR}/.clangd.in)
|
||||
if(${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
message(STATUS "${Cyan}Found ${CMAKE_CURRENT_LIST_DIR}/.clangd.in${ColorReset}")
|
||||
# Configure the .clangd file from the template
|
||||
configure_file(
|
||||
${CMAKE_CURRENT_LIST_DIR}/.clangd.in
|
||||
${CMAKE_BINARY_DIR}/.clangd
|
||||
@ONLY
|
||||
)
|
||||
# Symlink .clangd to the project root
|
||||
execute_process(
|
||||
COMMAND ln -sf ${CMAKE_BINARY_DIR}/.clangd ${WORKSPACE_ROOT}/.clangd
|
||||
)
|
||||
message(STATUS "${Cyan}Created ${CMAKE_BINARY_DIR}/.clangd${ColorReset}")
|
||||
else()
|
||||
message("${Cyan} ${CMAKE_CURRENT_SOURCE_DIR} is not the root of the project, skipping IDE setup${ColorReset}")
|
||||
endif()
|
||||
else()
|
||||
message("${Yellow}No .clangd.in found, skipping IDE setup${ColorReset}")
|
||||
endif()
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(ordered-map-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(ordered-map
|
||||
GIT_REPOSITORY https://github.com/Tessil/ordered-map.git
|
||||
GIT_TAG v1.0.0
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/ordered-map-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/ordered-map-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,23 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(protobuf-download NONE)
|
||||
|
||||
if("" STREQUAL "${LIBPROTOBUF_VERSION}")
|
||||
set(LIBPROTOBUF_VERSION v33.2)
|
||||
message(FATAL_ERROR "LIBPROTOBUF_VERSION not set, using default: ${D}{LIBPROTOBUF_VERSION}")
|
||||
else()
|
||||
set(LIBPROTOBUF_VERSION ${LIBPROTOBUF_VERSION})
|
||||
message(STATUS "LIBPROTOBUF_VERSION set to: ${D}{LIBPROTOBUF_VERSION}")
|
||||
endif()
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(protobuf
|
||||
GIT_REPOSITORY https://github.com/protocolbuffers/protobuf.git
|
||||
GIT_TAG "${D}{LIBPROTOBUF_VERSION}"
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/protobuf-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/protobuf-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,15 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(rapidjson-download NONE)
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(rapidjson
|
||||
GIT_REPOSITORY https://github.com/tencent/rapidjson.git
|
||||
GIT_TAG d621dc9e9c77f81e5c8a35b8dcc16dcd63351321
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/rapidjson-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/rapidjson-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
cmake_minimum_required(VERSION 3.7.2)
|
||||
|
||||
project(re2-download NONE)
|
||||
|
||||
# *******************************************************************
|
||||
# NOTE: this depedency's CMakeLists.txt is overloaded, make sure to
|
||||
# adjust if you update the tag
|
||||
# *******************************************************************
|
||||
|
||||
include(ExternalProject)
|
||||
ExternalProject_Add(re2
|
||||
GIT_REPOSITORY https://github.com/google/re2.git
|
||||
GIT_TAG 2021-11-01
|
||||
SOURCE_DIR "${CMAKE_BINARY_DIR}/re2-src"
|
||||
BINARY_DIR "${CMAKE_BINARY_DIR}/re2-build"
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
TEST_COMMAND ""
|
||||
)
|
||||
@@ -0,0 +1,176 @@
|
||||
# Copyright 2015 The RE2 Authors. All Rights Reserved.
|
||||
# Use of this source code is governed by a BSD-style
|
||||
# license that can be found in the LICENSE file.
|
||||
|
||||
# Old enough to support Ubuntu Xenial.
|
||||
cmake_minimum_required(VERSION 3.5.1)
|
||||
|
||||
project(RE2 CXX)
|
||||
include(CTest)
|
||||
include(GNUInstallDirs)
|
||||
|
||||
if(NOT CMAKE_CXX_STANDARD)
|
||||
set(CMAKE_CXX_STANDARD 11)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
endif()
|
||||
|
||||
option(BUILD_SHARED_LIBS "build shared libraries" OFF)
|
||||
option(USEPCRE "use PCRE in tests and benchmarks" OFF)
|
||||
|
||||
# CMake seems to have no way to enable/disable testing per subproject,
|
||||
# so we provide an option similar to BUILD_TESTING, but just for RE2.
|
||||
option(RE2_BUILD_TESTING "enable testing for RE2" ON)
|
||||
|
||||
# ABI version
|
||||
# http://tldp.org/HOWTO/Program-Library-HOWTO/shared-libraries.html
|
||||
set(SONAME 9)
|
||||
|
||||
set(EXTRA_TARGET_LINK_LIBRARIES)
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
|
||||
if(MSVC_VERSION LESS 1900)
|
||||
message(FATAL_ERROR "you need Visual Studio 2015 or later")
|
||||
endif()
|
||||
if(BUILD_SHARED_LIBS)
|
||||
# See http://www.kitware.com/blog/home/post/939 for details.
|
||||
set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
|
||||
endif()
|
||||
# CMake defaults to /W3, but some users like /W4 (or /Wall) and /WX,
|
||||
# so we disable various warnings that aren't particularly helpful.
|
||||
add_compile_options(/wd4100 /wd4201 /wd4456 /wd4457 /wd4702 /wd4815)
|
||||
# Without a byte order mark (BOM), Visual Studio assumes that the source
|
||||
# file is encoded using the current user code page, so we specify UTF-8.
|
||||
add_compile_options(/utf-8)
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
add_definitions(-DUNICODE -D_UNICODE -DSTRICT -DNOMINMAX)
|
||||
add_definitions(-D_CRT_SECURE_NO_WARNINGS -D_SCL_SECURE_NO_WARNINGS)
|
||||
# ************************************
|
||||
# TKP NOTE: THIS SECTION IS CUSTOM
|
||||
# ************************************
|
||||
elseif(APPLE OR (${CMAKE_SYSTEM_NAME} MATCHES "Emscripten"))
|
||||
add_definitions(-DTARGET_OS_OSX=1)
|
||||
# assume built-in pthreads on MacOS
|
||||
set(CMAKE_THREAD_LIBS_INIT "-lpthread")
|
||||
set(CMAKE_HAVE_THREADS_LIBRARY 1)
|
||||
set(CMAKE_USE_WIN32_THREADS_INIT 0)
|
||||
set(CMAKE_USE_PTHREADS_INIT 1)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
# ************************************
|
||||
elseif(UNIX)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
find_package(Threads REQUIRED)
|
||||
endif()
|
||||
|
||||
if(USEPCRE)
|
||||
add_definitions(-DUSEPCRE)
|
||||
list(APPEND EXTRA_TARGET_LINK_LIBRARIES pcre)
|
||||
endif()
|
||||
|
||||
set(RE2_SOURCES
|
||||
re2/bitstate.cc
|
||||
re2/compile.cc
|
||||
re2/dfa.cc
|
||||
re2/filtered_re2.cc
|
||||
re2/mimics_pcre.cc
|
||||
re2/nfa.cc
|
||||
re2/onepass.cc
|
||||
re2/parse.cc
|
||||
re2/perl_groups.cc
|
||||
re2/prefilter.cc
|
||||
re2/prefilter_tree.cc
|
||||
re2/prog.cc
|
||||
re2/re2.cc
|
||||
re2/regexp.cc
|
||||
re2/set.cc
|
||||
re2/simplify.cc
|
||||
re2/stringpiece.cc
|
||||
re2/tostring.cc
|
||||
re2/unicode_casefold.cc
|
||||
re2/unicode_groups.cc
|
||||
util/rune.cc
|
||||
util/strutil.cc
|
||||
)
|
||||
|
||||
add_library(re2 ${RE2_SOURCES})
|
||||
target_include_directories(re2 PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}>)
|
||||
set_target_properties(re2 PROPERTIES SOVERSION ${SONAME} VERSION ${SONAME}.0.0)
|
||||
add_library(re2::re2 ALIAS re2)
|
||||
|
||||
if(UNIX)
|
||||
target_link_libraries(re2 PUBLIC Threads::Threads)
|
||||
endif()
|
||||
|
||||
if(RE2_BUILD_TESTING)
|
||||
set(TESTING_SOURCES
|
||||
re2/testing/backtrack.cc
|
||||
re2/testing/dump.cc
|
||||
re2/testing/exhaustive_tester.cc
|
||||
re2/testing/null_walker.cc
|
||||
re2/testing/regexp_generator.cc
|
||||
re2/testing/string_generator.cc
|
||||
re2/testing/tester.cc
|
||||
util/pcre.cc
|
||||
)
|
||||
|
||||
add_library(testing STATIC ${TESTING_SOURCES})
|
||||
target_link_libraries(testing PUBLIC re2)
|
||||
|
||||
set(TEST_TARGETS
|
||||
charclass_test
|
||||
compile_test
|
||||
filtered_re2_test
|
||||
mimics_pcre_test
|
||||
parse_test
|
||||
possible_match_test
|
||||
re2_test
|
||||
re2_arg_test
|
||||
regexp_test
|
||||
required_prefix_test
|
||||
search_test
|
||||
set_test
|
||||
simplify_test
|
||||
string_generator_test
|
||||
|
||||
dfa_test
|
||||
exhaustive1_test
|
||||
exhaustive2_test
|
||||
exhaustive3_test
|
||||
exhaustive_test
|
||||
random_test
|
||||
)
|
||||
|
||||
set(BENCHMARK_TARGETS
|
||||
regexp_benchmark
|
||||
)
|
||||
|
||||
foreach(target ${TEST_TARGETS})
|
||||
add_executable(${target} re2/testing/${target}.cc util/test.cc)
|
||||
target_link_libraries(${target} testing ${EXTRA_TARGET_LINK_LIBRARIES})
|
||||
add_test(NAME ${target} COMMAND ${target})
|
||||
endforeach()
|
||||
|
||||
foreach(target ${BENCHMARK_TARGETS})
|
||||
add_executable(${target} re2/testing/${target}.cc util/benchmark.cc)
|
||||
target_link_libraries(${target} testing ${EXTRA_TARGET_LINK_LIBRARIES})
|
||||
endforeach()
|
||||
endif()
|
||||
|
||||
set(RE2_HEADERS
|
||||
re2/filtered_re2.h
|
||||
re2/re2.h
|
||||
re2/set.h
|
||||
re2/stringpiece.h
|
||||
)
|
||||
|
||||
install(FILES ${RE2_HEADERS}
|
||||
DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/re2)
|
||||
install(TARGETS re2 EXPORT re2Config
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
|
||||
install(EXPORT re2Config
|
||||
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/re2 NAMESPACE re2::)
|
||||
@@ -0,0 +1,24 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
# Minimal toolchain file to ensure that Arrow's bundled dependency builds use
|
||||
# the correct architecture for Mac builds.
|
||||
|
||||
# (Arrow does not as of Oct 2024 pass on CMAKE_OSX_ARCHITECTURES to its
|
||||
# dependencies if it is set, but does pass on a toolchain file)
|
||||
|
||||
set(CMAKE_OSX_ARCHITECTURES "arm64" CACHE STRING "Target architecture for Mac builds")
|
||||
# Arrow uses this to choose a `-march` flag
|
||||
set(CMAKE_SYSTEM_PROCESSOR "arm64")
|
||||
# Prevent cmake from overwriting CMAKE_SYSTEM_PROCESSOR
|
||||
# https://github.com/apache/arrow/issues/44448#issuecomment-2418649378
|
||||
set(CMAKE_SYSTEM_NAME "Darwin")
|
||||
@@ -0,0 +1,24 @@
|
||||
# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
# ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
# ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
# Minimal toolchain file to ensure that Arrow's bundled dependency builds use
|
||||
# the correct architecture for Mac builds.
|
||||
|
||||
# (Arrow does not as of Oct 2024 pass on CMAKE_OSX_ARCHITECTURES to its
|
||||
# dependencies if it is set, but does pass on a toolchain file)
|
||||
|
||||
set(CMAKE_OSX_ARCHITECTURES "x86_64" CACHE STRING "Target architecture for Mac builds")
|
||||
# Arrow uses this to choose a `-march` flag
|
||||
set(CMAKE_SYSTEM_PROCESSOR "x86_64")
|
||||
# Prevent cmake from overwriting CMAKE_SYSTEM_PROCESSOR
|
||||
# https://github.com/apache/arrow/issues/44448#issuecomment-2418649378
|
||||
set(CMAKE_SYSTEM_NAME "Darwin")
|
||||
@@ -0,0 +1,3 @@
|
||||
CompileFlags:
|
||||
@CLANGD_ADD_FLAGS@
|
||||
CompilationDatabase: @CMAKE_BINARY_DIR@
|
||||
@@ -0,0 +1,736 @@
|
||||
cmake_minimum_required(VERSION 3.18.2)
|
||||
project(psp)
|
||||
include(CheckCCompilerFlag)
|
||||
|
||||
set(CMAKE_BUILD_TYPE "Release")
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
|
||||
set(CMAKE_IGNORE_PATH "/opt/homebrew")
|
||||
|
||||
# CMAKE POLICIES
|
||||
# option() should use new cmake behavior wrt variable clobbering
|
||||
cmake_policy(SET CMP0077 NEW)
|
||||
|
||||
# BOOST_ROOT has been removed on Windows VMs in Azure:
|
||||
#
|
||||
# - https://github.com/actions/virtual-environments/issues/687
|
||||
# - https://github.com/actions/virtual-environments/issues/319
|
||||
#
|
||||
# `BOOST_ROOT` must be set in the environment, and policy CMP0074
|
||||
# must be set to `NEW` to allow BOOST_ROOT to be defined by env var
|
||||
cmake_policy(SET CMP0074 NEW)
|
||||
|
||||
# Set CMP0094 to NEW - find the first version that matches constraints,
|
||||
# instead of the latest version installed.
|
||||
cmake_policy(SET CMP0094 NEW)
|
||||
|
||||
if(NOT DEFINED PSP_CMAKE_MODULE_PATH)
|
||||
set(PSP_CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake")
|
||||
endif()
|
||||
|
||||
set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH})
|
||||
set(MSVC_RUNTIME_LIBRARY MultiThreaded)
|
||||
|
||||
option(PSP_WASM64 "Enable WASM Memory64 support" OFF)
|
||||
option(PSP_WASM_OPFS "Enable on-disk (OPFS) tables in the browser via the custom `psp_opfs_*` bridge in `perspective-server.poly.ts` (NO_FILESYSTEM + JSPI; no WasmFS, no SharedArrayBuffer). Backwards compatible: the module loads and runs in-memory tables on non-JSPI browsers; on-disk activates only in a JSPI-capable Worker." ON)
|
||||
|
||||
if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
|
||||
set(WIN32 ON)
|
||||
set(MACOS OFF)
|
||||
set(LINUX OFF)
|
||||
elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
|
||||
set(WIN32 OFF)
|
||||
set(MACOS ON)
|
||||
set(LINUX OFF)
|
||||
else()
|
||||
set(WIN32 OFF)
|
||||
set(MACOS OFF)
|
||||
set(LINUX ON)
|
||||
endif()
|
||||
|
||||
if (WIN32)
|
||||
add_compile_options("/EHsc")
|
||||
set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
|
||||
endif()
|
||||
|
||||
|
||||
if(WIN32)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /MT /c /bigobj")
|
||||
else()
|
||||
# set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS}")
|
||||
endif()
|
||||
|
||||
# # Helper function
|
||||
function(string_starts_with str search)
|
||||
string(FIND "${str}" "${search}" out)
|
||||
|
||||
if("${out}" EQUAL 0)
|
||||
return(true)
|
||||
endif()
|
||||
return(false)
|
||||
endfunction()
|
||||
|
||||
set(BUILD_MESSAGE "")
|
||||
|
||||
function(psp_build_message message)
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${message}")
|
||||
endfunction()
|
||||
|
||||
# ######################
|
||||
# BUILD CONFIGURATION #
|
||||
# ######################
|
||||
find_package(Color)
|
||||
find_package(InstallDependency)
|
||||
|
||||
# OPTIONS
|
||||
option(CMAKE_BUILD_TYPE "Release/Debug build" RELEASE)
|
||||
option(PSP_WASM_BUILD "Build the WebAssembly Project" ON)
|
||||
option(PSP_CPP_BUILD "Build the C++ Project" OFF)
|
||||
option(PSP_PYTHON_BUILD "Build the Python Bindings" OFF)
|
||||
option(PSP_CPP_BUILD_STRICT "Build the C++ with strict warnings" OFF)
|
||||
option(PSP_SANITIZE "Build with sanitizers" OFF)
|
||||
option(PSP_HEAP_INSTRUMENTS "Build with heap inspection tooling" OFF)
|
||||
|
||||
if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
|
||||
set(PSP_WASM_BUILD ON)
|
||||
set(PSP_CPP_BUILD OFF)
|
||||
else()
|
||||
set(PSP_WASM_BUILD OFF)
|
||||
set(PSP_CPP_BUILD ON)
|
||||
endif()
|
||||
|
||||
if(PSP_WASM_BUILD AND PSP_CPP_BUILD)
|
||||
message(FATAL_ERROR "${Red}CPP and Emscripten builds must be done separately${ColorReset}")
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{PSP_DEBUG})
|
||||
set(CMAKE_BUILD_TYPE DEBUG)
|
||||
else()
|
||||
if(NOT DEFINED CMAKE_BUILD_TYPE)
|
||||
set(CMAKE_BUILD_TYPE RELEASE)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{PSP_HEAP_INSTRUMENTS})
|
||||
set(PSP_HEAP_INSTRUMENTS ON)
|
||||
else()
|
||||
set(PSP_HEAP_INSTRUMENTS OFF)
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{PSP_MANYLINUX})
|
||||
set(MANYLINUX ON)
|
||||
else()
|
||||
set(MANYLINUX OFF)
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{PSP_USE_CCACHE})
|
||||
set(CMAKE_C_COMPILE_LAUNCHER ccache)
|
||||
set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED PSP_CPP_BUILD)
|
||||
set(PSP_CPP_BUILD ON)
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED PSP_PYTHON_BUILD)
|
||||
set(PSP_PYTHON_BUILD OFF)
|
||||
elseif(PSP_PYTHON_BUILD)
|
||||
if(NOT DEFINED PSP_PYTHON_VERSION)
|
||||
set(PSP_PYTHON_VERSION 3.10)
|
||||
endif()
|
||||
if(PSP_WASM_BUILD)
|
||||
set(PSP_PYODIDE 1)
|
||||
else()
|
||||
set(PSP_PYODIDE 0)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED PSP_CPP_BUILD_STRICT)
|
||||
set(PSP_CPP_BUILD_STRICT OFF)
|
||||
endif()
|
||||
|
||||
if(PSP_WASM_BUILD)
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}")
|
||||
else()
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping WASM binding${ColorReset}")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED PSP_CPP_SRC)
|
||||
set(PSP_CPP_SRC "${CMAKE_CURRENT_SOURCE_DIR}")
|
||||
endif()
|
||||
|
||||
if(PSP_CPP_BUILD)
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ binding${ColorReset}")
|
||||
else()
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping C++ binding${ColorReset}")
|
||||
endif()
|
||||
|
||||
if(PSP_PYTHON_BUILD)
|
||||
if(NOT DEFINED PSP_PYTHON_SRC)
|
||||
set(PSP_PYTHON_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../python/perspective/perspective")
|
||||
endif()
|
||||
|
||||
# set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Python ${Red}${PSP_PYTHON_VERSION}${Cyan} binding${ColorReset}")
|
||||
else()
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping Python binding${ColorReset}")
|
||||
endif()
|
||||
|
||||
if(PSP_CPP_BUILD AND NOT PSP_CPP_BUILD_STRICT)
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Building C++ without strict warnings${ColorReset}")
|
||||
else()
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ with strict warnings${ColorReset}")
|
||||
endif()
|
||||
|
||||
string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER)
|
||||
|
||||
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Red}Building DEBUG${ColorReset}")
|
||||
add_definitions(-DPSP_DEBUG)
|
||||
add_definitions(-DPSP_STORAGE_VERIFY)
|
||||
add_definitions(-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST)
|
||||
add_definitions(-D_GLIBCXX_ASSERTIONS)
|
||||
else()
|
||||
set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building RELEASE${ColorReset}")
|
||||
endif()
|
||||
|
||||
if(PSP_PYTHON_BUILD AND MACOS)
|
||||
# fix for threads on osx
|
||||
# assume built-in pthreads on MacOS
|
||||
set(CMAKE_THREAD_LIBS_INIT "-lpthread")
|
||||
set(CMAKE_HAVE_THREADS_LIBRARY 1)
|
||||
set(CMAKE_USE_WIN32_THREADS_INIT 0)
|
||||
set(CMAKE_USE_PTHREADS_INIT 1)
|
||||
set(THREADS_PREFER_PTHREAD_FLAG ON)
|
||||
|
||||
# don't link against build python
|
||||
# https://blog.tim-smith.us/2015/09/python-extension-modules-os-x/
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
|
||||
endif()
|
||||
|
||||
if(EMSCRIPTEN)
|
||||
set(CMAKE_HAVE_LIBC_PTHREAD TRUE CACHE BOOL "psp: satisfy FindThreads on WASM without enabling pthreads" FORCE)
|
||||
endif()
|
||||
|
||||
# ######################
|
||||
set(psp_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/src/include")
|
||||
|
||||
if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD)
|
||||
set(PSP_WASM_EXCEPTIONS ON)
|
||||
endif()
|
||||
|
||||
if(PSP_PYODIDE)
|
||||
set(PSP_WASM_EXCEPTIONS ON)
|
||||
endif()
|
||||
|
||||
set(DEBUG_LEVEL "0")
|
||||
if(PSP_HEAP_INSTRUMENTS)
|
||||
set(DEBUG_LEVEL "3")
|
||||
endif()
|
||||
|
||||
if(PSP_WASM_BUILD)
|
||||
####################
|
||||
# EMSCRIPTEN BUILD #
|
||||
####################
|
||||
set(CMAKE_EXECUTABLE_SUFFIX ".js")
|
||||
list(APPEND CMAKE_PREFIX_PATH /usr/local)
|
||||
|
||||
set(EXTENDED_FLAGS " \
|
||||
-Wall \
|
||||
-fcolor-diagnostics \
|
||||
")
|
||||
|
||||
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
|
||||
# Pyodide DEBUG block
|
||||
set(OPT_FLAGS " \
|
||||
-O0 \
|
||||
-g3 \
|
||||
-Wcast-align \
|
||||
-Wover-aligned \
|
||||
--emit-tsd=perspective-server.d.ts \
|
||||
")
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
|
||||
endif()
|
||||
if(PSP_SANITIZE)
|
||||
set(OPT_FLAGS "${OPT_FLAGS} \
|
||||
-fsanitize=undefined \
|
||||
-fsanitize=address \
|
||||
")
|
||||
endif()
|
||||
else()
|
||||
set(OPT_FLAGS " \
|
||||
-O3 -g${DEBUG_LEVEL} \
|
||||
-mbulk-memory \
|
||||
-msimd128 \
|
||||
-mrelaxed-simd \
|
||||
-flto \
|
||||
--emit-tsd=perspective-server.d.ts \
|
||||
")
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
|
||||
endif()
|
||||
endif()
|
||||
elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
|
||||
if(WIN32)
|
||||
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
|
||||
set(OPT_FLAGS " \
|
||||
/DEBUG \
|
||||
/Z7 \
|
||||
/Zi
|
||||
")
|
||||
else()
|
||||
set(OPT_FLAGS " \
|
||||
/NDEBUG \
|
||||
/O2 \
|
||||
")
|
||||
endif()
|
||||
else()
|
||||
if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
|
||||
if (PSP_PYODIDE)
|
||||
set(OPT_FLAGS " \
|
||||
-O0 \
|
||||
-g3 \
|
||||
-gsource-map \
|
||||
--profiling \
|
||||
-Wcast-align \
|
||||
-Wover-aligned \
|
||||
")
|
||||
else ()
|
||||
set(OPT_FLAGS " \
|
||||
-O0 \
|
||||
-fexceptions \
|
||||
-g3 \
|
||||
")
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
|
||||
endif()
|
||||
endif ()
|
||||
else()
|
||||
set(OPT_FLAGS " \
|
||||
-O3 \
|
||||
-fexceptions \
|
||||
-g1 \
|
||||
-fopenmp-simd \
|
||||
")
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(ASYNC_MODE_FLAGS "")
|
||||
|
||||
if(WIN32)
|
||||
foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996)
|
||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}")
|
||||
endforeach(warning)
|
||||
else()
|
||||
endif()
|
||||
|
||||
if(PSP_PYTHON_BUILD)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
|
||||
-O3 \
|
||||
-g${DEBUG_LEVEL} \
|
||||
")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
|
||||
-fwasm-exceptions \
|
||||
-O3 \
|
||||
-g${DEBUG_LEVEL} \
|
||||
")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
|
||||
-O3 \
|
||||
")
|
||||
endif()
|
||||
|
||||
if (PSP_WASM64)
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
|
||||
-s MEMORY64=1 \
|
||||
")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
|
||||
-s MEMORY64=1 \
|
||||
")
|
||||
endif()
|
||||
|
||||
set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
|
||||
|
||||
if(PSP_PYODIDE)
|
||||
set(RELOCATABLE_FLAGS " -sRELOCATABLE=1 -sSIDE_MODULE=2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm \ ")
|
||||
string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS} -sWASM_BIGINT=1 ")
|
||||
string(APPEND CMAKE_C_FLAGS "${RELOCATABLE_FLAGS}")
|
||||
string(APPEND CMAKE_CXX_FLAGS "${RELOCATABLE_FLAGS}")
|
||||
string(APPEND CMAKE_CXX_FLAGS " -fvisibility=hidden")
|
||||
|
||||
# Emscripten exceptions
|
||||
string(APPEND CMAKE_CXX_FLAGS " -fexceptions")
|
||||
endif()
|
||||
|
||||
# Build (read: download and extract) header-only dependencies from external sources
|
||||
set(all_deps_INCLUDE_DIRS "")
|
||||
psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in")
|
||||
psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in")
|
||||
psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in")
|
||||
psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in")
|
||||
list(APPEND all_deps_INCLUDE_DIRS
|
||||
${date_INCLUDE_DIRS}
|
||||
${hopscotch_INCLUDE_DIRS}
|
||||
${ordered-map_INCLUDE_DIRS}
|
||||
${rapidjson_INCLUDE_DIRS})
|
||||
|
||||
psp_build_dep("Boost" "${PSP_CMAKE_MODULE_PATH}/Boost.txt.in")
|
||||
list(APPEND all_deps_INCLUDE_DIRS ${Boost_INCLUDE_DIRS})
|
||||
|
||||
# Build dependencies as static libraries with add_subdirectory()
|
||||
# Note that values set above for CMAKE_C_FLAGS, CMAKE_CXX_FLAGS, etc. will
|
||||
# apply to dependency cmake builds
|
||||
|
||||
# Arrow builds its own dependencies
|
||||
psp_build_message("${Cyan}Building Apache Arrow${ColorReset}")
|
||||
psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in")
|
||||
list(APPEND all_deps_INCLUDE_DIRS
|
||||
${arrow_INCLUDE_DIRS})
|
||||
|
||||
# Build re2 as our regex library
|
||||
# this is a workaround for some re2-specific weirdness
|
||||
add_definitions(-DTARGET_OS_OSX=1)
|
||||
psp_build_dep("re2" "${PSP_CMAKE_MODULE_PATH}/re2.txt.in")
|
||||
list(APPEND all_deps_INCLUDE_DIRS
|
||||
${re2_INCLUDE_DIRS})
|
||||
|
||||
# Build exprtk for expression parsing
|
||||
psp_build_dep("exprtk" "${PSP_CMAKE_MODULE_PATH}/exprtk.txt.in")
|
||||
list(APPEND all_deps_INCLUDE_DIRS
|
||||
${exprtk_INCLUDE_DIRS})
|
||||
|
||||
# Protobuf setup
|
||||
set(protobuf_BUILD_TESTS OFF FORCE)
|
||||
add_subdirectory(${PSP_CMAKE_MODULE_PATH}/../cpp/protos "${CMAKE_BINARY_DIR}/protos-build")
|
||||
|
||||
# ####################
|
||||
set(CMAKE_C_FLAGS " \
|
||||
${CMAKE_C_FLAGS} \
|
||||
${CMAKE_C_FLAGS_RELEASE} \
|
||||
${EXTENDED_FLAGS} \
|
||||
${OPT_FLAGS} \
|
||||
")
|
||||
|
||||
set(CMAKE_CXX_FLAGS " \
|
||||
${CMAKE_CXX_FLAGS} \
|
||||
${CMAKE_CXX_FLAGS_RELEASE} \
|
||||
${EXTENDED_FLAGS} \
|
||||
${OPT_FLAGS} \
|
||||
")
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c++1y")
|
||||
endif()
|
||||
|
||||
set(SOURCE_FILES
|
||||
${PSP_CPP_SRC}/src/cpp/aggregate.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/aggspec.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/arg_sort.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/arrow_loader.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/arrow_writer.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/base.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/base_impl_linux.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/base_impl_osx.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/base_impl_wasm.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/base_impl_win.cpp
|
||||
# ${PSP_CPP_SRC}/src/cpp/binding.cpp
|
||||
|
||||
# ${PSP_CPP_SRC}/src/cpp/build_filter.cpp
|
||||
# ${PSP_CPP_SRC}/src/cpp/calc_agg_dtype.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/column.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/comparators.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/compat.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/compat_impl_linux.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/compat_impl_osx.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/compat_impl_wasm.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/compat_impl_win.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/computed_expression.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/computed_function.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/config.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_base.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_grouped_pkey.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_handle.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_one.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_two.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_zero.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/context_unit.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/data.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/data_slice.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/data_table.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/date.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/dense_nodes.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/dense_tree_context.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/dense_tree.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/dependency.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/expression_tables.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/expression_vocab.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/extract_aggregate.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/filter.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/flat_traversal.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/get_data_extents.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/gnode.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/gnode_state.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/mask.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/multi_sort.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/none.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/path.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/pivot.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/pool.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/port.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/process_state.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/pyutils.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/raii.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/raii_impl_linux.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/raii_impl_osx.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/raii_impl_win.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/range.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/regex.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/rlookup.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/scalar.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/schema_column.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/schema.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/slice.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/sort_specification.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/sparse_tree.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/sparse_tree_node.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/residency.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/step_delta.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/storage.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/storage_impl_linux.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/storage_impl_osx.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/storage_impl_wasm.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/storage_impl_win.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/sym_table.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/table.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/time.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/traversal.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/traversal_nodes.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/tree_context_common.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/utils.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/update_task.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/view.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/view_config.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/vocab.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/arrow_csv.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/join_engine.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/server.cpp
|
||||
${PSP_CPP_SRC}/src/cpp/binding_api.cpp
|
||||
)
|
||||
|
||||
if(PSP_HEAP_INSTRUMENTS)
|
||||
list(APPEND SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/heap_instruments.cpp)
|
||||
add_compile_definitions(HEAP_INSTRUMENTS=1)
|
||||
endif()
|
||||
|
||||
set(PYTHON_SOURCE_FILES ${SOURCE_FILES})
|
||||
set(WASM_SOURCE_FILES ${SOURCE_FILES})
|
||||
|
||||
message("${BUILD_MESSAGE}\n")
|
||||
|
||||
if(WIN32)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /MT /c /bigobj")
|
||||
else()
|
||||
# set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS}")
|
||||
endif()
|
||||
|
||||
set(PSP_EXPORTED_FUNCTIONS
|
||||
_psp_poll
|
||||
_psp_new_server
|
||||
_psp_free
|
||||
_psp_alloc
|
||||
_psp_handle_request
|
||||
_psp_new_session
|
||||
_psp_close_session
|
||||
_psp_delete_server
|
||||
_psp_is_memory64
|
||||
_psp_num_cpus
|
||||
_psp_set_num_cpus
|
||||
_psp_residency_prepare
|
||||
_psp_residency_victim_fname
|
||||
_psp_residency_commit
|
||||
)
|
||||
|
||||
if(PSP_HEAP_INSTRUMENTS)
|
||||
list(APPEND PSP_EXPORTED_FUNCTIONS
|
||||
_psp_print_used_memory
|
||||
_psp_dump_stack_traces
|
||||
_psp_clear_stack_traces
|
||||
)
|
||||
endif()
|
||||
|
||||
string(JOIN "," PSP_EXPORTED_FUNCTIONS_JOINED ${PSP_EXPORTED_FUNCTIONS})
|
||||
|
||||
# Common flags for WASM/JS build and Pyodide
|
||||
if(PSP_PYODIDE)
|
||||
set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
|
||||
--no-entry \
|
||||
-s EXPORTED_FUNCTIONS=${PSP_EXPORTED_FUNCTIONS_JOINED} \
|
||||
-s SIDE_MODULE=2 \
|
||||
")
|
||||
else()
|
||||
# -s MEMORY_GROWTH_GEOMETRIC_STEP=1.0 \
|
||||
# -s MEMORY_GROWTH_GEOMETRIC_CAP=536870912 \
|
||||
|
||||
set(PSP_FILESYSTEM_FLAGS "-s NO_FILESYSTEM=1 ")
|
||||
set(PSP_CLOSURE_FLAGS "--closure=1 ")
|
||||
|
||||
# On-disk tables in the browser are backed by OPFS via emscripten's WasmFS.
|
||||
if(PSP_WASM_OPFS)
|
||||
add_compile_definitions(PSP_WASM_OPFS_PERSIST=1)
|
||||
endif()
|
||||
|
||||
set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
|
||||
--no-entry \
|
||||
${PSP_CLOSURE_FLAGS} \
|
||||
${PSP_FILESYSTEM_FLAGS} \
|
||||
-s ALLOW_MEMORY_GROWTH=1 \
|
||||
-s MODULARIZE=1 \
|
||||
-s WASM_BIGINT=1 \
|
||||
-s INCOMING_MODULE_JS_API=locateFile,psp_heap_size,psp_stack_trace,HEAPU8,HEAPU32,instantiateWasm \
|
||||
-s TEXTDECODER=2 \
|
||||
-s STANDALONE_WASM=1 \
|
||||
-s DYNAMIC_EXECUTION=0 \
|
||||
-s POLYFILL=0 \
|
||||
-s EXPORT_NAME=\"load_perspective\" \
|
||||
-s ERROR_ON_UNDEFINED_SYMBOLS=0 \
|
||||
-s NODEJS_CATCH_EXIT=0 \
|
||||
-s NODEJS_CATCH_REJECTION=0 \
|
||||
-s USE_ES6_IMPORT_META=1 \
|
||||
-s EXPORT_ES6=1 \
|
||||
-s EXPORTED_RUNTIME_METHODS=HEAPU8 \
|
||||
-s EXPORTED_FUNCTIONS=${PSP_EXPORTED_FUNCTIONS_JOINED} \
|
||||
")
|
||||
|
||||
if(PSP_WASM64)
|
||||
set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
|
||||
-s MAXIMUM_MEMORY=16gb \
|
||||
")
|
||||
else()
|
||||
set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
|
||||
-s MAXIMUM_MEMORY=4gb \
|
||||
")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (PSP_WASM_EXCEPTIONS)
|
||||
set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} -s EXCEPTION_STACK_TRACES=1 ")
|
||||
endif()
|
||||
|
||||
if(PSP_SANITIZE)
|
||||
set(PSP_SANITIZE_FLAGS
|
||||
-sINITIAL_MEMORY=640mb
|
||||
-sTOTAL_MEMORY=640mb
|
||||
-sALLOW_MEMORY_GROWTH=1
|
||||
)
|
||||
else()
|
||||
set(PSP_SANITIZE_FLAGS)
|
||||
endif()
|
||||
|
||||
if(PSP_WASM_BUILD AND NOT PSP_PYTHON_BUILD)
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
|
||||
|
||||
add_library(psp ${WASM_SOURCE_FILES})
|
||||
target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
|
||||
target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
|
||||
target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1)
|
||||
set_target_properties(psp PROPERTIES COMPILE_FLAGS "")
|
||||
target_link_libraries(psp PRIVATE arrow_static re2 protos)
|
||||
|
||||
add_executable(perspective_esm src/cpp/binding_api.cpp)
|
||||
message(STATUS "all_deps_INCLUDE_DIRS ${all_deps_INCLUDE_DIRS}")
|
||||
target_include_directories(perspective_esm PRIVATE ${psp_INCLUDE_DIRS})
|
||||
target_include_directories(perspective_esm SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
|
||||
target_link_libraries(perspective_esm psp protos)
|
||||
target_compile_definitions(perspective_esm PRIVATE PSP_ENABLE_WASM=1)
|
||||
target_link_options(perspective_esm PUBLIC -sENVIRONMENT=web ${PSP_SANITIZE_FLAGS})
|
||||
set_target_properties(perspective_esm PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./web/")
|
||||
set_target_properties(perspective_esm PROPERTIES OUTPUT_NAME "perspective-server")
|
||||
elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
|
||||
if(NOT WIN32)
|
||||
set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
|
||||
|
||||
# Look for the binary using @loader_path (relative to binary location)
|
||||
set(CMAKE_MACOSX_RPATH TRUE)
|
||||
set(CMAKE_SKIP_BUILD_RPATH FALSE)
|
||||
set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
|
||||
set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
|
||||
set(CMAKE_INSTALL_NAME_DIR "@rpath/")
|
||||
|
||||
# module_origin_path is the location of the binary
|
||||
if(MACOS)
|
||||
set(module_origin_path "@loader_path/")
|
||||
else()
|
||||
set(module_origin_path "\$ORIGIN")
|
||||
endif()
|
||||
else()
|
||||
set(CMAKE_SHARED_LIBRARY_PREFIX lib)
|
||||
endif()
|
||||
|
||||
if(PSP_PYTHON_BUILD)
|
||||
# #######################
|
||||
# Python extra targets #
|
||||
# #######################
|
||||
if(PSP_WASM_BUILD)
|
||||
# Pyodide
|
||||
set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
|
||||
add_library(psp STATIC ${PYTHON_SOURCE_FILES})
|
||||
target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
|
||||
target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
|
||||
target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1)
|
||||
else()
|
||||
# Cpython
|
||||
add_library(psp STATIC ${PYTHON_SOURCE_FILES})
|
||||
target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
|
||||
target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
|
||||
target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
|
||||
endif()
|
||||
|
||||
if(MACOS OR NOT MANYLINUX)
|
||||
set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
|
||||
|
||||
target_compile_options(psp PRIVATE -fvisibility=hidden)
|
||||
elseif(MANYLINUX)
|
||||
# intentionally blank
|
||||
else()
|
||||
target_compile_options(psp PRIVATE -fvisibility=hidden)
|
||||
endif()
|
||||
|
||||
# Linking against arrow_static also links against its bundled dependencies
|
||||
target_link_libraries(psp PRIVATE arrow_static re2 protos)
|
||||
else()
|
||||
add_library(psp STATIC ${WASM_SOURCE_FILES})
|
||||
target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
|
||||
target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
|
||||
target_compile_options(psp PRIVATE -fvisibility=hidden)
|
||||
target_link_libraries(psp PRIVATE arrow_static re2 protos)
|
||||
endif()
|
||||
|
||||
if(PSP_CPP_BUILD_STRICT AND NOT WIN32)
|
||||
target_compile_options(psp PRIVATE -Wall -Werror)
|
||||
target_compile_options(psp PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
|
||||
if(PSP_PYTHON_BUILD)
|
||||
target_compile_options(psppy PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
|
||||
endif()
|
||||
elseif(WIN32)
|
||||
target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1)
|
||||
target_compile_definitions(psp PRIVATE WIN32=1)
|
||||
target_compile_definitions(psp PRIVATE _WIN32=1)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# TODO this needs to be omitted when built by `cmake-rs`, because
|
||||
#`cargo publish --dry-run` complains when this generate `.clangd` in the Rust
|
||||
# root dir instead of teh `target` dir.
|
||||
if(NOT DEFINED ENV{PSP_DISABLE_CLANGD})
|
||||
include(SetupClangd)
|
||||
endif()
|
||||
@@ -0,0 +1,15 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
// this line/hack allows us to bypass the emscripten environment
|
||||
// so we can run the same JS-file on both web and node.
|
||||
globalThis.window = {};
|
||||
@@ -0,0 +1,53 @@
|
||||
diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
|
||||
index c8457eae8..aa2b41cc9 100644
|
||||
--- a/cpp/src/arrow/util/decimal.cc
|
||||
+++ b/cpp/src/arrow/util/decimal.cc
|
||||
@@ -111,7 +111,11 @@ struct DecimalRealConversion : public BaseDecimalRealConversion {
|
||||
|
||||
// 2. Losslessly convert `real` to `mant * 2**k`
|
||||
int binary_exp = 0;
|
||||
+#ifdef __EMSCRIPTEN__
|
||||
+ const Real real_mant = std::frexpf(real, &binary_exp);
|
||||
+#else
|
||||
const Real real_mant = std::frexp(real, &binary_exp);
|
||||
+#endif
|
||||
// `real_mant` is within 0.5 and 1 and has M bits of precision.
|
||||
// Multiply it by 2^M to get an exact integer.
|
||||
const uint64_t mant = static_cast<uint64_t>(std::ldexp(real_mant, kMantissaBits));
|
||||
diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h
|
||||
index 609906052..1e3dfae7c 100644
|
||||
--- a/cpp/src/arrow/util/value_parsing.h
|
||||
+++ b/cpp/src/arrow/util/value_parsing.h
|
||||
@@ -804,7 +804,7 @@ static inline bool ParseTimestampStrptime(const char* buf, size_t length,
|
||||
std::string clean_copy(buf, length);
|
||||
struct tm result;
|
||||
memset(&result, 0, sizeof(struct tm));
|
||||
-#ifdef _WIN32
|
||||
+#if defined(_WIN32) || defined(__EMSCRIPTEN__)
|
||||
char* ret = arrow_strptime(clean_copy.c_str(), format, &result);
|
||||
#else
|
||||
char* ret = strptime(clean_copy.c_str(), format, &result);
|
||||
diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.h b/cpp/src/arrow/vendored/xxhash/xxhash.h
|
||||
index a18e8c762..235590b19 100644
|
||||
--- a/cpp/src/arrow/vendored/xxhash/xxhash.h
|
||||
+++ b/cpp/src/arrow/vendored/xxhash/xxhash.h
|
||||
@@ -169,6 +169,11 @@
|
||||
* xxHash prototypes and implementation
|
||||
*/
|
||||
|
||||
+
|
||||
+#ifdef __EMSCRIPTEN__
|
||||
+#include <emscripten.h>
|
||||
+#endif
|
||||
+
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
@@ -3422,6 +3427,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can
|
||||
# endif
|
||||
#endif
|
||||
|
||||
+
|
||||
#if defined(__GNUC__) || defined(__clang__)
|
||||
# if defined(__ARM_FEATURE_SVE)
|
||||
# include <arm_sve.h>
|
||||
@@ -0,0 +1,59 @@
|
||||
From f2b12a311d1a14ab5d9f67428201e1389748a9c1 Mon Sep 17 00:00:00 2001
|
||||
From: Tom Jakubowski <tom@prospective.dev>
|
||||
Date: Fri, 11 Oct 2024 12:23:29 -0700
|
||||
Subject: [PATCH] GH-44384: [C++] Use CMAKE_LIBTOOL on macOS
|
||||
|
||||
When a builder sets `CMAKE_LIBTOOL`, use that as the program to bundle
|
||||
dependencies. This matches the behavior of the Windows build.
|
||||
|
||||
Also make a nitpicky minor update to the error message when a non-Apple
|
||||
libtool is detected.
|
||||
---
|
||||
cpp/cmake_modules/BuildUtils.cmake | 30 +++++++++++++++++-------------
|
||||
1 file changed, 17 insertions(+), 13 deletions(-)
|
||||
|
||||
diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
|
||||
index 692efa78376f4..eb94563afcf77 100644
|
||||
--- a/cpp/cmake_modules/BuildUtils.cmake
|
||||
+++ b/cpp/cmake_modules/BuildUtils.cmake
|
||||
@@ -97,23 +97,27 @@ function(arrow_create_merged_static_lib output_target)
|
||||
endforeach()
|
||||
|
||||
if(APPLE)
|
||||
- # The apple-distributed libtool is what we want for bundling, but there is
|
||||
- # a GNU libtool that has a namecollision (and happens to be bundled with R, too).
|
||||
- # We are not compatible with GNU libtool, so we need to avoid it.
|
||||
-
|
||||
- # check in the obvious places first to find Apple's libtool
|
||||
- # HINTS is used before system paths and before PATHS, so we use that
|
||||
- # even though hard coded paths should go in PATHS
|
||||
- # TODO: use a VALIDATOR when we require cmake >= 3.25
|
||||
- find_program(LIBTOOL_MACOS libtool HINTS /usr/bin
|
||||
- /Library/Developer/CommandLineTools/usr/bin)
|
||||
-
|
||||
- # confirm that the libtool we found is not GNU libtool
|
||||
+ if(CMAKE_LIBTOOL)
|
||||
+ set(LIBTOOL_MACOS ${CMAKE_LIBTOOL})
|
||||
+ else()
|
||||
+ # The apple-distributed libtool is what we want for bundling, but there is
|
||||
+ # a GNU libtool that has a namecollision (and happens to be bundled with R, too).
|
||||
+ # We are not compatible with GNU libtool, so we need to avoid it.
|
||||
+
|
||||
+ # check in the obvious places first to find Apple's libtool
|
||||
+ # HINTS is used before system paths and before PATHS, so we use that
|
||||
+ # even though hard coded paths should go in PATHS
|
||||
+ # TODO: use a VALIDATOR when we require cmake >= 3.25
|
||||
+ find_program(LIBTOOL_MACOS libtool
|
||||
+ HINTS /usr/bin /Library/Developer/CommandLineTools/usr/bin)
|
||||
+ endif()
|
||||
+
|
||||
+ # confirm that the libtool we found is Apple's libtool
|
||||
execute_process(COMMAND ${LIBTOOL_MACOS} -V
|
||||
OUTPUT_VARIABLE LIBTOOL_V_OUTPUT
|
||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||
if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*")
|
||||
- message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}"
|
||||
+ message(FATAL_ERROR "libtool found appears not to be Apple's libtool: ${LIBTOOL_MACOS}"
|
||||
)
|
||||
endif()
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/aggregate.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_aggregate::t_aggregate(
|
||||
const t_dtree& tree,
|
||||
t_aggtype aggtype,
|
||||
std::vector<std::shared_ptr<const t_column>> icolumns,
|
||||
std::shared_ptr<t_column> ocolumn
|
||||
) :
|
||||
m_tree(tree),
|
||||
m_aggtype(aggtype),
|
||||
m_icolumns(std::move(icolumns)),
|
||||
m_ocolumn(std::move(ocolumn)) {}
|
||||
|
||||
void
|
||||
t_aggregate::init() {
|
||||
switch (m_aggtype) {
|
||||
case AGGTYPE_SUM:
|
||||
case AGGTYPE_PCT_SUM_PARENT:
|
||||
case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::int64_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::int32_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::int16_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<
|
||||
t_aggimpl_sum<std::int8_t, std::int64_t, std::int64_t>>(
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::uint64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::uint32_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::uint16_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<t_aggimpl_sum<double, double, double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<t_aggimpl_sum<float, double, double>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_sum<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case AGGTYPE_MUL: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::int64_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::int32_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::int16_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<
|
||||
t_aggimpl_mul<std::int8_t, std::int64_t, std::int64_t>>(
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::uint64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::uint32_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::uint16_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<t_aggimpl_mul<double, double, double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<t_aggimpl_mul<float, double, double>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_mul<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case AGGTYPE_COUNT: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_STR: {
|
||||
build_aggregate<
|
||||
t_aggimpl_count<t_uindex, std::uint64_t, std::uint64_t>>(
|
||||
);
|
||||
} break;
|
||||
case DTYPE_TIME:
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::int64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::int32_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::int16_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::int8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::uint64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_DATE:
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::uint32_t,
|
||||
std::uint32_t,
|
||||
std::uint32_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::uint16_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<
|
||||
t_aggimpl_count<double, std::uint64_t, std::uint64_t>>(
|
||||
);
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<
|
||||
t_aggimpl_count<float, std::uint64_t, std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_count<
|
||||
std::uint8_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case AGGTYPE_MEAN: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::int64_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::int32_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::int16_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::int8_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::uint64_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::uint32_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::uint16_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::uint8_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
double,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
float,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_mean<
|
||||
std::uint8_t,
|
||||
std::pair<double, double>,
|
||||
double>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case AGGTYPE_HIGH_WATER_MARK: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_TIME:
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::int64_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::int32_t,
|
||||
std::int32_t,
|
||||
std::int32_t>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::int16_t,
|
||||
std::int16_t,
|
||||
std::int16_t>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<
|
||||
t_aggimpl_hwm<std::int8_t, std::int8_t, std::int8_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::uint64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_DATE:
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::uint32_t,
|
||||
std::uint32_t,
|
||||
std::uint32_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::uint16_t,
|
||||
std::uint16_t,
|
||||
std::uint16_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::uint8_t,
|
||||
std::uint8_t,
|
||||
std::uint8_t>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<t_aggimpl_hwm<double, double, double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<t_aggimpl_hwm<float, float, float>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_hwm<
|
||||
std::uint8_t,
|
||||
std::uint8_t,
|
||||
std::uint8_t>>();
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
build_aggregate<
|
||||
t_aggimpl_hwm<const char*, const char*, const char*>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
case AGGTYPE_LOW_WATER_MARK: {
|
||||
switch (m_icolumns[0]->get_dtype()) {
|
||||
case DTYPE_TIME:
|
||||
case DTYPE_INT64: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::int64_t,
|
||||
std::int64_t,
|
||||
std::int64_t>>();
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::int32_t,
|
||||
std::int32_t,
|
||||
std::int32_t>>();
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::int16_t,
|
||||
std::int16_t,
|
||||
std::int16_t>>();
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
build_aggregate<
|
||||
t_aggimpl_lwm<std::int8_t, std::int8_t, std::int8_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::uint64_t,
|
||||
std::uint64_t,
|
||||
std::uint64_t>>();
|
||||
} break;
|
||||
case DTYPE_DATE:
|
||||
case DTYPE_UINT32: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::uint32_t,
|
||||
std::uint32_t,
|
||||
std::uint32_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::uint16_t,
|
||||
std::uint16_t,
|
||||
std::uint16_t>>();
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::uint8_t,
|
||||
std::uint8_t,
|
||||
std::uint8_t>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
build_aggregate<t_aggimpl_lwm<double, double, double>>();
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
build_aggregate<t_aggimpl_lwm<float, float, float>>();
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
build_aggregate<t_aggimpl_lwm<
|
||||
std::uint8_t,
|
||||
std::uint8_t,
|
||||
std::uint8_t>>();
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
build_aggregate<
|
||||
t_aggimpl_lwm<const char*, const char*, const char*>>();
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
|
||||
}
|
||||
}
|
||||
} break;
|
||||
default: {
|
||||
// Other aggregates will be filled in later
|
||||
}
|
||||
}
|
||||
}
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,468 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/aggspec.h>
|
||||
#include <perspective/base.h>
|
||||
#include <sstream>
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_col_name_type::t_col_name_type() : m_type(DTYPE_NONE) {}
|
||||
|
||||
t_col_name_type::t_col_name_type(std::string name, t_dtype type) :
|
||||
m_name(std::move(name)),
|
||||
m_type(type) {}
|
||||
|
||||
t_aggspec::t_aggspec() = default;
|
||||
|
||||
t_aggspec::t_aggspec(
|
||||
const std::string& name,
|
||||
t_aggtype agg,
|
||||
const std::vector<t_dep>& dependencies
|
||||
) :
|
||||
m_name(name),
|
||||
m_disp_name(name),
|
||||
m_agg(agg),
|
||||
m_dependencies(dependencies) {}
|
||||
|
||||
t_aggspec::t_aggspec(
|
||||
const std::string& aggname, t_aggtype agg, const std::string& dep
|
||||
) :
|
||||
m_name(aggname),
|
||||
m_disp_name(aggname),
|
||||
m_agg(agg),
|
||||
m_dependencies(std::vector<t_dep>{t_dep(dep, DEPTYPE_COLUMN)}) {}
|
||||
|
||||
t_aggspec::t_aggspec(t_aggtype agg, const std::string& dep) :
|
||||
m_agg(agg),
|
||||
m_dependencies(std::vector<t_dep>{t_dep(dep, DEPTYPE_COLUMN)}) {}
|
||||
|
||||
t_aggspec::t_aggspec(
|
||||
std::string name,
|
||||
std::string disp_name,
|
||||
t_aggtype agg,
|
||||
const std::vector<t_dep>& dependencies
|
||||
) :
|
||||
m_name(std::move(name)),
|
||||
m_disp_name(std::move(disp_name)),
|
||||
m_agg(agg),
|
||||
m_dependencies(dependencies) {}
|
||||
|
||||
t_aggspec::t_aggspec(
|
||||
std::string name,
|
||||
std::string disp_name,
|
||||
t_aggtype agg,
|
||||
const std::vector<t_dep>& dependencies,
|
||||
t_sorttype sort_type
|
||||
) :
|
||||
m_name(std::move(name)),
|
||||
m_disp_name(std::move(disp_name)),
|
||||
m_agg(agg),
|
||||
m_dependencies(dependencies),
|
||||
m_sort_type(sort_type) {}
|
||||
|
||||
t_aggspec::t_aggspec(
|
||||
std::string aggname,
|
||||
std::string disp_aggname,
|
||||
t_aggtype agg,
|
||||
t_uindex agg_one_idx,
|
||||
t_uindex agg_two_idx,
|
||||
double agg_one_weight,
|
||||
double agg_two_weight
|
||||
) :
|
||||
m_name(std::move(aggname)),
|
||||
m_disp_name(std::move(disp_aggname)),
|
||||
m_agg(agg),
|
||||
m_agg_one_idx(agg_one_idx),
|
||||
m_agg_two_idx(agg_two_idx),
|
||||
m_agg_one_weight(agg_one_weight),
|
||||
m_agg_two_weight(agg_two_weight) {}
|
||||
|
||||
t_aggspec::~t_aggspec() = default;
|
||||
|
||||
std::string
|
||||
t_aggspec::name() const {
|
||||
return m_name;
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_aggspec::name_scalar() const {
|
||||
t_tscalar s;
|
||||
s.set(m_name.c_str());
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_aggspec::disp_name() const {
|
||||
return m_disp_name;
|
||||
}
|
||||
|
||||
t_aggtype
|
||||
t_aggspec::agg() const {
|
||||
return m_agg;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_aggspec::agg_str() const {
|
||||
switch (m_agg) {
|
||||
case AGGTYPE_SUM: {
|
||||
return "sum";
|
||||
} break;
|
||||
case AGGTYPE_SUM_ABS: {
|
||||
return "sum abs";
|
||||
} break;
|
||||
case AGGTYPE_ABS_SUM: {
|
||||
return "abs sum";
|
||||
} break;
|
||||
case AGGTYPE_MUL: {
|
||||
return "mul";
|
||||
} break;
|
||||
case AGGTYPE_COUNT: {
|
||||
return "count";
|
||||
} break;
|
||||
case AGGTYPE_MEAN: {
|
||||
return "mean";
|
||||
} break;
|
||||
case AGGTYPE_WEIGHTED_MEAN: {
|
||||
return "weighted mean";
|
||||
} break;
|
||||
case AGGTYPE_UNIQUE: {
|
||||
return "unique";
|
||||
} break;
|
||||
case AGGTYPE_ANY: {
|
||||
return "any";
|
||||
} break;
|
||||
case AGGTYPE_Q1: {
|
||||
return "q1";
|
||||
} break;
|
||||
case AGGTYPE_Q3: {
|
||||
return "q3";
|
||||
} break;
|
||||
case AGGTYPE_MEDIAN: {
|
||||
return "median";
|
||||
} break;
|
||||
case AGGTYPE_JOIN: {
|
||||
return "join";
|
||||
} break;
|
||||
case AGGTYPE_SCALED_DIV: {
|
||||
return "scaled div";
|
||||
} break;
|
||||
case AGGTYPE_SCALED_ADD: {
|
||||
return "scaled add";
|
||||
} break;
|
||||
case AGGTYPE_SCALED_MUL: {
|
||||
return "scaled mul";
|
||||
} break;
|
||||
case AGGTYPE_DOMINANT: {
|
||||
return "dominant";
|
||||
} break;
|
||||
case AGGTYPE_FIRST: {
|
||||
return "first";
|
||||
} break;
|
||||
case AGGTYPE_LAST_BY_INDEX: {
|
||||
return "last by index";
|
||||
} break;
|
||||
case AGGTYPE_LAST_MINUS_FIRST: {
|
||||
return "last minus first";
|
||||
} break;
|
||||
case AGGTYPE_PY_AGG: {
|
||||
return "py agg";
|
||||
} break;
|
||||
case AGGTYPE_AND: {
|
||||
return "and";
|
||||
} break;
|
||||
case AGGTYPE_OR: {
|
||||
return "or";
|
||||
} break;
|
||||
case AGGTYPE_LAST_VALUE: {
|
||||
return "last value";
|
||||
}
|
||||
case AGGTYPE_MAX_BY: {
|
||||
return "max by";
|
||||
}
|
||||
case AGGTYPE_MIN_BY: {
|
||||
return "min by";
|
||||
}
|
||||
case AGGTYPE_MAX: {
|
||||
return "max";
|
||||
}
|
||||
case AGGTYPE_MIN: {
|
||||
return "min";
|
||||
}
|
||||
case AGGTYPE_HIGH_WATER_MARK: {
|
||||
return "high water mark";
|
||||
}
|
||||
case AGGTYPE_LOW_WATER_MARK: {
|
||||
return "low water mark";
|
||||
}
|
||||
case AGGTYPE_HIGH_MINUS_LOW: {
|
||||
return "high minuslow";
|
||||
} break;
|
||||
case AGGTYPE_UDF_COMBINER: {
|
||||
std::stringstream ss;
|
||||
ss << "udf_combiner_" << disp_name();
|
||||
return ss.str();
|
||||
}
|
||||
case AGGTYPE_UDF_REDUCER: {
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "udf_reducer_" << disp_name();
|
||||
return ss.str();
|
||||
}
|
||||
case AGGTYPE_SUM_NOT_NULL: {
|
||||
return "sum not null";
|
||||
}
|
||||
case AGGTYPE_MEAN_BY_COUNT: {
|
||||
return "mean by count";
|
||||
}
|
||||
case AGGTYPE_IDENTITY: {
|
||||
return "identity";
|
||||
}
|
||||
case AGGTYPE_DISTINCT_COUNT: {
|
||||
return "distinct count";
|
||||
}
|
||||
case AGGTYPE_DISTINCT_LEAF: {
|
||||
return "distinct leaf";
|
||||
}
|
||||
case AGGTYPE_PCT_SUM_PARENT: {
|
||||
return "pct sum parent";
|
||||
}
|
||||
case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
|
||||
return "pct sum grand total";
|
||||
}
|
||||
case AGGTYPE_VARIANCE: {
|
||||
return "variance";
|
||||
}
|
||||
case AGGTYPE_STANDARD_DEVIATION: {
|
||||
return "stddev";
|
||||
}
|
||||
case AGGTYPE_GMV: {
|
||||
return "gmv";
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unknown agg type");
|
||||
return "unknown";
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
const std::vector<t_dep>&
|
||||
t_aggspec::get_dependencies() const {
|
||||
return m_dependencies;
|
||||
}
|
||||
|
||||
t_dtype
|
||||
get_simple_accumulator_type(t_dtype coltype) {
|
||||
switch (coltype) {
|
||||
case DTYPE_BOOL:
|
||||
case DTYPE_INT64:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT8: {
|
||||
return DTYPE_INT64;
|
||||
} break;
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT8: {
|
||||
return DTYPE_UINT64;
|
||||
}
|
||||
case DTYPE_FLOAT64:
|
||||
case DTYPE_FLOAT32: {
|
||||
return DTYPE_FLOAT64;
|
||||
}
|
||||
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected coltype");
|
||||
}
|
||||
}
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
t_sorttype
|
||||
t_aggspec::get_sort_type() const {
|
||||
return m_sort_type;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_aggspec::get_agg_one_idx() const {
|
||||
return m_agg_one_idx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_aggspec::get_agg_two_idx() const {
|
||||
return m_agg_two_idx;
|
||||
}
|
||||
|
||||
double
|
||||
t_aggspec::get_agg_one_weight() const {
|
||||
return m_agg_one_weight;
|
||||
}
|
||||
|
||||
double
|
||||
t_aggspec::get_agg_two_weight() const {
|
||||
return m_agg_two_weight;
|
||||
}
|
||||
|
||||
t_invmode
|
||||
t_aggspec::get_inv_mode() const {
|
||||
return m_invmode;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_aggspec::get_input_depnames() const {
|
||||
std::vector<std::string> rval;
|
||||
rval.reserve(m_dependencies.size());
|
||||
for (const auto& d : m_dependencies) {
|
||||
rval.push_back(d.name());
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_aggspec::get_output_depnames() const {
|
||||
std::vector<std::string> rval;
|
||||
rval.reserve(m_dependencies.size());
|
||||
for (const auto& d : m_dependencies) {
|
||||
rval.push_back(d.name());
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_col_name_type>
|
||||
t_aggspec::get_output_specs(const t_schema& schema) const {
|
||||
switch (agg()) {
|
||||
case AGGTYPE_SUM:
|
||||
case AGGTYPE_SUM_ABS:
|
||||
case AGGTYPE_ABS_SUM:
|
||||
case AGGTYPE_GMV:
|
||||
case AGGTYPE_PCT_SUM_PARENT:
|
||||
case AGGTYPE_PCT_SUM_GRAND_TOTAL:
|
||||
case AGGTYPE_MUL:
|
||||
case AGGTYPE_SUM_NOT_NULL: {
|
||||
t_dtype coltype = schema.get_dtype(m_dependencies[0].name());
|
||||
return mk_col_name_type_vec(
|
||||
name(), get_simple_accumulator_type(coltype)
|
||||
);
|
||||
}
|
||||
case AGGTYPE_ANY:
|
||||
case AGGTYPE_UNIQUE:
|
||||
case AGGTYPE_DOMINANT:
|
||||
case AGGTYPE_Q1:
|
||||
case AGGTYPE_Q3:
|
||||
case AGGTYPE_MEDIAN:
|
||||
case AGGTYPE_FIRST:
|
||||
case AGGTYPE_LAST_BY_INDEX:
|
||||
case AGGTYPE_LAST_MINUS_FIRST:
|
||||
case AGGTYPE_OR:
|
||||
case AGGTYPE_LAST_VALUE:
|
||||
case AGGTYPE_MAX:
|
||||
case AGGTYPE_MAX_BY:
|
||||
case AGGTYPE_MIN_BY:
|
||||
case AGGTYPE_MIN:
|
||||
case AGGTYPE_HIGH_WATER_MARK:
|
||||
case AGGTYPE_LOW_WATER_MARK:
|
||||
case AGGTYPE_HIGH_MINUS_LOW:
|
||||
case AGGTYPE_IDENTITY:
|
||||
case AGGTYPE_DISTINCT_LEAF: {
|
||||
t_dtype coltype = schema.get_dtype(m_dependencies[0].name());
|
||||
std::vector<t_col_name_type> rval(1);
|
||||
rval[0].m_name = name();
|
||||
rval[0].m_type = coltype;
|
||||
return rval;
|
||||
}
|
||||
case AGGTYPE_COUNT: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_INT64);
|
||||
}
|
||||
case AGGTYPE_MEAN:
|
||||
case AGGTYPE_MEAN_BY_COUNT:
|
||||
case AGGTYPE_WEIGHTED_MEAN: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_F64PAIR);
|
||||
}
|
||||
case AGGTYPE_JOIN: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_STR);
|
||||
}
|
||||
case AGGTYPE_SCALED_DIV:
|
||||
case AGGTYPE_SCALED_ADD:
|
||||
case AGGTYPE_SCALED_MUL:
|
||||
case AGGTYPE_VARIANCE:
|
||||
case AGGTYPE_STANDARD_DEVIATION: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_FLOAT64);
|
||||
}
|
||||
case AGGTYPE_UDF_COMBINER:
|
||||
case AGGTYPE_UDF_REDUCER: {
|
||||
std::vector<t_col_name_type> rval;
|
||||
rval.reserve(m_odependencies.size());
|
||||
for (const auto& d : m_odependencies) {
|
||||
t_col_name_type tp(d.name(), d.dtype());
|
||||
rval.push_back(tp);
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
case AGGTYPE_AND: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_BOOL);
|
||||
}
|
||||
case AGGTYPE_DISTINCT_COUNT: {
|
||||
return mk_col_name_type_vec(name(), DTYPE_UINT32);
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unknown agg type");
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<t_col_name_type>
|
||||
t_aggspec::mk_col_name_type_vec(const std::string& name, t_dtype dtype) const {
|
||||
std::vector<t_col_name_type> rval(1);
|
||||
rval[0].m_name = name;
|
||||
rval[0].m_type = dtype;
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_aggspec::is_combiner_agg() const {
|
||||
return m_agg == AGGTYPE_UDF_COMBINER;
|
||||
}
|
||||
|
||||
bool
|
||||
t_aggspec::is_reducer_agg() const {
|
||||
return m_agg == AGGTYPE_UDF_REDUCER;
|
||||
}
|
||||
|
||||
bool
|
||||
t_aggspec::is_non_delta() const {
|
||||
switch (m_agg) {
|
||||
case AGGTYPE_LAST_VALUE:
|
||||
case AGGTYPE_LOW_WATER_MARK:
|
||||
case AGGTYPE_HIGH_WATER_MARK: {
|
||||
return true;
|
||||
}
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_aggspec::get_first_depname() const {
|
||||
if (m_dependencies.empty()) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return m_dependencies[0].name();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,113 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <functional>
|
||||
#include <perspective/arg_sort.h>
|
||||
#include <perspective/multi_sort.h>
|
||||
#include <perspective/scalar.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
void
|
||||
argsort(std::vector<t_index>& output, const t_multisorter& sorter) {
|
||||
if (output.empty()) {
|
||||
return;
|
||||
}
|
||||
// Output should be the same size is v
|
||||
for (t_index i = 0, loop_end = output.size(); i != loop_end; ++i) {
|
||||
output[i] = i;
|
||||
}
|
||||
std::sort(output.begin(), output.end(), sorter);
|
||||
}
|
||||
|
||||
t_argsort_comparator::t_argsort_comparator(
|
||||
const std::vector<t_tscalar>& v, const t_sorttype& sort_type
|
||||
) :
|
||||
m_v(v),
|
||||
m_sort_type(sort_type) {}
|
||||
|
||||
bool
|
||||
t_argsort_comparator::operator()(t_index a, t_index b) const {
|
||||
switch (m_sort_type) {
|
||||
case SORTTYPE_ASCENDING:
|
||||
return t_argsort_comparator_impl<SORTTYPE_ASCENDING>(m_v)(a, b);
|
||||
case SORTTYPE_DESCENDING:
|
||||
return t_argsort_comparator_impl<SORTTYPE_DESCENDING>(m_v)(a, b);
|
||||
case SORTTYPE_ASCENDING_ABS:
|
||||
return t_argsort_comparator_impl<SORTTYPE_ASCENDING_ABS>(m_v)(a, b);
|
||||
case SORTTYPE_DESCENDING_ABS:
|
||||
return t_argsort_comparator_impl<SORTTYPE_DESCENDING_ABS>(m_v)(
|
||||
a, b
|
||||
);
|
||||
case SORTTYPE_NONE:
|
||||
return t_argsort_comparator_impl<SORTTYPE_NONE>(m_v)(a, b);
|
||||
}
|
||||
|
||||
return a < b;
|
||||
}
|
||||
|
||||
namespace {
|
||||
|
||||
void
|
||||
init_output(std::vector<t_index>& output) {
|
||||
for (t_index i = 0, loop_end = output.size(); i != loop_end; ++i) {
|
||||
output[i] = i;
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void
|
||||
simple_argsort(
|
||||
std::vector<t_tscalar>& v,
|
||||
std::vector<t_index>& output,
|
||||
const t_sorttype& sort_type
|
||||
) {
|
||||
init_output(output);
|
||||
|
||||
switch (sort_type) {
|
||||
case SORTTYPE_ASCENDING: {
|
||||
std::sort(
|
||||
output.begin(),
|
||||
output.end(),
|
||||
t_argsort_comparator_impl<SORTTYPE_ASCENDING>(v)
|
||||
);
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING: {
|
||||
std::sort(
|
||||
output.begin(),
|
||||
output.end(),
|
||||
t_argsort_comparator_impl<SORTTYPE_DESCENDING>(v)
|
||||
);
|
||||
} break;
|
||||
case SORTTYPE_ASCENDING_ABS: {
|
||||
std::sort(
|
||||
output.begin(),
|
||||
output.end(),
|
||||
t_argsort_comparator_impl<SORTTYPE_ASCENDING_ABS>(v)
|
||||
);
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING_ABS: {
|
||||
std::sort(
|
||||
output.begin(),
|
||||
output.end(),
|
||||
t_argsort_comparator_impl<SORTTYPE_DESCENDING_ABS>(v)
|
||||
);
|
||||
} break;
|
||||
case SORTTYPE_NONE: {
|
||||
// output is already identity-initialized
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,639 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <chrono>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/arrow_csv.h>
|
||||
#include <arrow/util/value_parsing.h>
|
||||
#include <arrow/io/memory.h>
|
||||
#include <arrow/csv/reader.h>
|
||||
|
||||
template <class TimePoint>
|
||||
static inline arrow::TimestampType::c_type
|
||||
ConvertTimePoint(TimePoint tp, arrow::TimeUnit::type unit) {
|
||||
auto duration = tp.time_since_epoch();
|
||||
switch (unit) {
|
||||
case arrow::TimeUnit::SECOND:
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(duration)
|
||||
.count();
|
||||
case arrow::TimeUnit::MILLI:
|
||||
return std::chrono::duration_cast<std::chrono::milliseconds>(
|
||||
duration
|
||||
)
|
||||
.count();
|
||||
case arrow::TimeUnit::MICRO:
|
||||
return std::chrono::duration_cast<std::chrono::microseconds>(
|
||||
duration
|
||||
)
|
||||
.count();
|
||||
case arrow::TimeUnit::NANO:
|
||||
return std::chrono::duration_cast<std::chrono::nanoseconds>(duration
|
||||
)
|
||||
.count();
|
||||
default:
|
||||
// Compiler errors without default case even though all enum cases
|
||||
// are handled
|
||||
assert(0);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseYYYY_MM_DD(const char* s, arrow_vendored::date::year_month_day* out) {
|
||||
uint16_t year = 0;
|
||||
uint8_t month = 0;
|
||||
uint8_t day = 0;
|
||||
if (ARROW_PREDICT_FALSE(s[4] != '-') || ARROW_PREDICT_FALSE(s[7] != '-')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 4, &year))) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 5, 2, &month)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 8, 2, &day))) {
|
||||
return false;
|
||||
}
|
||||
*out = {
|
||||
arrow_vendored::date::year{year},
|
||||
arrow_vendored::date::month{month},
|
||||
arrow_vendored::date::day{day}
|
||||
};
|
||||
return out->ok();
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseYYYY_DD_MM(const char* s, arrow_vendored::date::year_month_day* out) {
|
||||
uint16_t year = 0;
|
||||
uint8_t month = 0;
|
||||
uint8_t day = 0;
|
||||
if (ARROW_PREDICT_FALSE(s[2] != '/') || ARROW_PREDICT_FALSE(s[5] != '/')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 6, 4, &year))) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 2, &month)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 3, 2, &day))) {
|
||||
return false;
|
||||
}
|
||||
*out = {
|
||||
arrow_vendored::date::year{year},
|
||||
arrow_vendored::date::month{month},
|
||||
arrow_vendored::date::day{day}
|
||||
};
|
||||
return out->ok();
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseYYYY_D_M(const char* s, arrow_vendored::date::year_month_day* out) {
|
||||
uint16_t year = 0;
|
||||
uint8_t month = 0;
|
||||
uint8_t day = 0;
|
||||
|
||||
if (ARROW_PREDICT_FALSE(s[1] != '/') || ARROW_PREDICT_FALSE(s[3] != '/')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 4, 4, &year))) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 1, &month)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 2, 1, &day))) {
|
||||
return false;
|
||||
}
|
||||
*out = {
|
||||
arrow_vendored::date::year{year},
|
||||
arrow_vendored::date::month{month},
|
||||
arrow_vendored::date::day{day}
|
||||
};
|
||||
return out->ok();
|
||||
}
|
||||
|
||||
static inline std::string
|
||||
extract_string(const char* ch, int start_idx, int number_of_chars) {
|
||||
std::string out;
|
||||
for (int i = start_idx; i < start_idx + number_of_chars; i++) {
|
||||
out += ch[i];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseAM_PM(const char* s, std::chrono::seconds& seconds, int length) {
|
||||
int hour = 0;
|
||||
int twelve_hours = 12;
|
||||
std::string am_pm;
|
||||
std::string hour_string;
|
||||
|
||||
if (length == 21) {
|
||||
am_pm = extract_string(s, 19, 2);
|
||||
hour_string = extract_string(s, 10, 2);
|
||||
hour = atoi(hour_string.c_str());
|
||||
if (hour == 0) {
|
||||
return false;
|
||||
}
|
||||
} else if (length == 23) {
|
||||
am_pm = extract_string(s, 21, 2);
|
||||
hour_string = extract_string(s, 12, 2);
|
||||
hour = atoi(hour_string.c_str());
|
||||
if (hour == 0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if ((am_pm == "PM" || am_pm == "pm") && (hour < twelve_hours)) {
|
||||
std::chrono::hours hours_obj(twelve_hours);
|
||||
seconds = std::chrono::duration_cast<std::chrono::seconds>(hours_obj);
|
||||
} else if ((am_pm == "AM" || am_pm == "am") && (hour == twelve_hours)) {
|
||||
std::chrono::hours hours_obj(twelve_hours);
|
||||
seconds =
|
||||
std::chrono::duration_cast<std::chrono::seconds>(hours_obj) * -1;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
namespace perspective::apachearrow {
|
||||
|
||||
class UnixTimestampParser : public arrow::TimestampParser {
|
||||
public:
|
||||
bool
|
||||
operator()(
|
||||
const char* s,
|
||||
size_t length,
|
||||
arrow::TimeUnit::type out_unit,
|
||||
int64_t* out,
|
||||
bool* out_zone_offset_present = NULLPTR
|
||||
) const override {
|
||||
size_t endptr;
|
||||
std::string val(s, s + length);
|
||||
int64_t value = std::stoll(static_cast<std::string>(val), &endptr, 10);
|
||||
if (endptr != length) {
|
||||
return false;
|
||||
}
|
||||
(*out) = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
const char*
|
||||
kind() const override {
|
||||
return "unixtimestamp";
|
||||
}
|
||||
};
|
||||
|
||||
static inline bool
|
||||
ParseSSS(const char* s, std::chrono::milliseconds* out) {
|
||||
uint16_t millis = 0;
|
||||
if (ARROW_PREDICT_FALSE(s[0] != '.')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 3, &millis)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ARROW_PREDICT_FALSE(millis >= 999)) {
|
||||
return false;
|
||||
}
|
||||
*out = std::chrono::milliseconds(millis);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseSSSSSS(const char* s, std::chrono::microseconds* out) {
|
||||
uint32_t nanos = 0;
|
||||
if (ARROW_PREDICT_FALSE(s[0] != '.')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 6, &nanos)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ARROW_PREDICT_FALSE(nanos >= 999999)) {
|
||||
return false;
|
||||
}
|
||||
*out = std::chrono::microseconds(nanos);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseSSSSSSSSS(const char* s, std::chrono::nanoseconds* out) {
|
||||
uint32_t nanos = 0;
|
||||
if (ARROW_PREDICT_FALSE(s[0] != '.')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 9, &nanos)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ARROW_PREDICT_FALSE(nanos >= 999999999)) {
|
||||
return false;
|
||||
}
|
||||
*out = std::chrono::nanoseconds(nanos);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool
|
||||
ParseTZ(const char* s, std::chrono::minutes* out) {
|
||||
uint8_t hours = 0;
|
||||
uint8_t minutes = 0;
|
||||
if ((ARROW_PREDICT_FALSE(s[0] != '+') && ARROW_PREDICT_FALSE(s[0] != '-'))
|
||||
|| ARROW_PREDICT_FALSE(s[3] != ':')) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 2, &hours)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 4, 2, &minutes)
|
||||
)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (ARROW_PREDICT_FALSE(hours >= 12)
|
||||
|| ARROW_PREDICT_FALSE(minutes >= 59)) {
|
||||
return false;
|
||||
}
|
||||
int32_t total = hours * 60 + minutes;
|
||||
if (s[0] == '+') {
|
||||
total = -total;
|
||||
}
|
||||
*out = std::chrono::minutes(total);
|
||||
return true;
|
||||
}
|
||||
|
||||
class CustomISO8601Parser : public arrow::TimestampParser {
|
||||
public:
|
||||
bool
|
||||
operator()(
|
||||
const char* s,
|
||||
size_t length,
|
||||
arrow::TimeUnit::type unit,
|
||||
int64_t* out,
|
||||
bool* out_zone_offset_present = NULLPTR
|
||||
) const override {
|
||||
// if we are trying to parse this with seconds, fail
|
||||
// and it will try to parse this again but as
|
||||
// nanoseconds :) then it wont truncate the fractional bits.
|
||||
if (unit == arrow::TimeUnit::SECOND) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!arrow::internal::ParseTimestampISO8601(s, length, unit, out)) {
|
||||
if (s[length - 1] == 'Z') {
|
||||
--length;
|
||||
}
|
||||
if (length == 23) {
|
||||
// "YYYY-MM-DD[ T]hh:mm:ss.sss" -- millis
|
||||
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::milliseconds millis;
|
||||
if (ARROW_PREDICT_FALSE(!ParseSSS(s + 19, &millis))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (length == 25) {
|
||||
// "2008-09-15[ T]15:53:00+05:00" -- seconds with TZ
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::minutes tz;
|
||||
if (ARROW_PREDICT_FALSE(!ParseTZ(s + 19, &tz))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + tz + seconds, unit
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (length == 26) {
|
||||
// YYYY-MM-DD[ T]hh:mm:ss.ssssss -- micros
|
||||
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::microseconds micros;
|
||||
if (ARROW_PREDICT_FALSE(!ParseSSSSSS(s + 19, µs))) {
|
||||
return false;
|
||||
}
|
||||
// round the micros into millis as Perspective does not support
|
||||
// nano precision.
|
||||
auto millis =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(micros
|
||||
);
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
if (length == 29) {
|
||||
// YYYY-MM-DD[ T]hh:mm:ss.sssssssss -- nanos
|
||||
// arrow handles YYYY-MM-DD[ T]hh:mm:ss.sss[+-]HH:MM
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
// we can now be at sss[+-]HH:MM -- millis and TZ
|
||||
// or sssssssss -- nanos
|
||||
std::chrono::nanoseconds nanos;
|
||||
if (ARROW_PREDICT_FALSE(!ParseSSSSSSSSS(s + 19, &nanos))) {
|
||||
return false;
|
||||
}
|
||||
// Truncate the nanos into millis as Perspective does not
|
||||
// support nano precision.
|
||||
auto millis =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(nanos
|
||||
);
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
if (length == 32) {
|
||||
// YYYY-MM-DD[ T]hh:mm:ss.ssssss[+-]HH:MM -- micros with TZ
|
||||
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::microseconds micros;
|
||||
if (ARROW_PREDICT_FALSE(!ParseSSSSSS(s + 19, µs))) {
|
||||
return false;
|
||||
}
|
||||
// round the micros into millis as Perspective does not support
|
||||
// nano precision.
|
||||
auto millis =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(micros
|
||||
);
|
||||
|
||||
std::chrono::minutes tz;
|
||||
if (ARROW_PREDICT_FALSE(!ParseTZ(s + 26, &tz))) {
|
||||
return false;
|
||||
}
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + millis + tz,
|
||||
unit
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
if (length == 35) {
|
||||
// YYYY-MM-DD[ T]hh:mm:ss.sssssssss[+-]HH:MM -- nanos with TZ
|
||||
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 11, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::nanoseconds nanos;
|
||||
if (ARROW_PREDICT_FALSE(!ParseSSSSSSSSS(s + 19, &nanos))) {
|
||||
return false;
|
||||
}
|
||||
// round the nanos into millis as Perspective does not support
|
||||
// nano precision.
|
||||
auto millis =
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(nanos
|
||||
);
|
||||
|
||||
std::chrono::minutes tz;
|
||||
if (ARROW_PREDICT_FALSE(!ParseTZ(s + 29, &tz))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + millis + tz,
|
||||
unit
|
||||
);
|
||||
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
const char*
|
||||
kind() const override {
|
||||
return "custom_ISO8601";
|
||||
}
|
||||
};
|
||||
|
||||
class USTimestampParser : public arrow::TimestampParser {
|
||||
public:
|
||||
bool
|
||||
operator()(
|
||||
const char* s,
|
||||
size_t length,
|
||||
arrow::TimeUnit::type unit,
|
||||
int64_t* out,
|
||||
bool* out_zone_offset_present = NULLPTR
|
||||
) const override {
|
||||
|
||||
if (!arrow::internal::ParseTimestampISO8601(s, length, unit, out)) {
|
||||
if (length == 23) {
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_DD_MM(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 12, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::chrono::seconds am_pm(0);
|
||||
if (ARROW_PREDICT_FALSE(!ParseAM_PM(s, am_pm, 23))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + am_pm, unit
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (length == 21) {
|
||||
arrow_vendored::date::year_month_day ymd;
|
||||
if (ARROW_PREDICT_FALSE(!ParseYYYY_D_M(s, &ymd))) {
|
||||
return false;
|
||||
}
|
||||
std::chrono::seconds seconds;
|
||||
if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
|
||||
s + 10, &seconds
|
||||
))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
std::chrono::seconds am_pm(0);
|
||||
if (ARROW_PREDICT_FALSE(!ParseAM_PM(s, am_pm, 21))) {
|
||||
return false;
|
||||
}
|
||||
|
||||
*out = ConvertTimePoint(
|
||||
arrow_vendored::date::sys_days(ymd) + seconds + am_pm, unit
|
||||
);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
[[nodiscard]]
|
||||
const char*
|
||||
kind() const override {
|
||||
return "USTimestampParser";
|
||||
}
|
||||
};
|
||||
|
||||
std::vector<std::shared_ptr<arrow::TimestampParser>> DATE_PARSERS{
|
||||
std::make_shared<CustomISO8601Parser>(),
|
||||
std::make_shared<USTimestampParser>(),
|
||||
arrow::TimestampParser::MakeStrptime("%Y-%m-%d\\D%H:%M:%S.%f"),
|
||||
arrow::TimestampParser::MakeStrptime("%m/%d/%Y, %I:%M:%S %p"),
|
||||
// US locale string
|
||||
arrow::TimestampParser::MakeStrptime("%m-%d-%Y"),
|
||||
arrow::TimestampParser::MakeStrptime("%m/%d/%Y"),
|
||||
arrow::TimestampParser::MakeStrptime("%d %m %Y"),
|
||||
// TODO: time type column
|
||||
arrow::TimestampParser::MakeStrptime("%H:%M:%S.%f")
|
||||
};
|
||||
|
||||
std::vector<std::shared_ptr<arrow::TimestampParser>> DATE_READERS{
|
||||
std::make_shared<UnixTimestampParser>(),
|
||||
std::make_shared<CustomISO8601Parser>(),
|
||||
std::make_shared<USTimestampParser>(),
|
||||
arrow::TimestampParser::MakeStrptime("%Y-%m-%d\\D%H:%M:%S.%f"),
|
||||
arrow::TimestampParser::MakeStrptime("%m/%d/%Y, %I:%M:%S %p"),
|
||||
// US locale
|
||||
arrow::TimestampParser::MakeStrptime("%m-%d-%Y"),
|
||||
arrow::TimestampParser::MakeStrptime("%m/%d/%Y"),
|
||||
arrow::TimestampParser::MakeStrptime("%d %m %Y"),
|
||||
arrow::TimestampParser::MakeStrptime("%H:%M:%S.%f")
|
||||
};
|
||||
|
||||
std::optional<int64_t>
|
||||
parseAsArrowTimestamp(const std::string& input) {
|
||||
for (const auto& candidate : DATE_PARSERS) {
|
||||
int64_t datetime;
|
||||
if (candidate->operator()(
|
||||
input.c_str(), input.size(), arrow::TimeUnit::MILLI, &datetime
|
||||
)) {
|
||||
return datetime;
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
std::shared_ptr<::arrow::Table>
|
||||
csvToTable(
|
||||
const std::string_view& csv,
|
||||
bool is_update,
|
||||
std::unordered_map<std::string, std::shared_ptr<arrow::DataType>>& schema
|
||||
) {
|
||||
const arrow::io::IOContext& io_context = arrow::io::default_io_context();
|
||||
auto input = std::make_shared<arrow::io::BufferReader>(csv);
|
||||
auto read_options = arrow::csv::ReadOptions::Defaults();
|
||||
auto parse_options = arrow::csv::ParseOptions::Defaults();
|
||||
auto convert_options = arrow::csv::ConvertOptions::Defaults();
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
read_options.use_threads = true;
|
||||
#else
|
||||
read_options.use_threads = false;
|
||||
#endif
|
||||
parse_options.newlines_in_values = true;
|
||||
|
||||
if (is_update) {
|
||||
convert_options.column_types = std::move(schema);
|
||||
convert_options.timestamp_parsers = DATE_READERS;
|
||||
} else {
|
||||
convert_options.timestamp_parsers = DATE_PARSERS;
|
||||
}
|
||||
|
||||
auto maybe_reader = arrow::csv::TableReader::Make(
|
||||
io_context, input, read_options, parse_options, convert_options
|
||||
);
|
||||
|
||||
std::shared_ptr<arrow::csv::TableReader> reader = *maybe_reader;
|
||||
|
||||
auto maybe_table = reader->Read();
|
||||
if (!maybe_table.ok()) {
|
||||
PSP_COMPLAIN_AND_ABORT(maybe_table.status().ToString());
|
||||
}
|
||||
return *maybe_table;
|
||||
}
|
||||
|
||||
} // namespace perspective::apachearrow
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,81 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/arrow_writer.h>
|
||||
|
||||
namespace perspective::apachearrow {
|
||||
using namespace perspective;
|
||||
|
||||
// TODO: unsure about efficacy of these functions when get<T> exists
|
||||
template <>
|
||||
double
|
||||
get_scalar<double>(t_tscalar& t) {
|
||||
return t.to_double();
|
||||
}
|
||||
template <>
|
||||
float
|
||||
get_scalar<float>(t_tscalar& t) {
|
||||
return static_cast<float>(t.to_double());
|
||||
}
|
||||
template <>
|
||||
std::uint8_t
|
||||
get_scalar<std::uint8_t>(t_tscalar& t) {
|
||||
return static_cast<std::uint8_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::int8_t
|
||||
get_scalar<std::int8_t>(t_tscalar& t) {
|
||||
return static_cast<std::int8_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::int16_t
|
||||
get_scalar<std::int16_t>(t_tscalar& t) {
|
||||
return static_cast<std::int16_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::uint16_t
|
||||
get_scalar<std::uint16_t>(t_tscalar& t) {
|
||||
return static_cast<std::uint16_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::int32_t
|
||||
get_scalar<std::int32_t>(t_tscalar& t) {
|
||||
return static_cast<std::int32_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::uint32_t
|
||||
get_scalar<std::uint32_t>(t_tscalar& t) {
|
||||
return static_cast<std::uint32_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::int64_t
|
||||
get_scalar<std::int64_t>(t_tscalar& t) {
|
||||
return static_cast<std::int64_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
std::uint64_t
|
||||
get_scalar<std::uint64_t>(t_tscalar& t) {
|
||||
return static_cast<std::uint64_t>(t.to_int64());
|
||||
}
|
||||
template <>
|
||||
bool
|
||||
get_scalar<bool>(t_tscalar& t) {
|
||||
return t.get<bool>();
|
||||
}
|
||||
|
||||
// std::int32_t
|
||||
// get_idx(std::int32_t cidx, std::int32_t ridx, std::int32_t stride,
|
||||
// t_get_data_extents extents) {
|
||||
// return (ridx - extents.m_srow) * stride + (cidx - extents.m_scol);
|
||||
// }
|
||||
|
||||
} // namespace perspective::apachearrow
|
||||
@@ -0,0 +1,821 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <cstdint>
|
||||
#include <limits>
|
||||
#include <perspective/exception.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
void
|
||||
psp_abort(const std::string& message) {
|
||||
throw PerspectiveException(message.c_str());
|
||||
}
|
||||
|
||||
bool
|
||||
is_numeric_type(t_dtype dtype) {
|
||||
switch (dtype) {
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT64:
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_FLOAT64: {
|
||||
return true;
|
||||
} break;
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
is_linear_order_type(t_dtype dtype) {
|
||||
switch (dtype) {
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT64:
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_FLOAT64:
|
||||
case DTYPE_DATE:
|
||||
case DTYPE_TIME:
|
||||
case DTYPE_BOOL: {
|
||||
return true;
|
||||
} break;
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
is_floating_point(t_dtype dtype) {
|
||||
return (dtype == DTYPE_FLOAT32 || dtype == DTYPE_FLOAT64);
|
||||
}
|
||||
|
||||
bool
|
||||
is_deterministic_sized(t_dtype dtype) {
|
||||
switch (dtype) {
|
||||
case DTYPE_OBJECT:
|
||||
case DTYPE_INT64:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_BOOL:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_FLOAT64:
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_STR:
|
||||
case DTYPE_TIME:
|
||||
case DTYPE_DATE:
|
||||
case DTYPE_F64PAIR: {
|
||||
return true;
|
||||
}
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT("Reached unreachable");
|
||||
return false;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
get_dtype_size(t_dtype dtype) {
|
||||
switch (dtype) {
|
||||
case DTYPE_OBJECT: {
|
||||
return sizeof(void*);
|
||||
}
|
||||
case DTYPE_INT64:
|
||||
case DTYPE_UINT64: {
|
||||
return sizeof(std::int64_t);
|
||||
}
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_UINT32: {
|
||||
return sizeof(std::int32_t);
|
||||
}
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_UINT16: {
|
||||
return 2;
|
||||
}
|
||||
case DTYPE_BOOL:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_NONE: {
|
||||
return 1;
|
||||
}
|
||||
case DTYPE_FLOAT64: {
|
||||
return sizeof(double);
|
||||
}
|
||||
case DTYPE_FLOAT32: {
|
||||
return sizeof(float);
|
||||
}
|
||||
case DTYPE_STR: {
|
||||
return sizeof(t_uindex);
|
||||
}
|
||||
case DTYPE_TIME: {
|
||||
return sizeof(std::int64_t);
|
||||
}
|
||||
case DTYPE_DATE: {
|
||||
return sizeof(std::uint32_t);
|
||||
}
|
||||
case DTYPE_F64PAIR: {
|
||||
return sizeof(std::pair<double, double>);
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unknown dtype");
|
||||
}
|
||||
}
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT("Reached unreachable");
|
||||
return sizeof(DTYPE_INT64);
|
||||
}
|
||||
|
||||
bool
|
||||
is_vlen_dtype(t_dtype dtype) {
|
||||
return dtype == DTYPE_STR || dtype == DTYPE_USER_VLEN;
|
||||
}
|
||||
|
||||
std::string
|
||||
get_dtype_descr(t_dtype dtype) {
|
||||
switch (dtype) {
|
||||
case DTYPE_NONE: {
|
||||
return "none";
|
||||
} break;
|
||||
case DTYPE_INT64: {
|
||||
return "int64";
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
return "int32";
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
return "int16";
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
return "int8";
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
return "uint64";
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
return "uint32";
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
return "uint16";
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
return "uint8";
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
return "bool";
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
return "float64";
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
return "float32";
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
return "str";
|
||||
} break;
|
||||
case DTYPE_TIME: {
|
||||
return "datetime";
|
||||
} break;
|
||||
case DTYPE_DATE: {
|
||||
return "date";
|
||||
} break;
|
||||
case DTYPE_ENUM: {
|
||||
return "e";
|
||||
} break;
|
||||
case DTYPE_OID: {
|
||||
return "oid";
|
||||
} break;
|
||||
case DTYPE_USER_FIXED: {
|
||||
return "ufix";
|
||||
} break;
|
||||
case DTYPE_LAST: {
|
||||
return "last";
|
||||
} break;
|
||||
case DTYPE_USER_VLEN: {
|
||||
return "uvlen";
|
||||
} break;
|
||||
case DTYPE_F64PAIR: {
|
||||
return "f64pair";
|
||||
} break;
|
||||
case DTYPE_OBJECT: {
|
||||
return "object";
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Encountered unknown dtype");
|
||||
}
|
||||
}
|
||||
return {"dummy"};
|
||||
}
|
||||
|
||||
std::string
|
||||
dtype_to_str(t_dtype dtype) {
|
||||
std::stringstream ss;
|
||||
switch (dtype) {
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_FLOAT64: {
|
||||
ss << "float";
|
||||
} break;
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT64: {
|
||||
ss << "integer";
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
ss << "boolean";
|
||||
} break;
|
||||
case DTYPE_DATE: {
|
||||
ss << "date";
|
||||
} break;
|
||||
case DTYPE_TIME: {
|
||||
ss << "datetime";
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
ss << "string";
|
||||
} break;
|
||||
case DTYPE_OBJECT: {
|
||||
ss << "object";
|
||||
} break;
|
||||
case DTYPE_NONE: {
|
||||
ss << "none";
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Cannot convert unknown dtype to string!");
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
t_dtype
|
||||
str_to_dtype(const std::string& typestring) {
|
||||
// returns most commonly used types in the JS/python public APIs.
|
||||
if (typestring == "integer") {
|
||||
return DTYPE_INT32;
|
||||
}
|
||||
if (typestring == "float") {
|
||||
return DTYPE_FLOAT64;
|
||||
}
|
||||
if (typestring == "boolean") {
|
||||
return DTYPE_BOOL;
|
||||
}
|
||||
if (typestring == "date") {
|
||||
return DTYPE_DATE;
|
||||
}
|
||||
if (typestring == "datetime") {
|
||||
return DTYPE_TIME;
|
||||
}
|
||||
if (typestring == "string") {
|
||||
return DTYPE_STR;
|
||||
}
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT(
|
||||
"Could not convert unknown type string `" + typestring + "` to dtype."
|
||||
);
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
std::string
|
||||
filter_op_to_str(t_filter_op op) {
|
||||
switch (op) {
|
||||
case FILTER_OP_LT: {
|
||||
return "<";
|
||||
} break;
|
||||
case FILTER_OP_LTEQ: {
|
||||
return "<=";
|
||||
} break;
|
||||
case FILTER_OP_GT: {
|
||||
return ">";
|
||||
} break;
|
||||
case FILTER_OP_GTEQ: {
|
||||
return ">=";
|
||||
} break;
|
||||
case FILTER_OP_EQ: {
|
||||
return "==";
|
||||
} break;
|
||||
case FILTER_OP_NE: {
|
||||
return "!=";
|
||||
} break;
|
||||
case FILTER_OP_BEGINS_WITH: {
|
||||
return "startswith";
|
||||
} break;
|
||||
case FILTER_OP_ENDS_WITH: {
|
||||
return "endswith";
|
||||
} break;
|
||||
case FILTER_OP_CONTAINS: {
|
||||
return "contains";
|
||||
} break;
|
||||
case FILTER_OP_OR: {
|
||||
return "or";
|
||||
} break;
|
||||
case FILTER_OP_IN: {
|
||||
return "in";
|
||||
} break;
|
||||
case FILTER_OP_NOT_IN: {
|
||||
return "not in";
|
||||
} break;
|
||||
case FILTER_OP_AND: {
|
||||
return "and";
|
||||
} break;
|
||||
case FILTER_OP_IS_NULL: {
|
||||
return "is null";
|
||||
} break;
|
||||
case FILTER_OP_IS_NOT_NULL: {
|
||||
return "is not null";
|
||||
} break;
|
||||
}
|
||||
PSP_COMPLAIN_AND_ABORT("Reached end of function");
|
||||
return "";
|
||||
}
|
||||
|
||||
t_filter_op
|
||||
str_to_filter_op(const std::string& str) {
|
||||
if (str == "<") {
|
||||
return t_filter_op::FILTER_OP_LT;
|
||||
}
|
||||
if (str == "<=") {
|
||||
return t_filter_op::FILTER_OP_LTEQ;
|
||||
}
|
||||
if (str == ">") {
|
||||
return t_filter_op::FILTER_OP_GT;
|
||||
}
|
||||
if (str == ">=") {
|
||||
return t_filter_op::FILTER_OP_GTEQ;
|
||||
}
|
||||
if (str == "==") {
|
||||
return t_filter_op::FILTER_OP_EQ;
|
||||
}
|
||||
if (str == "!=") {
|
||||
return t_filter_op::FILTER_OP_NE;
|
||||
}
|
||||
if (str == "begins with" || str == "startswith") {
|
||||
return t_filter_op::FILTER_OP_BEGINS_WITH;
|
||||
}
|
||||
if (str == "ends with" || str == "endswith") {
|
||||
return t_filter_op::FILTER_OP_ENDS_WITH;
|
||||
}
|
||||
if (str == "in") {
|
||||
return t_filter_op::FILTER_OP_IN;
|
||||
}
|
||||
if (str == "contains") {
|
||||
return t_filter_op::FILTER_OP_CONTAINS;
|
||||
}
|
||||
if (str == "not in") {
|
||||
return t_filter_op::FILTER_OP_NOT_IN;
|
||||
}
|
||||
if (str == "&" || str == "and") {
|
||||
return t_filter_op::FILTER_OP_AND;
|
||||
}
|
||||
if (str == "|" || str == "or") {
|
||||
return t_filter_op::FILTER_OP_OR;
|
||||
}
|
||||
if (str == "is null" || str == "is None") {
|
||||
return t_filter_op::FILTER_OP_IS_NULL;
|
||||
}
|
||||
if (str == "is not null" || str == "is not None") {
|
||||
return t_filter_op::FILTER_OP_IS_NOT_NULL;
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "Unknown filter operator string: `" << str << "`" << std::endl;
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
return t_filter_op::FILTER_OP_AND;
|
||||
}
|
||||
|
||||
t_sorttype
|
||||
str_to_sorttype(const std::string& str) {
|
||||
if (str == "none") {
|
||||
return SORTTYPE_NONE;
|
||||
}
|
||||
if (str == "asc" || str == "col asc") {
|
||||
return SORTTYPE_ASCENDING;
|
||||
}
|
||||
if (str == "desc" || str == "col desc") {
|
||||
return SORTTYPE_DESCENDING;
|
||||
}
|
||||
if (str == "asc abs" || str == "col asc abs") {
|
||||
return SORTTYPE_ASCENDING_ABS;
|
||||
}
|
||||
if (str == "desc abs" || str == "col desc abs") {
|
||||
return SORTTYPE_DESCENDING_ABS;
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "Unknown sort type string: `" << str << std::endl;
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
return SORTTYPE_DESCENDING;
|
||||
}
|
||||
|
||||
std::string
|
||||
sorttype_to_str(t_sorttype type) {
|
||||
switch (type) {
|
||||
case SORTTYPE_NONE: {
|
||||
return "none";
|
||||
} break;
|
||||
case SORTTYPE_ASCENDING: {
|
||||
return "asc";
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING: {
|
||||
return "desc";
|
||||
} break;
|
||||
case SORTTYPE_ASCENDING_ABS: {
|
||||
return "asc abs";
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING_ABS: {
|
||||
return "desc abs";
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unknown sort type");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_aggtype
|
||||
str_to_aggtype(const std::string& str) {
|
||||
if (str == "distinct count" || str == "distinctcount" || str == "distinct"
|
||||
|| str == "distinct_count") {
|
||||
return t_aggtype::AGGTYPE_DISTINCT_COUNT;
|
||||
}
|
||||
if (str == "sum") {
|
||||
return t_aggtype::AGGTYPE_SUM;
|
||||
}
|
||||
if (str == "mul") {
|
||||
return t_aggtype::AGGTYPE_MUL;
|
||||
}
|
||||
if (str == "avg" || str == "mean") {
|
||||
return t_aggtype::AGGTYPE_MEAN;
|
||||
}
|
||||
if (str == "count") {
|
||||
return t_aggtype::AGGTYPE_COUNT;
|
||||
}
|
||||
if (str == "weighted mean" || str == "weighted_mean") {
|
||||
return t_aggtype::AGGTYPE_WEIGHTED_MEAN;
|
||||
}
|
||||
if (str == "unique") {
|
||||
return t_aggtype::AGGTYPE_UNIQUE;
|
||||
}
|
||||
if (str == "any") {
|
||||
return t_aggtype::AGGTYPE_ANY;
|
||||
}
|
||||
if (str == "q1") {
|
||||
return t_aggtype::AGGTYPE_Q1;
|
||||
}
|
||||
if (str == "q3") {
|
||||
return t_aggtype::AGGTYPE_Q3;
|
||||
}
|
||||
if (str == "median") {
|
||||
return t_aggtype::AGGTYPE_MEDIAN;
|
||||
}
|
||||
if (str == "join") {
|
||||
return t_aggtype::AGGTYPE_JOIN;
|
||||
}
|
||||
if (str == "div") {
|
||||
return t_aggtype::AGGTYPE_SCALED_DIV;
|
||||
}
|
||||
if (str == "add") {
|
||||
return t_aggtype::AGGTYPE_SCALED_ADD;
|
||||
}
|
||||
if (str == "dominant") {
|
||||
return t_aggtype::AGGTYPE_DOMINANT;
|
||||
}
|
||||
if (str == "first by index" || str == "first") {
|
||||
return t_aggtype::AGGTYPE_FIRST;
|
||||
}
|
||||
if (str == "last by index") {
|
||||
return t_aggtype::AGGTYPE_LAST_BY_INDEX;
|
||||
}
|
||||
if (str == "last minus first") {
|
||||
return t_aggtype::AGGTYPE_LAST_MINUS_FIRST;
|
||||
}
|
||||
if (str == "py_agg") {
|
||||
return t_aggtype::AGGTYPE_PY_AGG;
|
||||
}
|
||||
if (str == "and") {
|
||||
return t_aggtype::AGGTYPE_AND;
|
||||
}
|
||||
if (str == "or") {
|
||||
return t_aggtype::AGGTYPE_OR;
|
||||
}
|
||||
if (str == "last" || str == "last_value") {
|
||||
return t_aggtype::AGGTYPE_LAST_VALUE;
|
||||
}
|
||||
if (str == "max_by" || str == "max by") {
|
||||
return t_aggtype::AGGTYPE_MAX_BY;
|
||||
}
|
||||
if (str == "min_by" || str == "min by") {
|
||||
return t_aggtype::AGGTYPE_MIN_BY;
|
||||
}
|
||||
if (str == "max") {
|
||||
return t_aggtype::AGGTYPE_MAX;
|
||||
}
|
||||
if (str == "min") {
|
||||
return t_aggtype::AGGTYPE_MIN;
|
||||
}
|
||||
if (str == "high" || str == "high_water_mark") {
|
||||
return t_aggtype::AGGTYPE_HIGH_WATER_MARK;
|
||||
}
|
||||
if (str == "low" || str == "low_water_mark") {
|
||||
return t_aggtype::AGGTYPE_LOW_WATER_MARK;
|
||||
}
|
||||
if (str == "high minus low") {
|
||||
return t_aggtype::AGGTYPE_HIGH_MINUS_LOW;
|
||||
}
|
||||
if (str == "sum abs" || str == "sum_abs") {
|
||||
return t_aggtype::AGGTYPE_SUM_ABS;
|
||||
}
|
||||
if (str == "abs sum" || str == "abs_sum") {
|
||||
return t_aggtype::AGGTYPE_ABS_SUM;
|
||||
}
|
||||
if (str == "sum not null" || str == "sum_not_null") {
|
||||
return t_aggtype::AGGTYPE_SUM_NOT_NULL;
|
||||
}
|
||||
if (str == "mean by count" || str == "mean_by_count") {
|
||||
return t_aggtype::AGGTYPE_MEAN_BY_COUNT;
|
||||
}
|
||||
if (str == "identity") {
|
||||
return t_aggtype::AGGTYPE_IDENTITY;
|
||||
}
|
||||
if (str == "distinct leaf" || str == "distinct_leaf") {
|
||||
return t_aggtype::AGGTYPE_DISTINCT_LEAF;
|
||||
}
|
||||
if (str == "pct sum parent" || str == "pct_sum_parent") {
|
||||
return t_aggtype::AGGTYPE_PCT_SUM_PARENT;
|
||||
}
|
||||
if (str == "pct sum total" || str == "pct_sum_total"
|
||||
|| str == "pct sum grand total" || str == "pct_sum_grand_total") {
|
||||
return t_aggtype::AGGTYPE_PCT_SUM_GRAND_TOTAL;
|
||||
}
|
||||
if (str.find("udf_combiner_") != std::string::npos) {
|
||||
return t_aggtype::AGGTYPE_UDF_COMBINER;
|
||||
}
|
||||
if (str.find("udf_reducer_") != std::string::npos) {
|
||||
return t_aggtype::AGGTYPE_UDF_REDUCER;
|
||||
}
|
||||
if (str == "var" || str == "variance") {
|
||||
return t_aggtype::AGGTYPE_VARIANCE;
|
||||
}
|
||||
if (str == "stddev" || str == "standard deviation") {
|
||||
return t_aggtype::AGGTYPE_STANDARD_DEVIATION;
|
||||
}
|
||||
if (str == "gmv") {
|
||||
return t_aggtype::AGGTYPE_GMV;
|
||||
}
|
||||
|
||||
std::stringstream ss;
|
||||
ss << "Encountered unknown aggregate operation: '" << str << "'"
|
||||
<< "\n";
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
// use any as default
|
||||
return t_aggtype::AGGTYPE_ANY;
|
||||
}
|
||||
|
||||
t_aggtype
|
||||
_get_default_aggregate(t_dtype dtype) {
|
||||
t_aggtype agg_op;
|
||||
switch (dtype) {
|
||||
case DTYPE_FLOAT64:
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT64: {
|
||||
agg_op = t_aggtype::AGGTYPE_SUM;
|
||||
} break;
|
||||
default: {
|
||||
agg_op = t_aggtype::AGGTYPE_COUNT;
|
||||
}
|
||||
}
|
||||
return agg_op;
|
||||
}
|
||||
|
||||
std::string
|
||||
_get_default_aggregate_string(t_dtype dtype) {
|
||||
std::string agg_op_str;
|
||||
switch (dtype) {
|
||||
case DTYPE_FLOAT64:
|
||||
case DTYPE_FLOAT32:
|
||||
case DTYPE_UINT8:
|
||||
case DTYPE_UINT16:
|
||||
case DTYPE_UINT32:
|
||||
case DTYPE_UINT64:
|
||||
case DTYPE_INT8:
|
||||
case DTYPE_INT16:
|
||||
case DTYPE_INT32:
|
||||
case DTYPE_INT64: {
|
||||
agg_op_str = "sum";
|
||||
} break;
|
||||
default: {
|
||||
agg_op_str = "count";
|
||||
}
|
||||
}
|
||||
return agg_op_str;
|
||||
}
|
||||
|
||||
std::string
|
||||
get_status_descr(t_status status) {
|
||||
switch (status) {
|
||||
case STATUS_INVALID: {
|
||||
return "i";
|
||||
}
|
||||
case STATUS_VALID: {
|
||||
return "v";
|
||||
}
|
||||
case STATUS_CLEAR: {
|
||||
return "c";
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected status found");
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
void
|
||||
check_init(bool init, const char* file, std::int32_t line) {
|
||||
PSP_VERBOSE_ASSERT(init, "touching uninited object");
|
||||
}
|
||||
|
||||
bool
|
||||
is_neq_transition(t_value_transition t) {
|
||||
return t > VALUE_TRANSITION_EQ_TT;
|
||||
}
|
||||
|
||||
std::string
|
||||
value_transition_to_str(t_value_transition t) {
|
||||
switch (t) {
|
||||
case VALUE_TRANSITION_EQ_FF:
|
||||
return "VALUE_TRANSITION_EQ_FF";
|
||||
case VALUE_TRANSITION_EQ_TT:
|
||||
return "VALUE_TRANSITION_EQ_TT";
|
||||
case VALUE_TRANSITION_NEQ_FT:
|
||||
return "VALUE_TRANSITION_NEQ_FT";
|
||||
case VALUE_TRANSITION_NEQ_TF:
|
||||
return "VALUE_TRANSITION_NEQ_TF";
|
||||
case VALUE_TRANSITION_NEQ_TT:
|
||||
return "VALUE_TRANSITION_NEQ_TT";
|
||||
case VALUE_TRANSITION_NEQ_TDF:
|
||||
return "VALUE_TRANSITION_NEQ_TDF";
|
||||
case VALUE_TRANSITION_NEQ_TDT:
|
||||
return "VALUE_TRANSITION_NEQ_TDT";
|
||||
case VALUE_TRANSITION_NVEQ_FT:
|
||||
return "VALUE_TRANSITION_NVEQ_FT";
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected value transition.");
|
||||
return "";
|
||||
}
|
||||
|
||||
t_uindex
|
||||
root_pidx() {
|
||||
return std::numeric_limits<t_uindex>::max();
|
||||
}
|
||||
|
||||
bool
|
||||
is_internal_colname(const std::string& c) {
|
||||
return c == "psp_";
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
t_dtype
|
||||
type_to_dtype() {
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::int64_t>() {
|
||||
return DTYPE_INT64;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::int32_t>() {
|
||||
return DTYPE_INT32;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::int16_t>() {
|
||||
return DTYPE_INT16;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::int8_t>() {
|
||||
return DTYPE_INT8;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::uint64_t>() {
|
||||
return DTYPE_UINT64;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::uint32_t>() {
|
||||
return DTYPE_UINT32;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::uint16_t>() {
|
||||
return DTYPE_UINT16;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::uint8_t>() {
|
||||
return DTYPE_UINT8;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<double>() {
|
||||
return DTYPE_FLOAT64;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<float>() {
|
||||
return DTYPE_FLOAT32;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<bool>() {
|
||||
return DTYPE_BOOL;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<t_time>() {
|
||||
return DTYPE_TIME;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<t_date>() {
|
||||
return DTYPE_DATE;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<std::string>() {
|
||||
return DTYPE_STR;
|
||||
}
|
||||
|
||||
template <>
|
||||
t_dtype
|
||||
type_to_dtype<void*>() {
|
||||
return DTYPE_OBJECT;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
namespace std {
|
||||
|
||||
void
|
||||
string_to_lower(string& str) {
|
||||
transform(str.begin(), str.end(), str.begin(), ::tolower);
|
||||
}
|
||||
|
||||
} // namespace std
|
||||
@@ -0,0 +1,26 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef __linux__
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
std::string
|
||||
get_error_str() {
|
||||
// handled by perror
|
||||
return std::string();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
@@ -0,0 +1,26 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
std::string
|
||||
get_error_str() {
|
||||
// handled by perror
|
||||
return {};
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
@@ -0,0 +1,26 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef PSP_ENABLE_WASM
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
std::string
|
||||
get_error_str() {
|
||||
// handled by perror
|
||||
return std::string();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
@@ -0,0 +1,47 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef WIN32
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
std::string
|
||||
get_error_str() {
|
||||
DWORD errid = ::GetLastError();
|
||||
|
||||
if (errid == 0) {
|
||||
return std::string();
|
||||
}
|
||||
|
||||
LPSTR buf = 0;
|
||||
|
||||
size_t size = FormatMessageA(
|
||||
FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM
|
||||
| FORMAT_MESSAGE_IGNORE_INSERTS,
|
||||
NULL,
|
||||
errid,
|
||||
MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
|
||||
(LPSTR)&buf,
|
||||
0,
|
||||
NULL
|
||||
);
|
||||
|
||||
std::string message(buf, size);
|
||||
LocalFree(buf);
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
@@ -0,0 +1,16 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/binding.h>
|
||||
using namespace perspective;
|
||||
@@ -0,0 +1,180 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include "perspective/exports.h"
|
||||
#include "perspective/server.h"
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <tsl/hopscotch_map.h>
|
||||
#include <arrow/util/thread_pool.h>
|
||||
|
||||
#if HEAP_INSTRUMENTS
|
||||
#include <emscripten/heap.h>
|
||||
|
||||
#define UNINSTRUMENTED_MALLOC(x) emscripten_builtin_malloc(x)
|
||||
#define UNINSTRUMENTED_FREE(x) emscripten_builtin_free(x)
|
||||
#else
|
||||
#define UNINSTRUMENTED_MALLOC(x) malloc(x)
|
||||
#define UNINSTRUMENTED_FREE(x) free(x)
|
||||
#endif
|
||||
|
||||
using namespace perspective::server;
|
||||
|
||||
#pragma pack(push, 1)
|
||||
struct EncodedApiResp {
|
||||
void* data;
|
||||
std::uint32_t size;
|
||||
std::uint32_t client_id;
|
||||
};
|
||||
|
||||
struct EncodedApiEntries {
|
||||
std::uint32_t size;
|
||||
EncodedApiResp* entries;
|
||||
};
|
||||
#pragma pack(pop)
|
||||
|
||||
void
|
||||
encode_api_response(
|
||||
const ProtoServerResp<std::string>& msg, EncodedApiResp* encoded
|
||||
) {
|
||||
auto* data = static_cast<char*>(UNINSTRUMENTED_MALLOC(msg.data.size()));
|
||||
std::copy(msg.data.begin(), msg.data.end(), data);
|
||||
|
||||
encoded->data = data;
|
||||
encoded->size = msg.data.size();
|
||||
encoded->client_id = msg.client_id;
|
||||
}
|
||||
|
||||
EncodedApiEntries*
|
||||
encode_api_responses(const std::vector<ProtoServerResp<std::string>>& msgs) {
|
||||
auto* encoded = static_cast<EncodedApiEntries*>(
|
||||
UNINSTRUMENTED_MALLOC(sizeof(EncodedApiEntries))
|
||||
);
|
||||
encoded->entries = static_cast<EncodedApiResp*>(
|
||||
UNINSTRUMENTED_MALLOC(sizeof(EncodedApiResp) * msgs.size())
|
||||
);
|
||||
|
||||
encoded->size = msgs.size();
|
||||
auto* encoded_mem = encoded->entries;
|
||||
for (int i = 0; i < msgs.size(); i++) {
|
||||
encode_api_response(msgs[i], &encoded_mem[i]);
|
||||
}
|
||||
|
||||
return encoded;
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
ProtoServer*
|
||||
psp_new_server(bool realtime_mode) {
|
||||
return new ProtoServer(realtime_mode);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
EncodedApiEntries*
|
||||
psp_handle_request(
|
||||
ProtoServer* server,
|
||||
std::uint32_t client_id,
|
||||
char* msg_ptr,
|
||||
std::size_t msg_len
|
||||
) {
|
||||
std::string msg(msg_ptr, msg_len);
|
||||
auto msgs = server->handle_request(client_id, msg);
|
||||
return encode_api_responses(msgs);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
EncodedApiEntries*
|
||||
psp_poll(ProtoServer* server) {
|
||||
auto responses = server->poll();
|
||||
return encode_api_responses(responses);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
std::size_t
|
||||
psp_residency_prepare(ProtoServer* server) {
|
||||
return server->residency_prepare();
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
const char*
|
||||
psp_residency_victim_fname(ProtoServer* server, std::size_t i) {
|
||||
return server->residency_victim_fname(i);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
void
|
||||
psp_residency_commit(ProtoServer* server) {
|
||||
server->residency_commit();
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
std::uint32_t
|
||||
psp_new_session(ProtoServer* server) {
|
||||
return server->new_session();
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
void
|
||||
psp_close_session(ProtoServer* server, std::uint32_t client_id) {
|
||||
server->close_session(client_id);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
std::size_t
|
||||
psp_alloc(std::size_t size) {
|
||||
// We use this to allocate stack traces for instrumentation with heap
|
||||
// profiling builds.
|
||||
auto* mem = (char*)UNINSTRUMENTED_MALLOC(size);
|
||||
return (size_t)mem;
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
void
|
||||
psp_free(void* ptr) {
|
||||
UNINSTRUMENTED_FREE(ptr);
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
void
|
||||
psp_delete_server(void* proto_server) {
|
||||
auto* server = (ProtoServer*)proto_server;
|
||||
delete server;
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
bool
|
||||
psp_is_memory64() {
|
||||
#if UINTPTR_MAX == 0xffffffff
|
||||
return false;
|
||||
#elif UINTPTR_MAX == 0xffffffffffffffff
|
||||
return true;
|
||||
#endif
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
std::int32_t
|
||||
psp_num_cpus() {
|
||||
return arrow::GetCpuThreadPoolCapacity();
|
||||
}
|
||||
|
||||
PERSPECTIVE_EXPORT
|
||||
void
|
||||
psp_set_num_cpus(std::int32_t num_cpus) {
|
||||
PSP_CHECK_ARROW_STATUS(arrow::SetCpuThreadPoolCapacity(num_cpus));
|
||||
|
||||
}
|
||||
|
||||
} // end extern "C"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,16 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/comparators.h>
|
||||
|
||||
namespace perspective {} // end namespace perspective
|
||||
@@ -0,0 +1,38 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/utils.h>
|
||||
|
||||
#ifndef WIN32
|
||||
#include <sys/mman.h>
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
namespace perspective {
|
||||
|
||||
std::pair<std::uint32_t, std::uint32_t>
|
||||
file_size_pair(t_handle h) {
|
||||
t_uindex sz = file_size(h);
|
||||
return std::pair<std::uint32_t, std::uint32_t>(upper32(sz), lower32(sz));
|
||||
}
|
||||
|
||||
t_rfmapping::t_rfmapping() = default;
|
||||
|
||||
t_rfmapping::t_rfmapping(t_handle fd, void* base, t_uindex size) :
|
||||
m_fd(fd),
|
||||
m_base(base),
|
||||
m_size(size) {}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,231 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
|
||||
#ifndef PSP_ENABLE_WASM
|
||||
#ifdef __linux__
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/raw_types.h>
|
||||
#include <perspective/utils.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
|
||||
namespace perspective {
|
||||
static void map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
);
|
||||
|
||||
t_uindex
|
||||
file_size(t_handle h) {
|
||||
struct stat st;
|
||||
t_index rcode = fstat(h, &st);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "Error in stat");
|
||||
return st.st_size;
|
||||
}
|
||||
|
||||
void
|
||||
close_file(t_handle h) {
|
||||
t_index rcode = close(h);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
|
||||
}
|
||||
|
||||
void
|
||||
flush_mapping(void* base, t_uindex len) {
|
||||
t_index rcode = msync(base, len, MS_SYNC);
|
||||
PSP_VERBOSE_ASSERT(rcode != -1, "Error in msync");
|
||||
}
|
||||
|
||||
t_rfmapping::~t_rfmapping() {
|
||||
t_index rcode = munmap(m_base, m_size);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "munmap failed.");
|
||||
|
||||
rcode = close(m_fd);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
|
||||
}
|
||||
|
||||
static void
|
||||
map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
) {
|
||||
t_file_handle fh(open(fname.c_str(), fflag, fmode));
|
||||
|
||||
PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
|
||||
|
||||
if (is_read) {
|
||||
size = file_size(fh.value());
|
||||
} else {
|
||||
t_index rcode = ftruncate(fh.value(), size);
|
||||
PSP_VERBOSE_ASSERT(rcode >= 0, "ftruncate failed.");
|
||||
}
|
||||
|
||||
void* ptr = mmap(0, size, mprot, mflag, fh.value(), 0);
|
||||
|
||||
PSP_VERBOSE_ASSERT(ptr != MAP_FAILED, "error in mmap");
|
||||
|
||||
t_handle fd = fh.value();
|
||||
fh.release();
|
||||
|
||||
out.m_fd = fd;
|
||||
out.m_base = ptr;
|
||||
out.m_size = size;
|
||||
}
|
||||
|
||||
void
|
||||
map_file_read(const std::string& fname, t_rfmapping& out) {
|
||||
map_file_internal_(
|
||||
fname,
|
||||
O_RDONLY,
|
||||
S_IRUSR,
|
||||
0, // no disposition
|
||||
PROT_READ,
|
||||
MAP_SHARED,
|
||||
true,
|
||||
0,
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
|
||||
return map_file_internal_(
|
||||
fname,
|
||||
O_RDWR | O_TRUNC | O_CREAT,
|
||||
S_IRUSR | S_IWUSR,
|
||||
0, // no disposition
|
||||
PROT_WRITE | PROT_READ,
|
||||
MAP_SHARED,
|
||||
false,
|
||||
size,
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curtime() {
|
||||
struct timespec t;
|
||||
std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "Failure in clock_gettime");
|
||||
std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
|
||||
return ns;
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
get_page_size() {
|
||||
static std::int64_t pgsize = getpagesize();
|
||||
return pgsize;
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curmem() {
|
||||
static double multiplier = getpagesize() / 1024000.;
|
||||
|
||||
struct t_statm {
|
||||
long int m_size, m_resident, m_share, m_text, m_lib, m_data, m_dt;
|
||||
};
|
||||
|
||||
t_statm result;
|
||||
|
||||
const char* statm_path = "/proc/self/statm";
|
||||
|
||||
FILE* f = fopen(statm_path, "r");
|
||||
if (!f) {
|
||||
perror(statm_path);
|
||||
abort();
|
||||
}
|
||||
|
||||
PSP_VERBOSE_ASSERT(
|
||||
fscanf(
|
||||
f,
|
||||
"%ld %ld %ld %ld %ld %ld %ld",
|
||||
&result.m_size,
|
||||
&result.m_resident,
|
||||
&result.m_share,
|
||||
&result.m_text,
|
||||
&result.m_lib,
|
||||
&result.m_data,
|
||||
&result.m_dt
|
||||
) == 7,
|
||||
"Failed to read memory size"
|
||||
);
|
||||
fclose(f);
|
||||
return result.m_resident * multiplier;
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(std::thread& thr, const std::string& name) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
auto handle = thr.native_handle();
|
||||
pthread_setname_np(handle, name.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(const std::string& name) {
|
||||
// prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
rmfile(const std::string& fname) {
|
||||
unlink(fname.c_str());
|
||||
}
|
||||
|
||||
void
|
||||
launch_proc(const std::string& cmdline) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
std::string
|
||||
cwd() {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return "";
|
||||
}
|
||||
|
||||
void*
|
||||
psp_dbg_malloc(size_t size) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
psp_dbg_free(void* mem) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,234 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifndef PSP_ENABLE_WASM
|
||||
#ifdef __APPLE__
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/raw_types.h>
|
||||
#include <perspective/utils.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <cstdio>
|
||||
|
||||
namespace perspective {
|
||||
static void map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
);
|
||||
|
||||
t_uindex
|
||||
file_size(t_handle h) {
|
||||
struct stat st;
|
||||
t_index rcode = fstat(h, &st);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "Error in stat");
|
||||
return st.st_size;
|
||||
}
|
||||
|
||||
void
|
||||
close_file(t_handle h) {
|
||||
t_index rcode = close(h);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
|
||||
}
|
||||
|
||||
void
|
||||
flush_mapping(void* base, t_uindex len) {
|
||||
t_index rcode = msync(base, len, MS_SYNC);
|
||||
PSP_VERBOSE_ASSERT(rcode, != -1, "Error in msync");
|
||||
}
|
||||
|
||||
t_rfmapping::~t_rfmapping() {
|
||||
t_index rcode = munmap(m_base, m_size);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "munmap failed.");
|
||||
|
||||
rcode = close(m_fd);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
|
||||
}
|
||||
|
||||
static void
|
||||
map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
) {
|
||||
t_file_handle fh(open(fname.c_str(), fflag, fmode));
|
||||
|
||||
PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
|
||||
|
||||
if (is_read) {
|
||||
size = file_size(fh.value());
|
||||
} else {
|
||||
t_index rcode = ftruncate(fh.value(), size);
|
||||
PSP_VERBOSE_ASSERT(rcode, >= 0, "ftruncate failed.");
|
||||
}
|
||||
|
||||
void* ptr = mmap(nullptr, size, mprot, mflag, fh.value(), 0);
|
||||
|
||||
PSP_VERBOSE_ASSERT(ptr, != MAP_FAILED, "error in mmap");
|
||||
|
||||
t_handle fd = fh.value();
|
||||
fh.release();
|
||||
|
||||
out.m_fd = fd;
|
||||
out.m_base = ptr;
|
||||
out.m_size = size;
|
||||
}
|
||||
|
||||
void
|
||||
map_file_read(const std::string& fname, t_rfmapping& out) {
|
||||
map_file_internal_(
|
||||
fname,
|
||||
O_RDONLY,
|
||||
S_IRUSR,
|
||||
0, // no disposition
|
||||
PROT_READ,
|
||||
MAP_SHARED,
|
||||
true,
|
||||
0,
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
|
||||
return map_file_internal_(
|
||||
fname,
|
||||
O_RDWR | O_TRUNC | O_CREAT,
|
||||
S_IRUSR | S_IWUSR,
|
||||
0, // no disposition
|
||||
PROT_WRITE | PROT_READ,
|
||||
MAP_SHARED,
|
||||
false,
|
||||
size,
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curtime() {
|
||||
// CLOCK_MONOTONIC and clock_gettime are not implemented in OSX < 10.12,
|
||||
// and it breaks builds on conda-forge. Because this method is only called
|
||||
// in logging and trace functions that are not called in the codebase,
|
||||
// deprecate this method for OSX and return 0.
|
||||
|
||||
// struct timespec t;
|
||||
// std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
// PSP_VERBOSE_ASSERT(rcode, == 0, "Failure in clock_gettime");
|
||||
// std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
get_page_size() {
|
||||
static std::int64_t pgsize = getpagesize();
|
||||
return pgsize;
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curmem() {
|
||||
static double multiplier = getpagesize() / 1024000.;
|
||||
|
||||
struct t_statm {
|
||||
long int m_size, m_resident, m_share, m_text, m_lib, m_data, m_dt;
|
||||
};
|
||||
|
||||
t_statm result;
|
||||
|
||||
const char* statm_path = "/proc/self/statm";
|
||||
|
||||
FILE* f = fopen(statm_path, "r");
|
||||
if (f == nullptr) {
|
||||
perror(statm_path);
|
||||
abort();
|
||||
}
|
||||
|
||||
PSP_VERBOSE_ASSERT(
|
||||
fscanf(
|
||||
f,
|
||||
"%ld %ld %ld %ld %ld %ld %ld",
|
||||
&result.m_size,
|
||||
&result.m_resident,
|
||||
&result.m_share,
|
||||
&result.m_text,
|
||||
&result.m_lib,
|
||||
&result.m_data,
|
||||
&result.m_dt
|
||||
) == 7,
|
||||
"Failed to read memory size"
|
||||
);
|
||||
fclose(f);
|
||||
return result.m_resident * multiplier;
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(std::thread& thr, const std::string& name) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
pthread_setname_np(name.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(const std::string& name) {
|
||||
// prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
rmfile(const std::string& fname) {
|
||||
unlink(fname.c_str());
|
||||
}
|
||||
|
||||
void
|
||||
launch_proc(const std::string& cmdline) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
std::string
|
||||
cwd() {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return "";
|
||||
}
|
||||
|
||||
void*
|
||||
psp_dbg_malloc(size_t size) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
psp_dbg_free(void* mem) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,168 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef PSP_ENABLE_WASM
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/opfs.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/raw_types.h>
|
||||
#include <perspective/utils.h>
|
||||
#include <fcntl.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/mman.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/stat.h>
|
||||
#include <sys/syscall.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
|
||||
namespace perspective {
|
||||
static void map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
);
|
||||
|
||||
t_uindex
|
||||
file_size(t_handle h) {
|
||||
struct stat st {};
|
||||
if (fstat(h, &st) != 0) {
|
||||
return 0;
|
||||
}
|
||||
return static_cast<t_uindex>(st.st_size);
|
||||
}
|
||||
|
||||
void
|
||||
close_file(t_handle h) {
|
||||
// On WASM (NO_FILESYSTEM) disk stores are backed by the OPFS/node-fs bridge
|
||||
// keyed by filename, not real fds — `create_file()` returns a sentinel (1).
|
||||
// Closing it issued `fd_close(1)` (stdout) on every disk-column free; never
|
||||
// touch the std streams (0/1/2). Real fds (none under NO_FILESYSTEM) close.
|
||||
if (h > 2) {
|
||||
close(h);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
flush_mapping(void* base, t_uindex len) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
t_rfmapping::~t_rfmapping() {}
|
||||
|
||||
static void
|
||||
map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
map_file_read(const std::string& fname, t_rfmapping& out) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curtime() {
|
||||
// CLOCK_MONOTONIC and clock_gettime are not implemented in OSX < 10.12,
|
||||
// and it breaks builds on conda-forge. Because this method is only called
|
||||
// in logging and trace functions that are not called in the codebase,
|
||||
// deprecate this method for OSX and return 0.
|
||||
|
||||
// struct timespec t;
|
||||
// std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
|
||||
// PSP_VERBOSE_ASSERT(rcode, == 0, "Failure in clock_gettime");
|
||||
// std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
get_page_size() {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
psp_curmem() {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(std::thread& thr, const std::string& name) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(const std::string& name) {
|
||||
// prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
void
|
||||
rmfile(const std::string& fname) {
|
||||
#ifdef PSP_HAS_OPFS_BRIDGE
|
||||
// Close the kept-open OPFS sync access handle and delete the backing file.
|
||||
// (`unlink` is a no-op under `NO_FILESYSTEM`, and would leak the open handle
|
||||
// the residency bridge holds for this file.)
|
||||
psp_opfs_remove(fname.c_str());
|
||||
#else
|
||||
unlink(fname.c_str());
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
launch_proc(const std::string& cmdline) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
std::string
|
||||
cwd() {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return "";
|
||||
}
|
||||
|
||||
void*
|
||||
psp_dbg_malloc(size_t size) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
psp_dbg_free(void* mem) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,312 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifndef PSP_ENABLE_WASM
|
||||
#ifdef WIN32
|
||||
#include <perspective/portable.h>
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/raw_types.h>
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/utils.h>
|
||||
#include <cstdio>
|
||||
#include <psapi.h>
|
||||
#include <cstdint>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
static void map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
);
|
||||
|
||||
t_uindex
|
||||
file_size(t_handle h) {
|
||||
LARGE_INTEGER sz;
|
||||
BOOL rb = GetFileSizeEx(h, &sz);
|
||||
PSP_VERBOSE_ASSERT(rb != 0, "Error getting filesize.");
|
||||
return sz.QuadPart;
|
||||
}
|
||||
|
||||
void
|
||||
close_file(t_handle h) {
|
||||
BOOL rb = CloseHandle(h);
|
||||
PSP_VERBOSE_ASSERT(rb != 0, "Error closing file");
|
||||
}
|
||||
|
||||
void
|
||||
flush_mapping(void* base, t_uindex len) {
|
||||
BOOL rb = FlushViewOfFile(base, size_t(len));
|
||||
PSP_VERBOSE_ASSERT(rb != 0, "Error flushing view");
|
||||
}
|
||||
|
||||
t_rfmapping::~t_rfmapping() {
|
||||
BOOL rb = UnmapViewOfFile(m_base);
|
||||
PSP_VERBOSE_ASSERT(rb != 0, "Error unmapping view");
|
||||
|
||||
close_file(m_fd);
|
||||
}
|
||||
|
||||
static void
|
||||
map_file_internal_(
|
||||
const std::string& fname,
|
||||
t_fflag fflag,
|
||||
t_fflag fmode,
|
||||
t_fflag creation_disposition,
|
||||
t_fflag mprot,
|
||||
t_fflag mflag,
|
||||
bool is_read,
|
||||
t_uindex size,
|
||||
t_rfmapping& out
|
||||
) {
|
||||
t_file_handle fh(CreateFile(
|
||||
fname.c_str(),
|
||||
fflag,
|
||||
FILE_SHARE_READ,
|
||||
0, // security
|
||||
creation_disposition,
|
||||
FILE_ATTRIBUTE_NORMAL,
|
||||
0 // template file
|
||||
));
|
||||
|
||||
PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
|
||||
|
||||
if (is_read) {
|
||||
size = file_size(fh.value());
|
||||
} else {
|
||||
LARGE_INTEGER sz;
|
||||
sz.QuadPart = size;
|
||||
|
||||
auto rb = SetFilePointerEx(fh.value(), sz, 0, FILE_BEGIN);
|
||||
PSP_VERBOSE_ASSERT(rb, "Error setting fpointer");
|
||||
|
||||
rb = SetEndOfFile(fh.value());
|
||||
PSP_VERBOSE_ASSERT(rb, "Error setting eof");
|
||||
}
|
||||
|
||||
t_handle m = CreateFileMapping(
|
||||
fh.value(),
|
||||
0, // default security
|
||||
mprot,
|
||||
upper32(size),
|
||||
lower32(size),
|
||||
0 // anonymous mapping
|
||||
);
|
||||
|
||||
PSP_VERBOSE_ASSERT(m != 0, "Error creating filemapping");
|
||||
|
||||
void* ptr = MapViewOfFile(
|
||||
m,
|
||||
mflag,
|
||||
0, // 0 offset
|
||||
0, // 0 offset
|
||||
0 // entire file
|
||||
);
|
||||
|
||||
PSP_VERBOSE_ASSERT(ptr != 0, "Error mapping file");
|
||||
|
||||
// Handle is safe to close once view is
|
||||
// created for it
|
||||
CloseHandle(m);
|
||||
|
||||
t_handle fd = fh.value();
|
||||
fh.release();
|
||||
|
||||
out.m_fd = fd;
|
||||
out.m_base = ptr;
|
||||
out.m_size = size;
|
||||
}
|
||||
|
||||
void
|
||||
map_file_read(const std::string& fname, t_rfmapping& out) {
|
||||
map_file_internal_(
|
||||
fname,
|
||||
GENERIC_READ,
|
||||
0, // unused
|
||||
OPEN_EXISTING,
|
||||
PAGE_READONLY,
|
||||
FILE_MAP_READ,
|
||||
true,
|
||||
0,
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
|
||||
map_file_internal_(
|
||||
fname,
|
||||
GENERIC_READ | GENERIC_WRITE,
|
||||
0, // unused
|
||||
CREATE_ALWAYS,
|
||||
PAGE_READWRITE,
|
||||
FILE_MAP_WRITE,
|
||||
false,
|
||||
static_cast<size_t>(size),
|
||||
out
|
||||
);
|
||||
}
|
||||
|
||||
int64_t
|
||||
psp_curtime() {
|
||||
return GetTickCount() * static_cast<int64_t>(1000000);
|
||||
}
|
||||
|
||||
int64_t
|
||||
psp_curmem() {
|
||||
PROCESS_MEMORY_COUNTERS mem;
|
||||
GetProcessMemoryInfo(GetCurrentProcess(), &mem, sizeof(mem));
|
||||
return mem.WorkingSetSize / 1024;
|
||||
}
|
||||
|
||||
#pragma pack(push, 8)
|
||||
typedef struct t_win_thrstruct {
|
||||
DWORD dwType; // Must be 0x1000.
|
||||
LPCSTR szName; // Pointer to name (in user addr space).
|
||||
DWORD dwThreadID; // Thread ID (-1=caller thread).
|
||||
DWORD dwFlags; // Reserved for future use, must be zero.
|
||||
} THREADNAME_INFO;
|
||||
#pragma pack(pop)
|
||||
|
||||
static void
|
||||
set_thread_name_win(uint32_t thrid, const std::string& name) {
|
||||
const DWORD MS_VC_EXCEPTION = 0x406D1388;
|
||||
t_win_thrstruct thrstruct;
|
||||
thrstruct.dwType = 0x1000;
|
||||
thrstruct.szName = name.c_str();
|
||||
thrstruct.dwThreadID = thrid;
|
||||
thrstruct.dwFlags = 0;
|
||||
|
||||
SUPPRESS_WARNINGS_VC(6320 6322)
|
||||
__try {
|
||||
RaiseException(
|
||||
MS_VC_EXCEPTION,
|
||||
0,
|
||||
sizeof(thrstruct) / sizeof(ULONG_PTR),
|
||||
(ULONG_PTR*)&thrstruct
|
||||
);
|
||||
} __except (EXCEPTION_EXECUTE_HANDLER) {}
|
||||
RESTORE_WARNINGS_VC()
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(std::thread& thr, const std::string& name) {
|
||||
auto thrid = ::GetThreadId(static_cast<HANDLE>(thr.native_handle()));
|
||||
set_thread_name_win(thrid, name);
|
||||
}
|
||||
|
||||
void
|
||||
set_thread_name(const std::string& name) {
|
||||
set_thread_name_win(GetCurrentThreadId(), name);
|
||||
}
|
||||
|
||||
void
|
||||
rmfile(const std::string& fname) {
|
||||
DeleteFile(fname.c_str());
|
||||
}
|
||||
|
||||
void
|
||||
launch_proc(const std::string& cmdline) {
|
||||
STARTUPINFO si;
|
||||
PROCESS_INFORMATION pi;
|
||||
|
||||
ZeroMemory(&si, sizeof(si));
|
||||
si.cb = sizeof(si);
|
||||
ZeroMemory(&pi, sizeof(pi));
|
||||
|
||||
if (!CreateProcess(
|
||||
NULL, // No module name (use command line)
|
||||
const_cast<char*>(cmdline.c_str()), // Command line
|
||||
NULL, // Process handle not inheritable
|
||||
NULL, // Thread handle not inheritable
|
||||
FALSE, // Set handle inheritance to FALSE
|
||||
0, // No creation flags
|
||||
NULL, // Use parent's environment block
|
||||
NULL, // Use parent's starting directory
|
||||
&si, // Pointer to STARTUPINFO structure
|
||||
&pi
|
||||
) // Pointer to PROCESS_INFORMATION structure
|
||||
) {
|
||||
std::cout << "CreateProcess failed => " << GetLastError() << "\n";
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait until child process exits.
|
||||
WaitForSingleObject(pi.hProcess, INFINITE);
|
||||
|
||||
// Close process and thread handles.
|
||||
CloseHandle(pi.hProcess);
|
||||
CloseHandle(pi.hThread);
|
||||
}
|
||||
|
||||
std::string
|
||||
cwd() {
|
||||
char path[FILENAME_MAX];
|
||||
auto rc = GetCurrentDirectory(FILENAME_MAX, path);
|
||||
PSP_VERBOSE_ASSERT(rc, "Error in cwd");
|
||||
return std::string(path);
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_page_size() {
|
||||
SYSTEM_INFO info;
|
||||
GetSystemInfo(&info);
|
||||
return info.dwPageSize;
|
||||
}
|
||||
|
||||
void*
|
||||
psp_dbg_malloc(size_t size) {
|
||||
SYSTEM_INFO sys_info;
|
||||
GetSystemInfo(&sys_info);
|
||||
auto page = 2 * sys_info.dwPageSize;
|
||||
assert((page & (static_cast<size_t>(page) - 1)) == 0);
|
||||
auto rounded_size =
|
||||
(size + static_cast<size_t>(page) - 1) & (-static_cast<size_t>(page));
|
||||
BYTE* start = (BYTE*)VirtualAlloc(
|
||||
NULL, rounded_size + page, MEM_COMMIT, PAGE_READWRITE
|
||||
);
|
||||
DWORD old_protect;
|
||||
BOOL res =
|
||||
VirtualProtect(start + rounded_size, page, PAGE_NOACCESS, &old_protect);
|
||||
assert(res);
|
||||
UNREFERENCED_PARAMETER(res);
|
||||
return start + (rounded_size - size);
|
||||
}
|
||||
|
||||
void
|
||||
psp_dbg_free(void* mem) {
|
||||
VirtualFree(mem, 0, MEM_RELEASE);
|
||||
}
|
||||
|
||||
void*
|
||||
psp_page_aligned_malloc(int64_t size) {
|
||||
return _aligned_malloc(
|
||||
static_cast<size_t>(size), static_cast<size_t>(get_page_size())
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
psp_page_aligned_free(void* mem) {
|
||||
_aligned_free(mem);
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,687 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/computed_expression.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
computed_function::bucket t_computed_expression_parser::BUCKET_FN =
|
||||
computed_function::bucket();
|
||||
|
||||
computed_function::hour_of_day t_computed_expression_parser::HOUR_OF_DAY_FN =
|
||||
computed_function::hour_of_day();
|
||||
|
||||
computed_function::percent_of t_computed_expression_parser::PERCENT_OF_FN =
|
||||
computed_function::percent_of();
|
||||
|
||||
computed_function::inrange_fn t_computed_expression_parser::INRANGE_FN =
|
||||
computed_function::inrange_fn();
|
||||
|
||||
computed_function::min_fn t_computed_expression_parser::MIN_FN =
|
||||
computed_function::min_fn();
|
||||
|
||||
computed_function::max_fn t_computed_expression_parser::MAX_FN =
|
||||
computed_function::max_fn();
|
||||
|
||||
computed_function::coalesce t_computed_expression_parser::COALESCE_FN =
|
||||
computed_function::coalesce();
|
||||
|
||||
computed_function::contains t_computed_expression_parser::CONTAINS_FN =
|
||||
computed_function::contains();
|
||||
|
||||
computed_function::diff3 t_computed_expression_parser::diff3 =
|
||||
computed_function::diff3();
|
||||
|
||||
computed_function::norm3 t_computed_expression_parser::norm3 =
|
||||
computed_function::norm3();
|
||||
|
||||
computed_function::cross_product3 t_computed_expression_parser::cross_product3 =
|
||||
computed_function::cross_product3();
|
||||
|
||||
computed_function::dot_product3 t_computed_expression_parser::dot_product3 =
|
||||
computed_function::dot_product3();
|
||||
|
||||
computed_function::length t_computed_expression_parser::LENGTH_FN =
|
||||
computed_function::length();
|
||||
|
||||
computed_function::is_null t_computed_expression_parser::IS_NULL_FN =
|
||||
computed_function::is_null();
|
||||
|
||||
computed_function::is_not_null t_computed_expression_parser::IS_NOT_NULL_FN =
|
||||
computed_function::is_not_null();
|
||||
|
||||
computed_function::to_integer t_computed_expression_parser::TO_INTEGER_FN =
|
||||
computed_function::to_integer();
|
||||
|
||||
computed_function::to_float t_computed_expression_parser::TO_FLOAT_FN =
|
||||
computed_function::to_float();
|
||||
|
||||
computed_function::to_boolean t_computed_expression_parser::TO_BOOLEAN_FN =
|
||||
computed_function::to_boolean();
|
||||
|
||||
computed_function::make_date t_computed_expression_parser::MAKE_DATE_FN =
|
||||
computed_function::make_date();
|
||||
|
||||
computed_function::make_datetime
|
||||
t_computed_expression_parser::MAKE_DATETIME_FN =
|
||||
computed_function::make_datetime();
|
||||
|
||||
computed_function::random t_computed_expression_parser::RANDOM_FN =
|
||||
computed_function::random();
|
||||
|
||||
t_tscalar t_computed_expression_parser::TRUE_SCALAR = mktscalar(true);
|
||||
|
||||
t_tscalar t_computed_expression_parser::FALSE_SCALAR = mktscalar(false);
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* t_computed_expression
|
||||
*/
|
||||
|
||||
t_computed_expression::t_computed_expression(
|
||||
std::string expression_alias,
|
||||
std::string expression_string,
|
||||
std::string parsed_expression_string,
|
||||
const std::vector<std::pair<std::string, std::string>>& column_ids,
|
||||
t_dtype dtype
|
||||
) :
|
||||
m_expression_alias(std::move(expression_alias)),
|
||||
m_expression_string(std::move(expression_string)),
|
||||
m_parsed_expression_string(std::move(parsed_expression_string)),
|
||||
m_column_ids(column_ids),
|
||||
m_dtype(dtype) {}
|
||||
|
||||
struct t_computed_expression_cache {
|
||||
t_computed_expression_cache() = default;
|
||||
t_computed_expression_cache(const t_computed_expression_cache&) = delete;
|
||||
t_computed_expression_cache& operator=(const t_computed_expression_cache&) =
|
||||
delete;
|
||||
t_computed_expression_cache(t_computed_expression_cache&&) = delete;
|
||||
t_computed_expression_cache& operator=(t_computed_expression_cache&&) =
|
||||
delete;
|
||||
|
||||
std::shared_ptr<t_data_table> m_current_source_table;
|
||||
t_uindex m_row_idx{0};
|
||||
std::vector<std::pair<std::string, t_tscalar>> m_values;
|
||||
std::vector<std::shared_ptr<t_column>> m_columns;
|
||||
std::vector<t_dtype> m_input_dtypes;
|
||||
exprtk::symbol_table<t_tscalar> m_sym_table;
|
||||
exprtk::expression<t_tscalar> m_expr;
|
||||
std::unique_ptr<t_computed_function_store> m_function_store;
|
||||
};
|
||||
|
||||
t_computed_expression::~t_computed_expression() = default;
|
||||
|
||||
void
|
||||
t_computed_expression::compute(
|
||||
const std::shared_ptr<t_data_table>& source_table,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
const std::shared_ptr<t_data_table>& destination_table,
|
||||
t_expression_vocab& vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) const {
|
||||
auto num_input_columns = m_column_ids.size();
|
||||
|
||||
bool needs_build = !m_cache;
|
||||
if (!needs_build) {
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
const std::string& column_name = m_column_ids[cidx].second;
|
||||
if (source_table->get_column(column_name)->get_dtype()
|
||||
!= m_cache->m_input_dtypes[cidx]) {
|
||||
needs_build = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (needs_build) {
|
||||
m_cache = std::make_unique<t_computed_expression_cache>();
|
||||
auto& cache = *m_cache;
|
||||
|
||||
cache.m_current_source_table = source_table;
|
||||
cache.m_sym_table.add_constants(); // pi, infinity, etc.
|
||||
|
||||
// is_type_validator = false: we are computing values, not type-checking.
|
||||
cache.m_function_store = std::make_unique<t_computed_function_store>(
|
||||
vocab,
|
||||
regex_mapping,
|
||||
false,
|
||||
cache.m_current_source_table,
|
||||
pkey_map,
|
||||
cache.m_row_idx
|
||||
);
|
||||
cache.m_function_store->register_computed_functions(cache.m_sym_table);
|
||||
|
||||
// Size exactly once; the symbol table binds a T* into each
|
||||
// m_values[cidx].second, so this storage must never be reallocated.
|
||||
cache.m_values.resize(num_input_columns);
|
||||
cache.m_columns.resize(num_input_columns);
|
||||
cache.m_input_dtypes.resize(num_input_columns);
|
||||
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
const std::string& column_id = m_column_ids[cidx].first;
|
||||
const std::string& column_name = m_column_ids[cidx].second;
|
||||
t_dtype dtype = source_table->get_column(column_name)->get_dtype();
|
||||
|
||||
cache.m_input_dtypes[cidx] = dtype;
|
||||
|
||||
t_tscalar rval;
|
||||
rval.clear();
|
||||
rval.m_type = dtype;
|
||||
|
||||
cache.m_values[cidx] =
|
||||
std::pair<std::string, t_tscalar>(column_id, rval);
|
||||
cache.m_sym_table.add_variable(
|
||||
column_id, cache.m_values[cidx].second
|
||||
);
|
||||
}
|
||||
|
||||
cache.m_expr.register_symbol_table(cache.m_sym_table);
|
||||
|
||||
// Load-bearing for compile-once: exprtk constant-folds an all-literal
|
||||
// function call (e.g. intern('x'), concat('a','b')) to a literal node at
|
||||
// compile time ONLY when the function reports no side effects. Our
|
||||
// functions inherit the igeneric_function/ifunction default
|
||||
// has_side_effects() == true and never call disable_has_side_effects(),
|
||||
// so such calls are not folded and re-evaluate on every value(). If a
|
||||
// future exprtk bump flips that default, the first call's result would
|
||||
// be frozen into the cached AST -- revisit this cache if so.
|
||||
if (!m_computed_expression_parser.m_parser->compile(
|
||||
m_parsed_expression_string, cache.m_expr
|
||||
)) {
|
||||
std::stringstream ss;
|
||||
ss << "[t_computed_expression::compute] Failed to parse "
|
||||
"expression: `"
|
||||
<< m_parsed_expression_string << "`, failed with error: "
|
||||
<< m_computed_expression_parser.m_parser->error() << '\n';
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
}
|
||||
}
|
||||
|
||||
auto& cache = *m_cache;
|
||||
|
||||
// Re-point the per-call inputs the compiled function objects read through
|
||||
// (the source table differs across the master/flattened/delta/prev/current
|
||||
// tables within a single update), and re-fetch this call's source columns.
|
||||
cache.m_current_source_table = source_table;
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
cache.m_columns[cidx] =
|
||||
source_table->get_column(m_column_ids[cidx].second);
|
||||
}
|
||||
|
||||
// create or get output column using m_expression_alias
|
||||
auto output_column =
|
||||
destination_table->add_column_sptr(m_expression_alias, m_dtype, true);
|
||||
auto num_rows = source_table->size();
|
||||
output_column->reserve(num_rows);
|
||||
|
||||
for (t_uindex ridx = 0; ridx < num_rows; ++ridx) {
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
cache.m_values[cidx].second.set(
|
||||
cache.m_columns[cidx]->get_scalar(ridx)
|
||||
);
|
||||
}
|
||||
cache.m_row_idx = ridx;
|
||||
|
||||
t_tscalar value = cache.m_expr.value();
|
||||
|
||||
if (!value.is_valid() || value.is_none()) {
|
||||
output_column->clear(ridx);
|
||||
continue;
|
||||
}
|
||||
|
||||
output_column->set_scalar(ridx, value);
|
||||
}
|
||||
|
||||
// order()'s accumulator must still be reset at the end of every call.
|
||||
cache.m_function_store->clear_computed_function_state();
|
||||
};
|
||||
|
||||
const std::string&
|
||||
t_computed_expression::get_expression_alias() const {
|
||||
return m_expression_alias;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_computed_expression::get_expression_string() const {
|
||||
return m_expression_string;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_computed_expression::get_parsed_expression_string() const {
|
||||
return m_parsed_expression_string;
|
||||
}
|
||||
|
||||
const std::vector<std::pair<std::string, std::string>>&
|
||||
t_computed_expression::get_column_ids() const {
|
||||
return m_column_ids;
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_computed_expression::get_dtype() const {
|
||||
return m_dtype;
|
||||
}
|
||||
|
||||
/******************************************************************************
|
||||
*
|
||||
* t_computed_expression_parser
|
||||
*/
|
||||
|
||||
t_computed_expression_parser::t_computed_expression_parser() {
|
||||
// Change ExprTk's default compilation options to only check for correctness
|
||||
// of brackets and sequences. ExprTk defaults will replace "true" and
|
||||
// "false" with 1 and 0, which we don't want. Using the tokens "true" and
|
||||
// "false" will raise a syntax error, which is the correct behavior.
|
||||
|
||||
m_parser = std::make_shared<exprtk::parser<t_tscalar>>(
|
||||
exprtk::parser<t_tscalar>::settings_t::e_joiner
|
||||
+ exprtk::parser<t_tscalar>::settings_t::e_numeric_check
|
||||
+ exprtk::parser<t_tscalar>::settings_t::e_bracket_check
|
||||
+ exprtk::parser<t_tscalar>::settings_t::e_sequence_check
|
||||
// exprtk::parser<t_tscalar>::settings_t::e_commutative_check;
|
||||
// exprtk::parser<t_tscalar>::settings_t::e_strength_reduction;
|
||||
);
|
||||
|
||||
m_parser->settings()
|
||||
.disable_control_structure(
|
||||
exprtk::parser<t_tscalar>::settings_store::e_ctrl_repeat_loop
|
||||
)
|
||||
.disable_base_function(
|
||||
exprtk::parser<t_tscalar>::settings_store::e_bf_inrange
|
||||
)
|
||||
.disable_base_function(
|
||||
exprtk::parser<t_tscalar>::settings_store::e_bf_min
|
||||
)
|
||||
.disable_base_function(
|
||||
exprtk::parser<t_tscalar>::settings_store::e_bf_max
|
||||
);
|
||||
}
|
||||
|
||||
std::shared_ptr<t_computed_expression>
|
||||
t_computed_expression_parser::precompute(
|
||||
const std::string& expression_alias,
|
||||
const std::string& expression_string,
|
||||
const std::string& parsed_expression_string,
|
||||
const std::vector<std::pair<std::string, std::string>>& column_ids,
|
||||
const std::shared_ptr<t_data_table>& source_table,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
const std::shared_ptr<t_schema>& schema,
|
||||
t_expression_vocab& vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) const {
|
||||
exprtk::symbol_table<t_tscalar> sym_table;
|
||||
sym_table.add_constants();
|
||||
|
||||
t_uindex row_idx = 0;
|
||||
|
||||
// Create a function store, with is_type_validator set to true as we are
|
||||
// just getting the output types.
|
||||
t_computed_function_store function_store(
|
||||
vocab, regex_mapping, true, source_table, pkey_map, row_idx
|
||||
);
|
||||
function_store.register_computed_functions(sym_table);
|
||||
|
||||
std::vector<t_tscalar> values;
|
||||
|
||||
auto num_input_columns = column_ids.size();
|
||||
values.resize(num_input_columns);
|
||||
|
||||
// Add a new scalar of the input column type to the symtable, which
|
||||
// will use the scalar to validate the output type of the expression.
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
const std::string& column_id = column_ids[cidx].first;
|
||||
const std::string& column_name = column_ids[cidx].second;
|
||||
|
||||
t_tscalar rval;
|
||||
rval.clear();
|
||||
rval.m_type = schema->get_dtype(column_name);
|
||||
|
||||
// If we are accessing string columns, the scalar CANNOT have a
|
||||
// nullptr as a string - use the empty string that we interned
|
||||
// in the gnode's expression vocab in t_gnode::init() at idx 0
|
||||
if (rval.m_type == DTYPE_STR) {
|
||||
rval.set(vocab.get_empty_string());
|
||||
rval.m_status = STATUS_INVALID;
|
||||
}
|
||||
|
||||
values[cidx] = rval;
|
||||
|
||||
sym_table.add_variable(column_id, values[cidx]);
|
||||
}
|
||||
|
||||
exprtk::expression<t_tscalar> expr_definition;
|
||||
expr_definition.register_symbol_table(sym_table);
|
||||
|
||||
if (!m_parser->compile(parsed_expression_string, expr_definition)) {
|
||||
std::stringstream ss;
|
||||
ss << "[t_computed_expression_parser::precompute] Failed to parse "
|
||||
"expression: `"
|
||||
<< parsed_expression_string
|
||||
<< "`, failed with error: " << m_parser->error() << '\n';
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
}
|
||||
|
||||
t_tscalar v = expr_definition.value();
|
||||
function_store.clear_computed_function_state();
|
||||
|
||||
return std::make_shared<t_computed_expression>(
|
||||
expression_alias,
|
||||
expression_string,
|
||||
parsed_expression_string,
|
||||
column_ids,
|
||||
v.get_dtype()
|
||||
);
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_computed_expression_parser::get_dtype(
|
||||
const std::string& expression_alias,
|
||||
const std::string& expression_string,
|
||||
const std::string& parsed_expression_string,
|
||||
const std::vector<std::pair<std::string, std::string>>& column_ids,
|
||||
const std::shared_ptr<t_data_table>& source_table,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
const t_schema& schema,
|
||||
t_expression_error& error,
|
||||
t_expression_vocab& vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) const {
|
||||
exprtk::symbol_table<t_tscalar> sym_table;
|
||||
sym_table.add_constants();
|
||||
|
||||
std::vector<t_tscalar> values;
|
||||
|
||||
t_uindex row_idx = 0;
|
||||
|
||||
// Create a function store, with is_type_validator set to true as we are
|
||||
// just validating the output types.
|
||||
t_computed_function_store function_store(
|
||||
vocab, regex_mapping, true, source_table, pkey_map, row_idx
|
||||
);
|
||||
function_store.register_computed_functions(sym_table);
|
||||
|
||||
auto num_input_columns = column_ids.size();
|
||||
values.resize(num_input_columns);
|
||||
|
||||
// Add a new scalar of the input column type to the symtable, which
|
||||
// will use the scalar to validate the output type of the expression.
|
||||
for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
|
||||
const std::string& column_id = column_ids[cidx].first;
|
||||
const std::string& column_name = column_ids[cidx].second;
|
||||
|
||||
if (!schema.has_column(column_name)) {
|
||||
error.m_error_message =
|
||||
("Value Error - Input column \"" + column_name
|
||||
+ "\" does not exist.");
|
||||
error.m_line = 0;
|
||||
error.m_column = 0;
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
t_tscalar rval;
|
||||
|
||||
// clear() is important as it sets the status to valid and the
|
||||
// underlying union value to 0, which prevents any reading of
|
||||
// uninitialized values out of the union.
|
||||
rval.clear();
|
||||
rval.m_type = schema.get_dtype(column_name);
|
||||
|
||||
// If we are accessing string columns, the scalar CANNOT have a
|
||||
// nullptr as a string - use the empty string that we interned
|
||||
// in the gnode's expression vocab in t_gnode::init() at idx 0
|
||||
if (rval.m_type == DTYPE_STR) {
|
||||
rval.set(vocab.get_empty_string());
|
||||
rval.m_status = STATUS_INVALID;
|
||||
}
|
||||
|
||||
values[cidx] = rval;
|
||||
|
||||
sym_table.add_variable(column_id, values[cidx]);
|
||||
}
|
||||
|
||||
exprtk::expression<t_tscalar> expr_definition;
|
||||
expr_definition.register_symbol_table(sym_table);
|
||||
|
||||
if (!m_parser->compile(parsed_expression_string, expr_definition)) {
|
||||
// Error count should always be above 0 if there is a compile error -
|
||||
// We simply take the first error and return it.
|
||||
if (m_parser->error_count() > 0) {
|
||||
auto parser_error = m_parser->get_error(0);
|
||||
|
||||
// Given an error object and an expression, `update_error` maps the
|
||||
// error to a line and column number inside the expression.
|
||||
exprtk::parser_error::update_error(
|
||||
parser_error, parsed_expression_string
|
||||
);
|
||||
|
||||
// Take the error message and strip the ExprTk error code
|
||||
std::string error_message(parser_error.diagnostic);
|
||||
|
||||
// strip the Exprtk error codes such as "ERR001 -"
|
||||
error.m_error_message =
|
||||
error_message.substr(error_message.find("- ") + 2);
|
||||
|
||||
error.m_line = parser_error.line_no;
|
||||
error.m_column = parser_error.column_no;
|
||||
} else {
|
||||
// If for whatever reason the error count is at 0, output a generic
|
||||
// parser error so that we report back a valid error object.
|
||||
error.m_error_message = "Parser Error";
|
||||
error.m_line = 0;
|
||||
error.m_column = 0;
|
||||
}
|
||||
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
t_tscalar v = expr_definition.value();
|
||||
t_dtype dtype = v.get_dtype();
|
||||
|
||||
function_store.clear_computed_function_state();
|
||||
|
||||
if (v.m_status == STATUS_CLEAR || dtype == DTYPE_NONE) {
|
||||
error.m_error_message =
|
||||
"Type Error - inputs do not resolve to a valid expression.";
|
||||
error.m_line = 0;
|
||||
error.m_column = 0;
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
return dtype;
|
||||
}
|
||||
|
||||
t_validated_expression_map::t_validated_expression_map() = default;
|
||||
|
||||
void
|
||||
t_validated_expression_map::add_expression(
|
||||
const std::string& expression_alias, const std::string& type_string
|
||||
) {
|
||||
// If the expression is in the error map, it should be removed as it is now
|
||||
// valid. This can happen when validating multiple expressions with the same
|
||||
// alias where the first instance is invalid, and the next occurence is
|
||||
// valid, and so on. This maintains the property that an expression will
|
||||
// exist in the schema OR the error map.
|
||||
auto error_iter = m_expression_errors.find(expression_alias);
|
||||
|
||||
if (error_iter != m_expression_errors.end()) {
|
||||
m_expression_errors.erase(error_iter);
|
||||
}
|
||||
|
||||
m_expression_schema[expression_alias] = type_string;
|
||||
}
|
||||
|
||||
void
|
||||
t_validated_expression_map::add_error(
|
||||
const std::string& expression_alias, t_expression_error& error
|
||||
) {
|
||||
// If the expression is in the schema, it should be removed as it is now
|
||||
// invalid. This can happen when validating multiple expressions with the
|
||||
// same alias where the first instance is valid, and the next occurence is
|
||||
// invalid, and so on. This maintains the property that an expression will
|
||||
// exist in the schema OR the error map.
|
||||
auto schema_iter = m_expression_schema.find(expression_alias);
|
||||
|
||||
if (schema_iter != m_expression_schema.end()) {
|
||||
m_expression_schema.erase(schema_iter);
|
||||
}
|
||||
|
||||
m_expression_errors[expression_alias] = error;
|
||||
}
|
||||
|
||||
std::map<std::string, std::string>
|
||||
t_validated_expression_map::get_expression_schema() const {
|
||||
return m_expression_schema;
|
||||
}
|
||||
|
||||
std::map<std::string, t_expression_error>
|
||||
t_validated_expression_map::get_expression_errors() const {
|
||||
return m_expression_errors;
|
||||
}
|
||||
|
||||
t_computed_function_store::t_computed_function_store(
|
||||
t_expression_vocab& vocab,
|
||||
t_regex_mapping& regex_mapping,
|
||||
bool is_type_validator,
|
||||
const std::shared_ptr<t_data_table>& source_table,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
t_uindex& row_idx
|
||||
) :
|
||||
m_day_of_week_fn(computed_function::day_of_week(vocab, is_type_validator)),
|
||||
m_month_of_year_fn(
|
||||
computed_function::month_of_year(vocab, is_type_validator)
|
||||
),
|
||||
m_intern_fn(computed_function::intern(vocab, is_type_validator)),
|
||||
m_concat_fn(computed_function::concat(vocab, is_type_validator)),
|
||||
m_order_fn(computed_function::order(is_type_validator)),
|
||||
m_upper_fn(computed_function::upper(vocab, is_type_validator)),
|
||||
m_lower_fn(computed_function::lower(vocab, is_type_validator)),
|
||||
m_to_string_fn(computed_function::to_string(vocab, is_type_validator)),
|
||||
m_match_fn(computed_function::match(regex_mapping)),
|
||||
m_match_all_fn(computed_function::match_all(regex_mapping)),
|
||||
m_search_fn(
|
||||
computed_function::search(vocab, regex_mapping, is_type_validator)
|
||||
),
|
||||
m_indexof_fn(computed_function::indexof(regex_mapping)),
|
||||
m_substring_fn(computed_function::substring(vocab, is_type_validator)),
|
||||
m_replace_fn(
|
||||
computed_function::replace(vocab, regex_mapping, is_type_validator)
|
||||
),
|
||||
m_replace_all_fn(
|
||||
computed_function::replace_all(vocab, regex_mapping, is_type_validator)
|
||||
),
|
||||
m_index_fn(computed_function::index(pkey_map, source_table, row_idx)),
|
||||
m_col_fn(
|
||||
computed_function::col(vocab, is_type_validator, source_table, row_idx)
|
||||
),
|
||||
m_vlookup_fn(computed_function::vlookup(
|
||||
vocab, is_type_validator, source_table, row_idx
|
||||
)) {}
|
||||
|
||||
void
|
||||
t_computed_function_store::register_computed_functions(
|
||||
exprtk::symbol_table<t_tscalar>& sym_table
|
||||
) {
|
||||
// General/numeric functions
|
||||
sym_table.add_function("bucket", t_computed_expression_parser::BUCKET_FN);
|
||||
sym_table.add_reserved_function(
|
||||
"inrange", t_computed_expression_parser::INRANGE_FN
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"min", t_computed_expression_parser::MIN_FN
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"max", t_computed_expression_parser::MAX_FN
|
||||
);
|
||||
sym_table.add_function(
|
||||
"coalesce", t_computed_expression_parser::COALESCE_FN
|
||||
);
|
||||
sym_table.add_function(
|
||||
"contains", t_computed_expression_parser::CONTAINS_FN
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"diff3", t_computed_expression_parser::diff3
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"norm3", t_computed_expression_parser::norm3
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"cross_product3", t_computed_expression_parser::cross_product3
|
||||
);
|
||||
sym_table.add_reserved_function(
|
||||
"dot_product3", t_computed_expression_parser::dot_product3
|
||||
);
|
||||
sym_table.add_function(
|
||||
"percent_of", t_computed_expression_parser::PERCENT_OF_FN
|
||||
);
|
||||
sym_table.add_function("is_null", t_computed_expression_parser::IS_NULL_FN);
|
||||
sym_table.add_function(
|
||||
"is_not_null", t_computed_expression_parser::IS_NOT_NULL_FN
|
||||
);
|
||||
sym_table.add_function("random", t_computed_expression_parser::RANDOM_FN);
|
||||
|
||||
// Date/datetime functions
|
||||
sym_table.add_function(
|
||||
"hour_of_day", t_computed_expression_parser::HOUR_OF_DAY_FN
|
||||
);
|
||||
sym_table.add_function("day_of_week", m_day_of_week_fn);
|
||||
sym_table.add_function("month_of_year", m_month_of_year_fn);
|
||||
sym_table.add_function("today", computed_function::today);
|
||||
sym_table.add_function("now", computed_function::now);
|
||||
|
||||
// String functions
|
||||
sym_table.add_function("intern", m_intern_fn);
|
||||
sym_table.add_function("concat", m_concat_fn);
|
||||
sym_table.add_function("order", m_order_fn);
|
||||
sym_table.add_function("upper", m_upper_fn);
|
||||
sym_table.add_function("lower", m_lower_fn);
|
||||
sym_table.add_function("length", t_computed_expression_parser::LENGTH_FN);
|
||||
|
||||
// Type conversion functions
|
||||
sym_table.add_function(
|
||||
"integer", t_computed_expression_parser::TO_INTEGER_FN
|
||||
);
|
||||
sym_table.add_function("float", t_computed_expression_parser::TO_FLOAT_FN);
|
||||
sym_table.add_function(
|
||||
"boolean", t_computed_expression_parser::TO_BOOLEAN_FN
|
||||
);
|
||||
sym_table.add_function("date", t_computed_expression_parser::MAKE_DATE_FN);
|
||||
sym_table.add_function(
|
||||
"datetime", t_computed_expression_parser::MAKE_DATETIME_FN
|
||||
);
|
||||
sym_table.add_function("string", m_to_string_fn);
|
||||
|
||||
// Regex functions
|
||||
sym_table.add_function("match", m_match_fn);
|
||||
sym_table.add_function("match_all", m_match_all_fn);
|
||||
sym_table.add_function("search", m_search_fn);
|
||||
sym_table.add_function("indexof", m_indexof_fn);
|
||||
sym_table.add_function("substring", m_substring_fn);
|
||||
sym_table.add_function("replace", m_replace_fn);
|
||||
sym_table.add_function("replace_all", m_replace_all_fn);
|
||||
sym_table.add_function("index", m_index_fn);
|
||||
sym_table.add_function("col", m_col_fn);
|
||||
sym_table.add_function("vlookup", m_vlookup_fn);
|
||||
|
||||
// And scalar constants
|
||||
sym_table.add_constant("True", t_computed_expression_parser::TRUE_SCALAR);
|
||||
sym_table.add_constant("False", t_computed_expression_parser::FALSE_SCALAR);
|
||||
}
|
||||
|
||||
void
|
||||
t_computed_function_store::clear_computed_function_state() {
|
||||
m_order_fn.clear_order_map();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,491 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/config.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
// t_ctxunit
|
||||
t_config::t_config(const std::vector<std::string>& detail_columns) :
|
||||
t_config(detail_columns, {}, FILTER_OP_AND, {}) {}
|
||||
|
||||
// t_ctx0
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& detail_columns,
|
||||
const std::vector<t_fterm>& fterms,
|
||||
t_filter_op combiner,
|
||||
const std::vector<std::shared_ptr<t_computed_expression>>& expressions
|
||||
) :
|
||||
m_detail_columns(detail_columns),
|
||||
m_fterms(fterms),
|
||||
m_expressions(expressions),
|
||||
m_combiner(combiner),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
setup(m_detail_columns);
|
||||
m_is_trivial_config = m_row_pivots.empty() && m_col_pivots.empty()
|
||||
&& m_sortby.empty() && m_sortspecs.empty() && m_col_sortspecs.empty()
|
||||
&& m_detail_columns.empty() && m_fterms.empty()
|
||||
&& m_expressions.empty();
|
||||
}
|
||||
|
||||
// t_ctx1
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots,
|
||||
const std::vector<t_aggspec>& aggregates,
|
||||
const std::vector<t_fterm>& fterms,
|
||||
t_filter_op combiner,
|
||||
const std::vector<std::shared_ptr<t_computed_expression>>& expressions
|
||||
) :
|
||||
m_aggregates(aggregates),
|
||||
m_fterms(fterms),
|
||||
m_expressions(expressions),
|
||||
m_combiner(combiner),
|
||||
m_is_trivial_config(false),
|
||||
m_totals(TOTALS_BEFORE),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
for (const auto& p : row_pivots) {
|
||||
m_row_pivots.emplace_back(p);
|
||||
}
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
// t_ctx2
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots,
|
||||
const std::vector<std::string>& col_pivots,
|
||||
const std::vector<t_aggspec>& aggregates,
|
||||
const t_totals totals,
|
||||
const std::vector<t_fterm>& fterms,
|
||||
t_filter_op combiner,
|
||||
const std::vector<std::shared_ptr<t_computed_expression>>& expressions,
|
||||
bool column_only
|
||||
) :
|
||||
m_aggregates(aggregates),
|
||||
m_fterms(fterms),
|
||||
m_expressions(expressions),
|
||||
m_combiner(combiner),
|
||||
m_column_only(column_only),
|
||||
m_is_trivial_config(false),
|
||||
m_totals(totals),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
for (const auto& p : row_pivots) {
|
||||
m_row_pivots.emplace_back(p);
|
||||
}
|
||||
for (const auto& p : col_pivots) {
|
||||
m_col_pivots.emplace_back(p);
|
||||
}
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
// Constructors used for C++ tests
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots,
|
||||
const std::vector<std::string>& col_pivots,
|
||||
const std::vector<t_aggspec>& aggregates
|
||||
) :
|
||||
t_config(
|
||||
row_pivots, col_pivots, aggregates, TOTALS_HIDDEN, FILTER_OP_AND, {}
|
||||
) {}
|
||||
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots,
|
||||
const std::vector<std::string>& col_pivots,
|
||||
const std::vector<t_aggspec>& aggregates,
|
||||
const t_totals totals,
|
||||
t_filter_op combiner,
|
||||
const std::vector<t_fterm>& fterms
|
||||
) :
|
||||
m_aggregates(aggregates),
|
||||
m_fterms(fterms),
|
||||
m_combiner(combiner),
|
||||
m_is_trivial_config(false),
|
||||
m_totals(totals),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
for (const auto& p : row_pivots) {
|
||||
m_row_pivots.emplace_back(p);
|
||||
}
|
||||
|
||||
for (const auto& p : col_pivots) {
|
||||
m_col_pivots.emplace_back(p);
|
||||
}
|
||||
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
t_config::t_config(
|
||||
const std::vector<t_pivot>& row_pivots,
|
||||
const std::vector<t_aggspec>& aggregates
|
||||
) :
|
||||
m_row_pivots(row_pivots),
|
||||
m_aggregates(aggregates),
|
||||
m_is_trivial_config(false),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots,
|
||||
const std::vector<t_aggspec>& aggregates
|
||||
) :
|
||||
m_aggregates(aggregates),
|
||||
m_combiner(FILTER_OP_AND),
|
||||
m_is_trivial_config(false),
|
||||
m_totals(TOTALS_BEFORE),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
for (const auto& p : row_pivots) {
|
||||
m_row_pivots.emplace_back(p);
|
||||
}
|
||||
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
t_config::t_config(
|
||||
const std::vector<std::string>& row_pivots, const t_aggspec& agg
|
||||
) :
|
||||
m_aggregates(std::vector<t_aggspec>{agg}),
|
||||
m_combiner(FILTER_OP_AND),
|
||||
m_is_trivial_config(false),
|
||||
m_totals(TOTALS_BEFORE),
|
||||
m_fmode(FMODE_SIMPLE_CLAUSES) {
|
||||
for (const auto& p : row_pivots) {
|
||||
m_row_pivots.emplace_back(p);
|
||||
}
|
||||
|
||||
setup(
|
||||
m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
|
||||
);
|
||||
}
|
||||
|
||||
t_config::t_config() = default;
|
||||
|
||||
void
|
||||
t_config::setup(const std::vector<std::string>& detail_columns) {
|
||||
t_index count = 0;
|
||||
for (const auto& detail_column : detail_columns) {
|
||||
m_detail_colmap[detail_column] = count;
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_config::setup(
|
||||
const std::vector<std::string>& detail_columns,
|
||||
const std::vector<std::string>& sort_pivot,
|
||||
const std::vector<std::string>& sort_pivot_by
|
||||
) {
|
||||
t_index count = 0;
|
||||
for (const auto& detail_column : detail_columns) {
|
||||
m_detail_colmap[detail_column] = count;
|
||||
count++;
|
||||
}
|
||||
|
||||
m_has_pkey_agg = false;
|
||||
|
||||
for (const auto& m_aggregate : m_aggregates) {
|
||||
switch (m_aggregate.agg()) {
|
||||
case AGGTYPE_AND:
|
||||
case AGGTYPE_OR:
|
||||
case AGGTYPE_ANY:
|
||||
case AGGTYPE_FIRST:
|
||||
case AGGTYPE_LAST_BY_INDEX:
|
||||
case AGGTYPE_LAST_MINUS_FIRST:
|
||||
case AGGTYPE_HIGH_MINUS_LOW:
|
||||
case AGGTYPE_MEAN:
|
||||
case AGGTYPE_WEIGHTED_MEAN:
|
||||
case AGGTYPE_UNIQUE:
|
||||
case AGGTYPE_Q1:
|
||||
case AGGTYPE_Q3:
|
||||
case AGGTYPE_MEDIAN:
|
||||
case AGGTYPE_JOIN:
|
||||
case AGGTYPE_DOMINANT:
|
||||
case AGGTYPE_PY_AGG:
|
||||
case AGGTYPE_MIN:
|
||||
case AGGTYPE_MAX:
|
||||
case AGGTYPE_MAX_BY:
|
||||
case AGGTYPE_MIN_BY:
|
||||
case AGGTYPE_SUM_NOT_NULL:
|
||||
case AGGTYPE_SUM_ABS:
|
||||
case AGGTYPE_ABS_SUM:
|
||||
case AGGTYPE_GMV:
|
||||
case AGGTYPE_MUL:
|
||||
case AGGTYPE_DISTINCT_COUNT:
|
||||
case AGGTYPE_DISTINCT_LEAF:
|
||||
case AGGTYPE_VARIANCE:
|
||||
case AGGTYPE_STANDARD_DEVIATION:
|
||||
m_has_pkey_agg = true;
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (m_has_pkey_agg) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
for (t_index idx = 0, loop_end = sort_pivot.size(); idx < loop_end; ++idx) {
|
||||
m_sortby[sort_pivot[idx]] = sort_pivot_by[idx];
|
||||
}
|
||||
|
||||
populate_sortby(m_row_pivots);
|
||||
populate_sortby(m_col_pivots);
|
||||
}
|
||||
|
||||
bool
|
||||
t_config::is_trivial_config() const {
|
||||
return m_is_trivial_config;
|
||||
}
|
||||
|
||||
void
|
||||
t_config::populate_sortby(const std::vector<t_pivot>& pivots) {
|
||||
for (const auto& pivot : pivots) {
|
||||
PSP_VERBOSE_ASSERT(
|
||||
pivot.mode() == PIVOT_MODE_NORMAL,
|
||||
"Only normal pivots supported for now"
|
||||
);
|
||||
const std::string& pstr = pivot.colname();
|
||||
if (m_sortby.find(pstr) == m_sortby.end()) {
|
||||
m_sortby[pstr] = pstr;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_index
|
||||
t_config::get_colidx(const std::string& colname) const {
|
||||
auto iter = m_detail_colmap.find(colname);
|
||||
if (iter == m_detail_colmap.end()) {
|
||||
return INVALID_INDEX;
|
||||
}
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_config<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_config::get_num_aggregates() const {
|
||||
return m_aggregates.size();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_config::get_num_columns() const {
|
||||
return m_detail_columns.size();
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::col_at(t_uindex idx) const {
|
||||
if (idx >= m_detail_columns.size()) {
|
||||
return "";
|
||||
}
|
||||
return m_detail_columns[idx];
|
||||
}
|
||||
|
||||
bool
|
||||
t_config::has_pkey_agg() const {
|
||||
return m_has_pkey_agg;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::get_totals_string() const {
|
||||
switch (m_totals) {
|
||||
case TOTALS_BEFORE: {
|
||||
return "before";
|
||||
} break;
|
||||
case TOTALS_HIDDEN: {
|
||||
return "hidden";
|
||||
} break;
|
||||
case TOTALS_AFTER: {
|
||||
return "after";
|
||||
} break;
|
||||
default: {
|
||||
return "INVALID_TOTALS";
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::get_sort_by(const std::string& pivot) const {
|
||||
std::string rval;
|
||||
auto iter = m_sortby.find(pivot);
|
||||
|
||||
if (iter == m_sortby.end()) {
|
||||
return pivot;
|
||||
}
|
||||
rval = iter->second;
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_config::validate_colidx(t_index idx) const {
|
||||
return idx >= 0 && idx < static_cast<t_index>(get_num_columns());
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_config::get_column_names() const {
|
||||
return m_detail_columns;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_config::get_num_rpivots() const {
|
||||
return m_row_pivots.size();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_config::get_num_cpivots() const {
|
||||
return m_col_pivots.size();
|
||||
}
|
||||
|
||||
bool
|
||||
t_config::is_column_only() const {
|
||||
return m_column_only;
|
||||
}
|
||||
|
||||
const std::vector<t_pivot>&
|
||||
t_config::get_row_pivots() const {
|
||||
return m_row_pivots;
|
||||
}
|
||||
|
||||
const std::vector<t_pivot>&
|
||||
t_config::get_column_pivots() const {
|
||||
return m_col_pivots;
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string, std::string>>
|
||||
t_config::get_sortby_pairs() const {
|
||||
std::vector<std::pair<std::string, std::string>> rval(m_sortby.size());
|
||||
t_uindex idx = 0;
|
||||
for (const auto& iter : m_sortby) {
|
||||
rval[idx].first = iter.first;
|
||||
rval[idx].second = iter.second;
|
||||
++idx;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
const std::vector<t_sortspec>&
|
||||
t_config::get_sortspecs() const {
|
||||
return m_sortspecs;
|
||||
}
|
||||
|
||||
const std::vector<t_sortspec>&
|
||||
t_config::get_col_sortspecs() const {
|
||||
return m_col_sortspecs;
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>&
|
||||
t_config::get_aggregates() const {
|
||||
return m_aggregates;
|
||||
}
|
||||
|
||||
bool
|
||||
t_config::has_filters() const {
|
||||
switch (m_fmode) {
|
||||
case FMODE_SIMPLE_CLAUSES: {
|
||||
return !m_fterms.empty();
|
||||
} break;
|
||||
default: {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
const std::vector<t_fterm>&
|
||||
t_config::get_fterms() const {
|
||||
return m_fterms;
|
||||
}
|
||||
|
||||
std::vector<std::shared_ptr<t_computed_expression>>
|
||||
t_config::get_expressions() const {
|
||||
return m_expressions;
|
||||
}
|
||||
|
||||
t_filter_op
|
||||
t_config::get_combiner() const {
|
||||
return m_combiner;
|
||||
}
|
||||
|
||||
t_totals
|
||||
t_config::get_totals() const {
|
||||
return m_totals;
|
||||
}
|
||||
|
||||
std::vector<t_pivot>
|
||||
t_config::get_pivots() const {
|
||||
std::vector<t_pivot> rval = m_row_pivots;
|
||||
for (const auto& piv : m_col_pivots) {
|
||||
rval.push_back(piv);
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::get_parent_pkey_column() const {
|
||||
return m_parent_pkey_column;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::get_child_pkey_column() const {
|
||||
return m_child_pkey_column;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_config::get_grouping_label_column() const {
|
||||
return m_grouping_label_column;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::unity_get_column_name(t_uindex idx) const {
|
||||
if (m_aggregates.empty()) {
|
||||
if (idx >= m_detail_columns.size()) {
|
||||
return "";
|
||||
}
|
||||
return m_detail_columns[idx];
|
||||
}
|
||||
|
||||
return m_aggregates[idx % m_aggregates.size()].name();
|
||||
}
|
||||
|
||||
std::string
|
||||
t_config::unity_get_column_display_name(t_uindex idx) const {
|
||||
if (m_aggregates.empty()) {
|
||||
if (idx >= m_detail_columns.size()) {
|
||||
return "";
|
||||
}
|
||||
return m_detail_columns[idx];
|
||||
}
|
||||
|
||||
return m_aggregates[idx % m_aggregates.size()].disp_name();
|
||||
}
|
||||
|
||||
t_fmode
|
||||
t_config::get_fmode() const {
|
||||
return m_fmode;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,16 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/context_base.h>
|
||||
|
||||
namespace perspective {} // end namespace perspective
|
||||
@@ -0,0 +1,939 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/context_grouped_pkey.h>
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/get_data_extents.h>
|
||||
#include <perspective/extract_aggregate.h>
|
||||
#include <perspective/filter.h>
|
||||
#include <perspective/sparse_tree.h>
|
||||
#include <perspective/tree_context_common.h>
|
||||
#include <perspective/sparse_tree_node.h>
|
||||
#include <perspective/traversal.h>
|
||||
#include <perspective/env_vars.h>
|
||||
#include <perspective/filter_utils.h>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include <tsl/hopscotch_set.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_ctx_grouped_pkey::t_ctx_grouped_pkey() : m_depth(0), m_depth_set(false) {}
|
||||
|
||||
t_ctx_grouped_pkey::t_ctx_grouped_pkey(
|
||||
const t_schema& schema, const t_config& config
|
||||
) :
|
||||
m_depth(0),
|
||||
m_depth_set(false) {
|
||||
PSP_COMPLAIN_AND_ABORT("Not Implemented");
|
||||
}
|
||||
|
||||
t_ctx_grouped_pkey::~t_ctx_grouped_pkey() = default;
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::init() {
|
||||
auto pivots = m_config.get_row_pivots();
|
||||
m_tree = std::make_shared<t_stree>(
|
||||
pivots, m_config.get_aggregates(), m_schema, m_config
|
||||
);
|
||||
m_tree->init();
|
||||
m_traversal = std::make_shared<t_traversal>(m_tree);
|
||||
|
||||
// Each context stores its own expression columns in separate
|
||||
// `t_data_table`s so that each context's expressions are isolated
|
||||
// and do not affect other contexts when they are calculated.
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
m_expression_tables = std::make_shared<t_expression_tables>(
|
||||
expressions, m_config.get_backing_store()
|
||||
);
|
||||
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_expression_tables>
|
||||
t_ctx_grouped_pkey::get_expression_tables() const {
|
||||
return m_expression_tables;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx_grouped_pkey::get_row_count() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_traversal->size();
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx_grouped_pkey::get_column_count() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_config.get_num_columns() + 1;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx_grouped_pkey::open(t_header header, t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return open(idx);
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx_grouped_pkey::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_ctx_grouped_pkey<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx_grouped_pkey::open(t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
// If we manually open/close a node, stop automatically expanding
|
||||
m_depth_set = false;
|
||||
m_depth = 0;
|
||||
|
||||
if (idx >= t_index(m_traversal->size())) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_index retval = m_traversal->expand_node(m_sortby, idx);
|
||||
m_rows_changed = (retval > 0);
|
||||
return retval;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx_grouped_pkey::close(t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
// If we manually open/close a node, stop automatically expanding
|
||||
m_depth_set = false;
|
||||
m_depth = 0;
|
||||
|
||||
if (idx >= t_index(m_traversal->size())) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_index retval = m_traversal->collapse_node(idx);
|
||||
m_rows_changed = (retval > 0);
|
||||
return retval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::get_data(
|
||||
t_index start_row, t_index end_row, t_index start_col, t_index end_col
|
||||
) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex ctx_nrows = get_row_count();
|
||||
t_uindex ncols = get_column_count();
|
||||
auto ext = sanitize_get_data_extents(
|
||||
ctx_nrows, ncols, start_row, end_row, start_col, end_col
|
||||
);
|
||||
|
||||
t_index nrows = ext.m_erow - ext.m_srow;
|
||||
t_index stride = ext.m_ecol - ext.m_scol;
|
||||
std::vector<t_tscalar> values(nrows * stride);
|
||||
std::vector<t_tscalar> tmpvalues(nrows * ncols);
|
||||
|
||||
std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
|
||||
|
||||
if (aggcols.empty()) {
|
||||
return values;
|
||||
}
|
||||
|
||||
auto* aggtable = m_tree->get_aggtable();
|
||||
|
||||
for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
// resolve by column index (== aggidx); skips the per-iteration
|
||||
// name->index map lookup + temp string and the schema deep-copy.
|
||||
aggcols[aggidx] = aggtable->_get_const_column(aggidx);
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
|
||||
|
||||
const std::string& grouping_label_col =
|
||||
m_config.get_grouping_label_column();
|
||||
|
||||
for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
|
||||
t_index nidx = m_traversal->get_tree_index(ridx);
|
||||
t_index pnidx = m_tree->get_parent_idx(nidx);
|
||||
|
||||
t_uindex agg_ridx = m_tree->get_aggidx(nidx);
|
||||
t_index agg_pridx =
|
||||
pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
|
||||
|
||||
t_tscalar tree_value = m_tree->get_value(nidx);
|
||||
|
||||
if (m_has_label && ridx > 0) {
|
||||
// Get pkey
|
||||
auto iters = m_tree->get_pkeys_for_leaf(nidx);
|
||||
tree_value.set(
|
||||
get_value_from_gstate(grouping_label_col, iters.first->m_pkey)
|
||||
);
|
||||
}
|
||||
|
||||
tmpvalues[(ridx - ext.m_srow) * ncols] = tree_value;
|
||||
|
||||
for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
t_tscalar value = extract_aggregate(
|
||||
aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
|
||||
);
|
||||
|
||||
tmpvalues[(ridx - ext.m_srow) * ncols + 1 + aggidx].set(value);
|
||||
}
|
||||
}
|
||||
|
||||
for (auto ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
|
||||
for (auto cidx = ext.m_scol; cidx < ext.m_ecol; ++cidx) {
|
||||
auto insert_idx = (ridx - ext.m_srow) * stride + cidx - ext.m_scol;
|
||||
auto src_idx = (ridx - ext.m_srow) * ncols + cidx;
|
||||
values[insert_idx].set(tmpvalues[src_idx]);
|
||||
}
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::notify(
|
||||
const t_data_table& flattened,
|
||||
const t_data_table& delta,
|
||||
const t_data_table& prev,
|
||||
const t_data_table& current,
|
||||
const t_data_table& transitions,
|
||||
const t_data_table& existed
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
rebuild();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::step_begin() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
reset_step_state();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::step_end() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
sort_by(m_sortby);
|
||||
if (m_depth_set) {
|
||||
set_depth(m_depth);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<t_aggspec>
|
||||
t_ctx_grouped_pkey::get_aggregates() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_config.get_aggregates();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::get_row_path(t_index idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return ctx_get_path(m_tree, m_traversal, idx);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::reset_sortby() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
m_sortby = std::vector<t_sortspec>();
|
||||
}
|
||||
|
||||
std::vector<t_path>
|
||||
t_ctx_grouped_pkey::get_expansion_state() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return ctx_get_expansion_state(m_tree, m_traversal);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::set_expansion_state(const std::vector<t_path>& paths) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
ctx_set_expansion_state(*this, HEADER_ROW, m_tree, m_traversal, paths);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::expand_path(const std::vector<t_tscalar>& path) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
ctx_expand_path(*this, HEADER_ROW, m_tree, m_traversal, path);
|
||||
}
|
||||
|
||||
t_stree*
|
||||
t_ctx_grouped_pkey::_get_tree() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_tree.get();
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_ctx_grouped_pkey::get_tree_value(t_index nidx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_tree->get_value(nidx);
|
||||
}
|
||||
|
||||
std::vector<t_ftreenode>
|
||||
t_ctx_grouped_pkey::get_flattened_tree(t_index idx, t_depth stop_depth) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return ctx_get_flattened_tree(
|
||||
idx, stop_depth, *(m_traversal), m_config, m_sortby
|
||||
);
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_traversal>
|
||||
t_ctx_grouped_pkey::get_traversal() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_traversal;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::sort_by(const std::vector<t_sortspec>& sortby) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
m_sortby = sortby;
|
||||
if (m_sortby.empty()) {
|
||||
return;
|
||||
}
|
||||
m_traversal->sort_by(m_config, sortby, *this);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::set_depth(t_depth depth) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_depth final_depth =
|
||||
std::min<t_depth>(m_config.get_num_rpivots() - 1, depth);
|
||||
t_index retval = 0;
|
||||
retval = m_traversal->set_depth(m_sortby, final_depth);
|
||||
m_rows_changed = (retval > 0);
|
||||
m_depth = depth;
|
||||
m_depth_set = true;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::get_pkeys(
|
||||
const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (!m_traversal->validate_cells(cells)) {
|
||||
std::vector<t_tscalar> rval;
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar> rval;
|
||||
|
||||
tsl::hopscotch_set<t_uindex> seen;
|
||||
|
||||
for (const auto& c : cells) {
|
||||
auto ptidx = m_traversal->get_tree_index(c.first);
|
||||
|
||||
if (static_cast<t_uindex>(ptidx) == static_cast<t_uindex>(-1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (seen.find(ptidx) == seen.end()) {
|
||||
auto iters = m_tree->get_pkeys_for_leaf(ptidx);
|
||||
for (auto iter = iters.first; iter != iters.second; ++iter) {
|
||||
rval.push_back(iter->m_pkey);
|
||||
}
|
||||
seen.insert(ptidx);
|
||||
}
|
||||
|
||||
auto desc = m_tree->get_descendents(ptidx);
|
||||
|
||||
for (auto d : desc) {
|
||||
if (seen.find(d) != seen.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto iters = m_tree->get_pkeys_for_leaf(d);
|
||||
for (auto iter = iters.first; iter != iters.second; ++iter) {
|
||||
rval.push_back(iter->m_pkey);
|
||||
}
|
||||
seen.insert(d);
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::set_feature_state(t_ctx_feature feature, bool state) {
|
||||
m_features[feature] = state;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::set_alerts_enabled(bool enabled_state) {
|
||||
m_features[CTX_FEAT_ALERT] = enabled_state;
|
||||
m_tree->set_alerts_enabled(enabled_state);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::set_deltas_enabled(bool enabled_state) {
|
||||
m_features[CTX_FEAT_DELTA] = enabled_state;
|
||||
m_tree->set_deltas_enabled(enabled_state);
|
||||
}
|
||||
|
||||
t_stepdelta
|
||||
t_ctx_grouped_pkey::get_step_delta(t_index bidx, t_index eidx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
bidx = std::min(bidx, t_index(m_traversal->size()));
|
||||
eidx = std::min(eidx, t_index(m_traversal->size()));
|
||||
|
||||
t_stepdelta rval(
|
||||
m_rows_changed, m_columns_changed, get_cell_delta(bidx, eidx)
|
||||
);
|
||||
m_tree->clear_deltas();
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_cellupd>
|
||||
t_ctx_grouped_pkey::get_cell_delta(t_index bidx, t_index eidx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
eidx = std::min(eidx, t_index(m_traversal->size()));
|
||||
std::vector<t_cellupd> rval;
|
||||
const auto& deltas = m_tree->get_deltas();
|
||||
for (t_index idx = bidx; idx < eidx; ++idx) {
|
||||
t_index ptidx = m_traversal->get_tree_index(idx);
|
||||
auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
|
||||
for (auto iter = iterators.first; iter != iterators.second; ++iter) {
|
||||
rval.emplace_back(
|
||||
idx, iter->m_aggidx + 1, iter->m_old_value, iter->m_new_value
|
||||
);
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::reset(bool reset_expressions) {
|
||||
auto pivots = m_config.get_row_pivots();
|
||||
m_tree = std::make_shared<t_stree>(
|
||||
pivots, m_config.get_aggregates(), m_schema, m_config
|
||||
);
|
||||
m_tree->init();
|
||||
m_tree->set_deltas_enabled(get_feature_state(CTX_FEAT_DELTA));
|
||||
m_traversal = std::make_shared<t_traversal>(m_tree);
|
||||
|
||||
if (reset_expressions) {
|
||||
m_expression_tables->reset();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::reset_step_state() {
|
||||
m_rows_changed = false;
|
||||
m_columns_changed = false;
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_ctx_grouped_pkey.reset_step_state " << repr() << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<t_stree*>
|
||||
t_ctx_grouped_pkey::get_trees() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
std::vector<t_stree*> rval(1);
|
||||
rval[0] = m_tree.get();
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx_grouped_pkey::has_deltas() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename DATA_T>
|
||||
void
|
||||
rebuild_helper(t_column* /*unused*/) {}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::rebuild() {
|
||||
auto tbl = m_gstate->get_pkeyed_table();
|
||||
|
||||
if (m_config.has_filters()) {
|
||||
auto mask = filter_table_for_config(*tbl, m_config);
|
||||
tbl = tbl->clone(mask);
|
||||
}
|
||||
|
||||
std::string child_col_name = m_config.get_child_pkey_column();
|
||||
|
||||
std::shared_ptr<const t_column> child_col_sptr =
|
||||
tbl->get_const_column(child_col_name);
|
||||
|
||||
const t_column* child_col = child_col_sptr.get();
|
||||
auto expansion_state = get_expansion_state();
|
||||
|
||||
std::sort(
|
||||
expansion_state.begin(),
|
||||
expansion_state.end(),
|
||||
[](const t_path& a, const t_path& b) {
|
||||
return a.path().size() < b.path().size();
|
||||
}
|
||||
);
|
||||
|
||||
for (auto& p : expansion_state) {
|
||||
std::reverse(p.path().begin(), p.path().end());
|
||||
}
|
||||
|
||||
reset();
|
||||
|
||||
t_uindex nrows = child_col->size();
|
||||
|
||||
if (nrows == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
struct t_datum {
|
||||
t_uindex m_pidx;
|
||||
t_tscalar m_parent;
|
||||
t_tscalar m_child;
|
||||
t_tscalar m_pkey;
|
||||
bool m_is_rchild;
|
||||
t_uindex m_idx;
|
||||
};
|
||||
|
||||
const auto* sortby_col =
|
||||
tbl->_get_const_column(m_config.get_sort_by(child_col_name));
|
||||
|
||||
const auto* parent_col =
|
||||
tbl->_get_const_column(m_config.get_parent_pkey_column());
|
||||
|
||||
const auto* pkey_col = tbl->_get_const_column("psp_pkey");
|
||||
|
||||
std::vector<t_datum> data(nrows);
|
||||
tsl::hopscotch_map<t_tscalar, t_uindex> child_ridx_map;
|
||||
std::vector<bool> self_pkey_eq(nrows);
|
||||
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
data[idx].m_child.set(child_col->get_scalar(idx));
|
||||
data[idx].m_pkey.set(pkey_col->get_scalar(idx));
|
||||
child_ridx_map[data[idx].m_child] = idx;
|
||||
}
|
||||
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
auto ppkey = parent_col->get_scalar(idx);
|
||||
data[idx].m_parent.set(ppkey);
|
||||
|
||||
auto p_iter = child_ridx_map.find(ppkey);
|
||||
bool missing_parent = p_iter == child_ridx_map.end();
|
||||
|
||||
data[idx].m_is_rchild =
|
||||
!ppkey.is_valid() || data[idx].m_child == ppkey || missing_parent;
|
||||
data[idx].m_pidx =
|
||||
data[idx].m_is_rchild ? 0 : child_ridx_map.at(data[idx].m_parent);
|
||||
data[idx].m_idx = idx;
|
||||
}
|
||||
|
||||
struct t_datumcmp {
|
||||
bool
|
||||
operator()(const t_datum& a, const t_datum& b) const {
|
||||
typedef std::tuple<bool, t_tscalar, t_tscalar> t_tuple;
|
||||
return t_tuple(!a.m_is_rchild, a.m_parent, a.m_child)
|
||||
< t_tuple(!b.m_is_rchild, b.m_parent, b.m_child);
|
||||
}
|
||||
};
|
||||
|
||||
t_datumcmp cmp;
|
||||
|
||||
std::sort(data.begin(), data.end(), cmp);
|
||||
|
||||
std::vector<t_uindex> root_children;
|
||||
|
||||
std::queue<t_uindex> queue;
|
||||
t_uindex nroot_children = 0;
|
||||
while (nroot_children < nrows && data[nroot_children].m_is_rchild) {
|
||||
queue.push(nroot_children);
|
||||
++nroot_children;
|
||||
}
|
||||
|
||||
tsl::hopscotch_map<t_tscalar, std::pair<t_uindex, t_uindex>> p_range_map;
|
||||
|
||||
t_uindex brange = nroot_children;
|
||||
for (t_uindex idx = nroot_children; idx < nrows; ++idx) {
|
||||
if (data[idx].m_parent != data[idx - 1].m_parent
|
||||
&& idx > nroot_children) {
|
||||
p_range_map[data[idx - 1].m_parent] =
|
||||
std::pair<t_uindex, t_uindex>(brange, idx);
|
||||
brange = idx;
|
||||
}
|
||||
}
|
||||
|
||||
p_range_map[data.back().m_parent] =
|
||||
std::pair<t_uindex, t_uindex>(brange, nrows);
|
||||
|
||||
// map from unsorted space to sorted space
|
||||
tsl::hopscotch_map<t_uindex, t_uindex> sortidx_map;
|
||||
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
sortidx_map[data[idx].m_idx] = idx;
|
||||
}
|
||||
|
||||
while (!queue.empty()) {
|
||||
// ridx is in sorted space
|
||||
t_uindex ridx = queue.front();
|
||||
queue.pop();
|
||||
|
||||
const t_datum& rec = data[ridx];
|
||||
t_uindex pridx = rec.m_is_rchild ? 0 : sortidx_map.at(rec.m_pidx);
|
||||
|
||||
auto sortby_value =
|
||||
m_symtable.get_interned_tscalar(sortby_col->get_scalar(rec.m_idx));
|
||||
|
||||
t_uindex nidx = ridx + 1;
|
||||
t_uindex pidx = rec.m_is_rchild ? 0 : pridx + 1;
|
||||
|
||||
auto pnode = m_tree->get_node(pidx);
|
||||
|
||||
auto value = m_symtable.get_interned_tscalar(rec.m_child);
|
||||
|
||||
t_stnode node(
|
||||
nidx, pidx, value, pnode.m_depth + 1, sortby_value, 1, nidx
|
||||
);
|
||||
|
||||
m_tree->insert_node(node);
|
||||
m_tree->add_pkey(nidx, m_symtable.get_interned_tscalar(rec.m_pkey));
|
||||
|
||||
auto riter = p_range_map.find(rec.m_child);
|
||||
|
||||
if (riter != p_range_map.end()) {
|
||||
auto range = riter->second;
|
||||
t_uindex bidx = range.first;
|
||||
t_uindex eidx = range.second;
|
||||
|
||||
for (t_uindex cidx = bidx; cidx < eidx; ++cidx) {
|
||||
queue.push(cidx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
auto* aggtable = m_tree->_get_aggtable();
|
||||
aggtable->extend(nrows + 1);
|
||||
|
||||
auto aggspecs = m_config.get_aggregates();
|
||||
t_uindex naggs = aggspecs.size();
|
||||
|
||||
std::vector<t_uindex> aggindices(nrows);
|
||||
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
aggindices[idx] = data[idx].m_idx;
|
||||
}
|
||||
|
||||
parallel_for(
|
||||
int(naggs),
|
||||
[&aggtable, &aggindices, &aggspecs, &tbl](int aggnum) {
|
||||
const t_aggspec& spec = aggspecs[aggnum];
|
||||
if (spec.agg() == AGGTYPE_IDENTITY) {
|
||||
auto* scol =
|
||||
aggtable->_get_column(spec.get_first_depname());
|
||||
scol->copy(
|
||||
tbl->_get_const_column(spec.get_first_depname()),
|
||||
aggindices,
|
||||
1
|
||||
);
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
m_traversal = std::make_shared<t_traversal>(m_tree);
|
||||
|
||||
set_expansion_state(expansion_state);
|
||||
|
||||
if (!m_sortby.empty()) {
|
||||
m_traversal->sort_by(m_config, m_sortby, *this);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::pprint() const {
|
||||
m_traversal->pprint();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::notify(
|
||||
const t_data_table& flattened, bool /* is_registration */
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
rebuild();
|
||||
}
|
||||
|
||||
// aggregates should be presized to be same size
|
||||
// as agg_indices
|
||||
void
|
||||
t_ctx_grouped_pkey::get_aggregates_for_sorting(
|
||||
t_uindex nidx,
|
||||
const std::vector<t_index>& agg_indices,
|
||||
std::vector<t_tscalar>& aggregates,
|
||||
t_ctx2* /*unused*/
|
||||
) const {
|
||||
for (t_uindex idx = 0, loop_end = agg_indices.size(); idx < loop_end;
|
||||
++idx) {
|
||||
auto which_agg = agg_indices[idx];
|
||||
|
||||
if (which_agg < 0) {
|
||||
aggregates[idx].set(m_tree->get_sortby_value(nidx));
|
||||
} else {
|
||||
aggregates[idx].set(m_tree->get_aggregate(nidx, which_agg));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_ctx_grouped_pkey::get_column_dtype(t_uindex idx) const {
|
||||
if (idx == 0 || idx >= static_cast<t_uindex>(get_column_count())) {
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
auto* aggtable = m_tree->_get_aggtable();
|
||||
return aggtable->get_const_column(idx - 1)->get_dtype();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::compute_expressions(
|
||||
const std::shared_ptr<t_data_table>& master,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
t_expression_vocab& expression_vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) {
|
||||
// Clear the transitional expression tables on the context so they are
|
||||
// ready for the next update.
|
||||
m_expression_tables->clear_transitional_tables();
|
||||
|
||||
std::shared_ptr<t_data_table> master_expression_table =
|
||||
m_expression_tables->m_master;
|
||||
|
||||
// Set the master table to the right size.
|
||||
t_uindex num_rows = master->size();
|
||||
master_expression_table->reserve(num_rows);
|
||||
master_expression_table->set_size(num_rows);
|
||||
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
for (const auto& expr : expressions) {
|
||||
// Compute the expressions on the master table.
|
||||
expr->compute(
|
||||
master,
|
||||
pkey_map,
|
||||
master_expression_table,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::compute_expressions(
|
||||
const std::shared_ptr<t_data_table>& master,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
const std::shared_ptr<t_data_table>& flattened,
|
||||
const std::shared_ptr<t_data_table>& delta,
|
||||
const std::shared_ptr<t_data_table>& prev,
|
||||
const std::shared_ptr<t_data_table>& current,
|
||||
const std::shared_ptr<t_data_table>& transitions,
|
||||
const std::shared_ptr<t_data_table>& existed,
|
||||
t_expression_vocab& expression_vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) {
|
||||
// Clear the tables so they are ready for this round of updates
|
||||
m_expression_tables->clear_transitional_tables();
|
||||
|
||||
// All tables are the same size
|
||||
t_uindex flattened_num_rows = flattened->size();
|
||||
m_expression_tables->reserve_transitional_table_size(flattened_num_rows);
|
||||
m_expression_tables->set_transitional_table_size(flattened_num_rows);
|
||||
|
||||
// Update the master expression table's size
|
||||
t_uindex master_num_rows = master->size();
|
||||
m_expression_tables->m_master->reserve(master_num_rows);
|
||||
m_expression_tables->m_master->set_size(master_num_rows);
|
||||
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
for (const auto& expr : expressions) {
|
||||
// master: compute based on latest state of the gnode state table
|
||||
expr->compute(
|
||||
master,
|
||||
pkey_map,
|
||||
m_expression_tables->m_master,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// flattened: compute based on the latest update dataset
|
||||
expr->compute(
|
||||
flattened,
|
||||
pkey_map,
|
||||
m_expression_tables->m_flattened,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// delta: for each numerical column, the numerical delta between the
|
||||
// previous value and the current value in the row.
|
||||
expr->compute(
|
||||
delta,
|
||||
pkey_map,
|
||||
m_expression_tables->m_delta,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// prev: the values of the updated rows before this update was applied
|
||||
expr->compute(
|
||||
prev,
|
||||
pkey_map,
|
||||
m_expression_tables->m_prev,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// current: the current values of the updated rows
|
||||
expr->compute(
|
||||
current,
|
||||
pkey_map,
|
||||
m_expression_tables->m_current,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate the transitions now that the intermediate tables are computed
|
||||
m_expression_tables->calculate_transitions(existed);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx_grouped_pkey::num_expressions() const {
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
return expressions.size();
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx_grouped_pkey::is_expression_column(const std::string& colname) const {
|
||||
const t_schema& schema = m_expression_tables->m_master->get_schema();
|
||||
return schema.has_column(colname);
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_ctx_grouped_pkey::get_value_from_gstate(
|
||||
const std::string& colname, const t_tscalar& pkey
|
||||
) const {
|
||||
if (is_expression_column(colname)) {
|
||||
return m_gstate->get_value(
|
||||
*(m_expression_tables->m_master), colname, pkey
|
||||
);
|
||||
}
|
||||
std::shared_ptr<t_data_table> master_table = m_gstate->get_table();
|
||||
return m_gstate->get_value(*master_table, colname, pkey);
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::unity_get_row_data(t_uindex idx) const {
|
||||
auto rval = get_data(idx, idx + 1, 0, get_column_count());
|
||||
if (rval.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {rval.begin() + 1, rval.end()};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::unity_get_column_data(t_uindex idx) const {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::unity_get_row_path(t_uindex idx) const {
|
||||
return get_row_path(idx);
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx_grouped_pkey::unity_get_column_path(t_uindex idx) const {
|
||||
return {};
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx_grouped_pkey::unity_get_row_depth(t_uindex ridx) const {
|
||||
return m_traversal->get_depth(ridx);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx_grouped_pkey::unity_get_column_depth(t_uindex cidx) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx_grouped_pkey::unity_get_column_name(t_uindex idx) const {
|
||||
return m_config.col_at(idx);
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx_grouped_pkey::unity_get_column_display_name(t_uindex idx) const {
|
||||
return m_config.col_at(idx);
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctx_grouped_pkey::unity_get_column_names() const {
|
||||
return m_config.get_column_names();
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctx_grouped_pkey::unity_get_column_display_names() const {
|
||||
return m_config.get_column_names();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx_grouped_pkey::unity_get_column_count() const {
|
||||
return get_column_count() - 1;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx_grouped_pkey::unity_get_row_count() const {
|
||||
return get_row_count();
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx_grouped_pkey::unity_get_row_expanded(t_uindex idx) const {
|
||||
return m_traversal->get_node_expanded(idx);
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx_grouped_pkey::unity_get_column_expanded(t_uindex idx) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::clear_deltas() {}
|
||||
|
||||
void
|
||||
t_ctx_grouped_pkey::unity_init_load_step_end() {}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,52 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/context_handle.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_ctx_handle::t_ctx_handle() : m_ctx_type(ZERO_SIDED_CONTEXT), m_ctx(nullptr) {}
|
||||
|
||||
t_ctx_handle::t_ctx_handle(void* ctx, t_ctx_type ctx_type) :
|
||||
m_ctx_type(ctx_type),
|
||||
m_ctx(ctx) {}
|
||||
|
||||
std::string
|
||||
t_ctx_handle::get_type_descr() const {
|
||||
switch (m_ctx_type) {
|
||||
case TWO_SIDED_CONTEXT: {
|
||||
return "TWO_SIDED_CONTEXT";
|
||||
} break;
|
||||
case ONE_SIDED_CONTEXT: {
|
||||
return "ONE_SIDED_CONTEXT";
|
||||
} break;
|
||||
case ZERO_SIDED_CONTEXT: {
|
||||
return "ZERO_SIDED_CONTEXT";
|
||||
} break;
|
||||
case UNIT_CONTEXT: {
|
||||
return "UNIT_CONTEXT";
|
||||
} break;
|
||||
case GROUPED_PKEY_CONTEXT: {
|
||||
return "GROUPED_PKEY_CONTEXT";
|
||||
} break;
|
||||
case GROUPED_COLUMNS_CONTEXT: {
|
||||
return "GROUPED_COLUMNS_CONTEXT";
|
||||
}
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Invalid context");
|
||||
} break;
|
||||
}
|
||||
return "ZERO_SIDED_CONTEXT";
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,246 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/sort_specification.h>
|
||||
#include <perspective/context_zero.h>
|
||||
#include <perspective/context_one.h>
|
||||
#include <perspective/context_two.h>
|
||||
#include <perspective/extract_aggregate.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <perspective/sparse_tree.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_trav_iter_node::t_trav_iter_node(
|
||||
const t_tscalar& value,
|
||||
bool expanded,
|
||||
t_depth depth,
|
||||
t_index ndesc,
|
||||
t_index tnid,
|
||||
t_index ridx,
|
||||
t_index pridx
|
||||
) :
|
||||
value(value),
|
||||
expanded(expanded),
|
||||
depth(depth),
|
||||
ndesc(ndesc),
|
||||
tree_handle(tnid),
|
||||
idx(ridx),
|
||||
pidx(pridx) {}
|
||||
|
||||
t_ctx2_trav_seq_iter::t_ctx2_trav_seq_iter(t_ctx2* ctx, t_header header) :
|
||||
m_ctx(ctx),
|
||||
m_header(header),
|
||||
m_curidx(0) {
|
||||
m_nrows = ctx->get_row_count(header);
|
||||
}
|
||||
|
||||
t_leaf_iter_node::t_leaf_iter_node(
|
||||
const t_tscalar& value,
|
||||
bool expanded,
|
||||
t_depth depth,
|
||||
t_index ndesc,
|
||||
t_index tnid,
|
||||
t_index ridx,
|
||||
t_index pridx,
|
||||
t_index nleaves
|
||||
) :
|
||||
value(value),
|
||||
expanded(expanded),
|
||||
depth(depth),
|
||||
ndesc(ndesc),
|
||||
tree_handle(tnid),
|
||||
idx(ridx),
|
||||
pidx(pridx),
|
||||
nleaves(nleaves) {}
|
||||
|
||||
bool
|
||||
t_ctx2_trav_seq_iter::has_next() const {
|
||||
return m_curidx < m_nrows;
|
||||
}
|
||||
|
||||
t_trav_iter_node*
|
||||
t_ctx2_trav_seq_iter::next() {
|
||||
auto node = m_ctx->get_trav_node(m_header, m_curidx);
|
||||
auto iter_node = new t_trav_iter_node(
|
||||
m_ctx->get_tree_value(m_header, node.m_tnid),
|
||||
node.m_expanded,
|
||||
node.m_depth,
|
||||
node.m_ndesc,
|
||||
node.m_tnid,
|
||||
m_curidx,
|
||||
m_curidx - node.m_rel_pidx
|
||||
);
|
||||
|
||||
m_curidx += 1;
|
||||
return iter_node;
|
||||
}
|
||||
|
||||
t_ctx2_leaf_iter::t_ctx2_leaf_iter(const t_ctx2* ctx, t_header header) :
|
||||
m_ctx(ctx),
|
||||
m_header(header),
|
||||
m_curidx(0) {
|
||||
m_nrows = ctx->get_row_count(header);
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx2_leaf_iter::has_next() const {
|
||||
return m_curidx < m_nrows;
|
||||
}
|
||||
|
||||
t_leaf_iter_node*
|
||||
t_ctx2_leaf_iter::next() {
|
||||
auto node = m_ctx->get_trav_node(m_header, m_curidx);
|
||||
t_index nleaves = m_ctx->get_trav_num_tree_leaves(m_header, m_curidx);
|
||||
|
||||
t_leaf_iter_node* iter_node = new t_leaf_iter_node(
|
||||
m_ctx->get_tree_value(m_header, node.m_tnid),
|
||||
node.m_expanded,
|
||||
node.m_depth,
|
||||
node.m_ndesc,
|
||||
node.m_tnid,
|
||||
m_curidx,
|
||||
m_curidx - node.m_rel_pidx,
|
||||
nleaves
|
||||
);
|
||||
|
||||
m_curidx += 1;
|
||||
return iter_node;
|
||||
}
|
||||
|
||||
t_ctx1_tree_iter::t_ctx1_tree_iter(t_ctx1* ctx, t_index root, t_depth depth) :
|
||||
m_curidx(0),
|
||||
m_ctx(ctx) {
|
||||
m_agg_indices = std::vector<t_index>(ctx->get_aggregates().size());
|
||||
|
||||
for (t_uindex idx = 0, loop_end = m_agg_indices.size(); idx < loop_end;
|
||||
++idx) {
|
||||
m_agg_indices[idx] = idx;
|
||||
}
|
||||
|
||||
m_has_next = !(m_agg_indices.empty());
|
||||
if (m_has_next) {
|
||||
t_depth rdepth = ctx->get_trav_depth(root);
|
||||
m_edepth = rdepth + depth;
|
||||
m_edepth = std::min(m_edepth, ctx->get_num_levels() - 1);
|
||||
m_vec = m_ctx->get_flattened_tree(root, m_edepth);
|
||||
if (m_vec.size() == 1 && m_vec[0].m_nchild == 0) {
|
||||
m_has_next = false;
|
||||
}
|
||||
|
||||
m_aggcols = ctx->_get_tree()->get_aggcols(m_agg_indices);
|
||||
}
|
||||
}
|
||||
|
||||
t_ctx1_tree_iter::t_ctx1_tree_iter(
|
||||
t_ctx1* ctx,
|
||||
t_index root,
|
||||
t_depth depth,
|
||||
const std::vector<t_index>& aggregates
|
||||
) :
|
||||
m_curidx(0),
|
||||
m_ctx(ctx),
|
||||
m_agg_indices(aggregates) {
|
||||
m_has_next = !(m_agg_indices.empty());
|
||||
if (m_has_next) {
|
||||
t_depth rdepth = ctx->get_trav_depth(root);
|
||||
m_edepth = rdepth + depth;
|
||||
m_edepth = std::min(m_edepth, ctx->get_num_levels() - 1);
|
||||
m_vec = m_ctx->get_flattened_tree(root, m_edepth);
|
||||
if (m_vec.size() == 1 && m_vec[0].m_nchild == 0) {
|
||||
m_has_next = false;
|
||||
}
|
||||
|
||||
m_aggcols = ctx->_get_tree()->get_aggcols(aggregates);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1_tree_iter::has_next() const {
|
||||
return m_has_next;
|
||||
}
|
||||
|
||||
PyObject*
|
||||
t_ctx1_tree_iter::next() {
|
||||
t_ftreenode& ftnode = m_vec[m_curidx];
|
||||
t_index bidx;
|
||||
t_index nrows;
|
||||
if (m_ctx->get_row_pivots().size() == 0 || m_edepth == 0) {
|
||||
bidx = 0;
|
||||
nrows = 1;
|
||||
} else {
|
||||
bidx = ftnode.m_fcidx;
|
||||
nrows = ftnode.m_nchild;
|
||||
}
|
||||
npy_intp dims[] = {static_cast<npy_intp>(nrows)};
|
||||
t_py_handle dtype(
|
||||
Py_BuildValue("[(s, s), (s, s)]", "id", "i4", "field_label", "O")
|
||||
);
|
||||
t_index n_aggs = m_agg_indices.size();
|
||||
|
||||
for (int i = 0; i < n_aggs; i++) {
|
||||
std::stringstream field_name;
|
||||
field_name << "field_" << i + 1;
|
||||
t_py_handle d_field(
|
||||
Py_BuildValue("(s,s)", field_name.str().c_str(), "f8")
|
||||
);
|
||||
PyList_Append(dtype.m_pyo, d_field.m_pyo);
|
||||
}
|
||||
PyArray_Descr* descr;
|
||||
PyArray_DescrConverter(dtype.m_pyo, &descr);
|
||||
t_py_handle array(PyArray_SimpleNewFromDescr(1, dims, descr));
|
||||
PyArrayObject* array_ = reinterpret_cast<PyArrayObject*>(array.m_pyo);
|
||||
|
||||
t_stree* tree = m_ctx->_get_tree();
|
||||
const std::vector<t_aggspec>& aggspecs =
|
||||
m_ctx->get_config().get_aggregates();
|
||||
|
||||
for (int i = 0; i < nrows; i++) {
|
||||
t_ftreenode& c_ftnode = m_vec[bidx + i];
|
||||
t_index cidx = c_ftnode.m_idx;
|
||||
void* elem_ptr = PyArray_GETPTR1(array_, i);
|
||||
t_py_handle val(PyTuple_New(n_aggs + 2));
|
||||
PyTuple_SetItem(val.m_pyo, 0, PyInt_FromLong(cidx));
|
||||
|
||||
t_tscalar tree_value = m_ctx->get_tree_value(cidx);
|
||||
PyObject* py_tree_value = tscalar_to_python(tree_value);
|
||||
PyTuple_SetItem(val.m_pyo, 1, py_tree_value);
|
||||
|
||||
t_index p_cidx = tree->get_parent_idx(cidx);
|
||||
|
||||
t_uindex agg_ridx = tree->get_aggidx(cidx);
|
||||
t_index agg_pridx =
|
||||
p_cidx == INVALID_INDEX ? INVALID_INDEX : tree->get_aggidx(p_cidx);
|
||||
|
||||
for (int j = 0; j < n_aggs; j++) {
|
||||
t_tscalar agg_scalar = extract_aggregate(
|
||||
aggspecs[j], m_aggcols[j], agg_ridx, agg_pridx
|
||||
);
|
||||
double aggdbl = agg_scalar.to_double();
|
||||
PyTuple_SetItem(val.m_pyo, j + 2, PyFloat_FromDouble(aggdbl));
|
||||
}
|
||||
PyArray_SETITEM(array_, static_cast<char*>(elem_ptr), val.m_pyo);
|
||||
}
|
||||
long is_leaf = ftnode.m_depth + 1 == m_edepth;
|
||||
PyObject* rval = Py_BuildValue("OO", PyBool_FromLong(is_leaf), array.m_pyo);
|
||||
m_curidx++;
|
||||
|
||||
if (m_vec[m_curidx].m_depth == t_depth(-1)
|
||||
|| m_ctx->get_row_pivots().size() == 0 || m_edepth == 0
|
||||
|| static_cast<std::uint64_t>(m_curidx) >= m_vec.size()) {
|
||||
m_has_next = false;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,944 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/sort_specification.h>
|
||||
#include <perspective/get_data_extents.h>
|
||||
#include <perspective/context_one.h>
|
||||
#include <perspective/extract_aggregate.h>
|
||||
#include <perspective/filter.h>
|
||||
#include <perspective/sparse_tree.h>
|
||||
#include <perspective/tree_context_common.h>
|
||||
|
||||
#include <perspective/env_vars.h>
|
||||
#include <perspective/traversal.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_ctx1::t_ctx1(const t_schema& schema, const t_config& pivot_config) :
|
||||
t_ctxbase<t_ctx1>(schema, pivot_config),
|
||||
m_depth(0),
|
||||
m_depth_set(false) {}
|
||||
|
||||
t_ctx1::~t_ctx1() = default;
|
||||
|
||||
void
|
||||
t_ctx1::init() {
|
||||
auto pivots = m_config.get_row_pivots();
|
||||
m_tree = std::make_shared<t_stree>(
|
||||
pivots, m_config.get_aggregates(), m_schema, m_config
|
||||
);
|
||||
m_tree->init();
|
||||
m_traversal = std::make_shared<t_traversal>(m_tree);
|
||||
|
||||
// Each context stores its own expression columns in separate
|
||||
// `t_data_table`s so that each context's expressions are isolated
|
||||
// and do not affect other contexts when they are calculated.
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
m_expression_tables = std::make_shared<t_expression_tables>(
|
||||
expressions, m_config.get_backing_store()
|
||||
);
|
||||
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::get_row_count() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_traversal->size();
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::get_column_count() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_config.get_num_aggregates() + 1;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::open(t_header header, t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return open(idx);
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx1::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_ctx1<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::open(t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
// If we manually open/close a node, stop automatically expanding
|
||||
m_depth_set = false;
|
||||
m_depth = 0;
|
||||
|
||||
if (idx >= t_index(m_traversal->size())) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_index retval = m_traversal->expand_node(m_sortby, idx);
|
||||
m_rows_changed = (retval > 0);
|
||||
return retval;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::close(t_index idx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
// If we manually open/close a node, stop automatically expanding
|
||||
m_depth_set = false;
|
||||
m_depth = 0;
|
||||
|
||||
if (idx >= t_index(m_traversal->size())) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_index retval = m_traversal->collapse_node(idx);
|
||||
m_rows_changed = (retval > 0);
|
||||
return retval;
|
||||
}
|
||||
|
||||
std::pair<t_tscalar, t_tscalar>
|
||||
t_ctx1::get_min_max(const std::string& colname) const {
|
||||
auto rval = std::make_pair(mknone(), mknone());
|
||||
auto* aggtable = m_tree->get_aggtable();
|
||||
const t_schema& aggschema = aggtable->get_schema();
|
||||
const auto* col = aggtable->_get_const_column(colname);
|
||||
auto colidx = aggschema.get_colidx(colname);
|
||||
auto depth = m_config.get_num_rpivots();
|
||||
const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
|
||||
bool is_finished = false;
|
||||
while (!is_finished && depth > 0) {
|
||||
for (std::size_t i = 0; i < m_traversal->size(); i++) {
|
||||
t_index nidx = m_traversal->get_tree_index(i);
|
||||
t_index pnidx = m_tree->get_parent_idx(nidx);
|
||||
if (m_tree->get_depth(nidx) != depth) {
|
||||
continue;
|
||||
}
|
||||
|
||||
t_uindex agg_ridx = m_tree->get_aggidx(nidx);
|
||||
t_index agg_pridx = pnidx == INVALID_INDEX
|
||||
? INVALID_INDEX
|
||||
: m_tree->get_aggidx(pnidx);
|
||||
t_tscalar val =
|
||||
extract_aggregate(aggspecs[colidx], col, agg_ridx, agg_pridx);
|
||||
|
||||
if (!val.is_valid()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
is_finished = true;
|
||||
if (rval.first.is_none() || (!val.is_none() && val < rval.first)) {
|
||||
rval.first = val;
|
||||
}
|
||||
|
||||
if (val > rval.second) {
|
||||
rval.second = val;
|
||||
}
|
||||
}
|
||||
|
||||
depth--;
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::get_data(
|
||||
t_index start_row, t_index end_row, t_index start_col, t_index end_col
|
||||
) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex ctx_nrows = get_row_count();
|
||||
t_uindex ncols = get_column_count();
|
||||
auto ext = sanitize_get_data_extents(
|
||||
ctx_nrows, ncols, start_row, end_row, start_col, end_col
|
||||
);
|
||||
|
||||
t_index nrows = ext.m_erow - ext.m_srow;
|
||||
t_index stride = ext.m_ecol - ext.m_scol;
|
||||
|
||||
std::vector<t_tscalar> values(nrows * stride);
|
||||
|
||||
std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
|
||||
|
||||
auto* aggtable = m_tree->get_aggtable();
|
||||
auto none = mknone();
|
||||
|
||||
for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
// `aggidx` is the column's position in the aggtable schema, so resolve
|
||||
// by index — avoids the per-iteration name->index map lookup + a temp
|
||||
// std::string, and the up-front schema deep-copy.
|
||||
aggcols[aggidx] = aggtable->_get_const_column(aggidx);
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
|
||||
|
||||
for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
|
||||
t_index nidx = m_traversal->get_tree_index(ridx);
|
||||
t_index pnidx = m_tree->get_parent_idx(nidx);
|
||||
|
||||
t_uindex agg_ridx = m_tree->get_aggidx(nidx);
|
||||
t_index agg_pridx =
|
||||
pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
|
||||
|
||||
t_index row_off = (ridx - ext.m_srow) * stride;
|
||||
|
||||
if (ext.m_scol == 0) {
|
||||
t_tscalar tree_value = m_tree->get_value(nidx);
|
||||
values[row_off] = tree_value;
|
||||
}
|
||||
|
||||
for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
t_index col = 1 + aggidx;
|
||||
if (col < ext.m_scol || col >= ext.m_ecol) {
|
||||
continue;
|
||||
}
|
||||
t_tscalar value = extract_aggregate(
|
||||
aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
|
||||
);
|
||||
if (!value.is_valid()) {
|
||||
value.set(none);
|
||||
}
|
||||
values[row_off + col - ext.m_scol].set(value);
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::get_data(const std::vector<t_uindex>& rows) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex nrows = rows.size();
|
||||
t_uindex ncols = get_column_count();
|
||||
|
||||
std::vector<t_tscalar> values(nrows * ncols);
|
||||
|
||||
std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
|
||||
|
||||
auto* aggtable = m_tree->get_aggtable();
|
||||
auto none = mknone();
|
||||
|
||||
for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
// `aggidx` is the column's position in the aggtable schema, so resolve
|
||||
// by index — avoids the per-iteration name->index map lookup + a temp
|
||||
// std::string, and the up-front schema deep-copy.
|
||||
aggcols[aggidx] = aggtable->_get_const_column(aggidx);
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
|
||||
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
t_uindex ridx = rows[idx];
|
||||
t_index nidx = m_traversal->get_tree_index(ridx);
|
||||
t_index pnidx = m_tree->get_parent_idx(nidx);
|
||||
|
||||
t_uindex agg_ridx = m_tree->get_aggidx(nidx);
|
||||
t_index agg_pridx =
|
||||
pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
|
||||
|
||||
t_tscalar tree_value = m_tree->get_value(nidx);
|
||||
values[idx * ncols] = tree_value;
|
||||
|
||||
for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
t_tscalar value = extract_aggregate(
|
||||
aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
|
||||
);
|
||||
if (!value.is_valid()) {
|
||||
value.set(none);
|
||||
}
|
||||
values[idx * ncols + 1 + aggidx].set(value);
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::notify(const t_data_table& flattened, bool /* is_registration */) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
notify_sparse_tree(
|
||||
m_tree,
|
||||
m_traversal,
|
||||
true,
|
||||
m_config.get_aggregates(),
|
||||
m_config.get_sortby_pairs(),
|
||||
m_sortby,
|
||||
flattened,
|
||||
m_config,
|
||||
*m_gstate,
|
||||
*(m_expression_tables->m_master)
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::notify(
|
||||
const t_data_table& flattened,
|
||||
const t_data_table& delta,
|
||||
const t_data_table& prev,
|
||||
const t_data_table& current,
|
||||
const t_data_table& transitions,
|
||||
const t_data_table& existed
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
notify_sparse_tree(
|
||||
m_tree,
|
||||
m_traversal,
|
||||
true,
|
||||
m_config.get_aggregates(),
|
||||
m_config.get_sortby_pairs(),
|
||||
m_sortby,
|
||||
flattened,
|
||||
delta,
|
||||
prev,
|
||||
current,
|
||||
transitions,
|
||||
existed,
|
||||
m_config,
|
||||
*m_gstate,
|
||||
*(m_expression_tables->m_master)
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::step_begin() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
reset_step_state();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::step_end() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
if (m_total_only) {
|
||||
m_traversal->rebuild_for_total();
|
||||
} else if (m_leaves_only) {
|
||||
m_traversal->rebuild_from_leaves(m_sortby);
|
||||
} else {
|
||||
sort_by(m_sortby);
|
||||
if (m_depth_set) {
|
||||
set_depth(m_depth);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_aggspec
|
||||
t_ctx1::get_aggregate(t_uindex idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
if (idx >= m_config.get_num_aggregates()) {
|
||||
return {};
|
||||
}
|
||||
return m_config.get_aggregates()[idx];
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_ctx1::get_aggregate_name(t_uindex idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_tscalar s;
|
||||
if (idx >= m_config.get_num_aggregates()) {
|
||||
return s;
|
||||
}
|
||||
s.set(m_config.get_aggregates()[idx].name_scalar());
|
||||
return s;
|
||||
}
|
||||
|
||||
std::vector<t_aggspec>
|
||||
t_ctx1::get_aggregates() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_config.get_aggregates();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::get_row_path(t_index idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
if (idx < 0) {
|
||||
return {};
|
||||
}
|
||||
return ctx_get_path(m_tree, m_traversal, idx);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::reset_sortby() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
m_sortby = std::vector<t_sortspec>();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::sort_by(const std::vector<t_sortspec>& sortby) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
m_sortby = sortby;
|
||||
if (m_sortby.empty()) {
|
||||
return;
|
||||
}
|
||||
m_traversal->sort_by(m_config, sortby, *(m_tree));
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_depth(t_depth depth) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
if (m_config.get_num_rpivots() == 0) {
|
||||
return;
|
||||
}
|
||||
depth = std::min<t_depth>(m_config.get_num_rpivots() - 1, depth);
|
||||
t_index retval = 0;
|
||||
retval = m_traversal->set_depth(m_sortby, depth);
|
||||
m_rows_changed = (retval > 0);
|
||||
m_depth = depth;
|
||||
m_depth_set = true;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_leaves_only(bool enabled) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
m_leaves_only = enabled;
|
||||
m_traversal->set_leaves_only(enabled, m_config.get_num_rpivots());
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_total_only(bool enabled) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
m_total_only = enabled;
|
||||
m_traversal->set_total_only(enabled);
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (!m_traversal->validate_cells(cells)) {
|
||||
std::vector<t_tscalar> rval;
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar> rval;
|
||||
std::vector<t_index> tindices(cells.size());
|
||||
for (const auto& c : cells) {
|
||||
auto ptidx = m_traversal->get_tree_index(c.first);
|
||||
auto pkeys = m_tree->get_pkeys(ptidx);
|
||||
|
||||
rval.insert(std::end(rval), std::begin(pkeys), std::end(pkeys));
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1::get_deltas_enabled() const {
|
||||
return m_features[CTX_FEAT_DELTA];
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_feature_state(t_ctx_feature feature, bool state) {
|
||||
m_features[feature] = state;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_alerts_enabled(bool enabled_state) {
|
||||
m_features[CTX_FEAT_ALERT] = enabled_state;
|
||||
m_tree->set_alerts_enabled(enabled_state);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::set_deltas_enabled(bool enabled_state) {
|
||||
m_features[CTX_FEAT_DELTA] = enabled_state;
|
||||
m_tree->set_deltas_enabled(enabled_state);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns updated cells.
|
||||
*
|
||||
* @param bidx
|
||||
* @param eidx
|
||||
* @return t_stepdelta
|
||||
*/
|
||||
t_stepdelta
|
||||
t_ctx1::get_step_delta(t_index bidx, t_index eidx) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
bidx = std::min(bidx, t_index(m_traversal->size()));
|
||||
eidx = std::min(eidx, t_index(m_traversal->size()));
|
||||
|
||||
t_stepdelta rval(
|
||||
m_rows_changed, m_columns_changed, get_cell_delta(bidx, eidx)
|
||||
);
|
||||
m_tree->clear_deltas();
|
||||
return rval;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns a `t_rowdelta` object containing:
|
||||
* - the row indices that have been updated
|
||||
* - the data from those updated rows
|
||||
*
|
||||
* @return t_rowdelta
|
||||
*/
|
||||
t_rowdelta
|
||||
t_ctx1::get_row_delta() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
std::vector<t_uindex> rows = get_rows_changed();
|
||||
std::vector<t_tscalar> data = get_data(rows);
|
||||
t_rowdelta rval(m_rows_changed, rows.size(), data);
|
||||
m_tree->clear_deltas();
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_uindex>
|
||||
t_ctx1::get_rows_changed() {
|
||||
std::vector<t_uindex> rows;
|
||||
const auto& deltas = m_tree->get_deltas();
|
||||
auto eidx = t_uindex(m_traversal->size());
|
||||
|
||||
// `idx` increases monotonically and is pushed at most once, so `rows` is
|
||||
// already unique and sorted: the per-iteration `std::find` (which made this
|
||||
// O(n^2)) and the trailing `std::sort` were both dead work.
|
||||
for (t_uindex idx = 0; idx < eidx; ++idx) {
|
||||
t_index ptidx = m_traversal->get_tree_index(idx);
|
||||
// Retrieve delta from storage and check if the row has been changed
|
||||
auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
|
||||
if (iterators.first != iterators.second) {
|
||||
rows.push_back(idx);
|
||||
}
|
||||
}
|
||||
|
||||
return rows;
|
||||
}
|
||||
|
||||
std::vector<t_cellupd>
|
||||
t_ctx1::get_cell_delta(t_index bidx, t_index eidx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
eidx = std::min(eidx, t_index(m_traversal->size()));
|
||||
std::vector<t_cellupd> rval;
|
||||
const auto& deltas = m_tree->get_deltas();
|
||||
for (t_index idx = bidx; idx < eidx; ++idx) {
|
||||
t_index ptidx = m_traversal->get_tree_index(idx);
|
||||
auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
|
||||
for (auto iter = iterators.first; iter != iterators.second; ++iter) {
|
||||
rval.emplace_back(
|
||||
idx, iter->m_aggidx + 1, iter->m_old_value, iter->m_new_value
|
||||
);
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::reset(bool reset_expressions) {
|
||||
auto pivots = m_config.get_row_pivots();
|
||||
m_tree = std::make_shared<t_stree>(
|
||||
pivots, m_config.get_aggregates(), m_schema, m_config
|
||||
);
|
||||
m_tree->init();
|
||||
m_tree->set_deltas_enabled(get_feature_state(CTX_FEAT_DELTA));
|
||||
m_traversal = std::make_shared<t_traversal>(m_tree);
|
||||
if (m_leaves_only) {
|
||||
m_traversal->set_leaves_only(true, m_config.get_num_rpivots());
|
||||
}
|
||||
|
||||
if (reset_expressions) {
|
||||
m_expression_tables->reset();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::reset_step_state() {
|
||||
m_rows_changed = false;
|
||||
m_columns_changed = false;
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_ctx1.reset_step_state " << repr() << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::sidedness() const {
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::vector<t_stree*>
|
||||
t_ctx1::get_trees() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
std::vector<t_stree*> rval(1);
|
||||
rval[0] = m_tree.get();
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1::has_deltas() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_tree->has_deltas();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::pprint() const {
|
||||
std::cout << "\t" << '\n';
|
||||
for (auto idx = 1; idx < get_column_count(); ++idx) {
|
||||
std::cout << get_aggregate(idx - 1).agg_str() << ", " << '\n';
|
||||
}
|
||||
|
||||
std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
|
||||
auto* aggtable = m_tree->get_aggtable();
|
||||
auto none = mknone();
|
||||
|
||||
for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
// `aggidx` is the column's position in the aggtable schema, so resolve
|
||||
// by index — avoids the per-iteration name->index map lookup + a temp
|
||||
// std::string, and the up-front schema deep-copy.
|
||||
aggcols[aggidx] = aggtable->_get_const_column(aggidx);
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
|
||||
|
||||
for (auto ridx = 0; ridx < get_row_count(); ++ridx) {
|
||||
t_index nidx = m_traversal->get_tree_index(ridx);
|
||||
t_index pnidx = m_tree->get_parent_idx(nidx);
|
||||
|
||||
t_uindex agg_ridx = m_tree->get_aggidx(nidx);
|
||||
t_index agg_pridx =
|
||||
pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
|
||||
|
||||
std::cout << get_row_path(ridx) << " => ";
|
||||
for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
|
||||
++aggidx) {
|
||||
t_tscalar value = extract_aggregate(
|
||||
aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
|
||||
);
|
||||
if (!value.is_valid()) {
|
||||
value.set(none); // todo: fix null handling
|
||||
}
|
||||
|
||||
std::cout << value << ", ";
|
||||
}
|
||||
|
||||
std::cout << "\n";
|
||||
}
|
||||
|
||||
std::cout << "=================" << '\n';
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctx1::get_row_idx(const std::vector<t_tscalar>& path) const {
|
||||
auto nidx = m_tree->resolve_path(0, path);
|
||||
if (nidx == INVALID_INDEX) {
|
||||
return nidx;
|
||||
}
|
||||
|
||||
return m_traversal->get_traversal_index(nidx);
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_ctx1::get_column_dtype(t_uindex idx) const {
|
||||
if (idx == 0 || idx >= static_cast<t_uindex>(get_column_count())) {
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
return m_tree->get_aggtable()->get_const_column(idx - 1)->get_dtype();
|
||||
}
|
||||
|
||||
t_depth
|
||||
t_ctx1::get_trav_depth(t_index idx) const {
|
||||
return m_traversal->get_depth(idx);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::compute_expressions(
|
||||
const std::shared_ptr<t_data_table>& master,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
t_expression_vocab& expression_vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) {
|
||||
// Clear the transitional expression tables on the context so they are
|
||||
// ready for the next update.
|
||||
m_expression_tables->clear_transitional_tables();
|
||||
|
||||
std::shared_ptr<t_data_table> master_expression_table =
|
||||
m_expression_tables->m_master;
|
||||
|
||||
// Set the master table to the right size.
|
||||
t_uindex num_rows = master->size();
|
||||
master_expression_table->reserve(num_rows);
|
||||
master_expression_table->set_size(num_rows);
|
||||
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
for (const auto& expr : expressions) {
|
||||
// Compute the expressions on the master table.
|
||||
expr->compute(
|
||||
master,
|
||||
pkey_map,
|
||||
master_expression_table,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::compute_expressions(
|
||||
const std::shared_ptr<t_data_table>& master,
|
||||
const t_gstate::t_mapping& pkey_map,
|
||||
const std::shared_ptr<t_data_table>& flattened,
|
||||
const std::shared_ptr<t_data_table>& delta,
|
||||
const std::shared_ptr<t_data_table>& prev,
|
||||
const std::shared_ptr<t_data_table>& current,
|
||||
const std::shared_ptr<t_data_table>& transitions,
|
||||
const std::shared_ptr<t_data_table>& existed,
|
||||
t_expression_vocab& expression_vocab,
|
||||
t_regex_mapping& regex_mapping
|
||||
) {
|
||||
// Clear the tables so they are ready for this round of updates
|
||||
m_expression_tables->clear_transitional_tables();
|
||||
|
||||
// All tables are the same size
|
||||
t_uindex flattened_num_rows = flattened->size();
|
||||
m_expression_tables->reserve_transitional_table_size(flattened_num_rows);
|
||||
m_expression_tables->set_transitional_table_size(flattened_num_rows);
|
||||
|
||||
// Update the master expression table's size
|
||||
t_uindex master_num_rows = master->size();
|
||||
m_expression_tables->m_master->reserve(master_num_rows);
|
||||
m_expression_tables->m_master->set_size(master_num_rows);
|
||||
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
for (const auto& expr : expressions) {
|
||||
// master: compute based on latest state of the gnode state table
|
||||
expr->compute(
|
||||
master,
|
||||
pkey_map,
|
||||
m_expression_tables->m_master,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// flattened: compute based on the latest update dataset
|
||||
expr->compute(
|
||||
flattened,
|
||||
pkey_map,
|
||||
m_expression_tables->m_flattened,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// delta: for each numerical column, the numerical delta between the
|
||||
// previous value and the current value in the row.
|
||||
expr->compute(
|
||||
delta,
|
||||
pkey_map,
|
||||
m_expression_tables->m_delta,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// prev: the values of the updated rows before this update was applied
|
||||
expr->compute(
|
||||
prev,
|
||||
pkey_map,
|
||||
m_expression_tables->m_prev,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
|
||||
// current: the current values of the updated rows
|
||||
expr->compute(
|
||||
current,
|
||||
pkey_map,
|
||||
m_expression_tables->m_current,
|
||||
expression_vocab,
|
||||
regex_mapping
|
||||
);
|
||||
}
|
||||
|
||||
// Calculate the transitions now that the intermediate tables are computed
|
||||
m_expression_tables->calculate_transitions(existed);
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1::is_expression_column(const std::string& colname) const {
|
||||
const t_schema& schema = m_expression_tables->m_master->get_schema();
|
||||
return schema.has_column(colname);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx1::num_expressions() const {
|
||||
const auto& expressions = m_config.get_expressions();
|
||||
return expressions.size();
|
||||
}
|
||||
|
||||
std::shared_ptr<t_expression_tables>
|
||||
t_ctx1::get_expression_tables() const {
|
||||
return m_expression_tables;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::unity_get_row_data(t_uindex idx) const {
|
||||
auto rval = get_data(idx, idx + 1, 0, get_column_count());
|
||||
if (rval.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return {rval.begin() + 1, rval.end()};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::unity_get_column_data(t_uindex idx) const {
|
||||
PSP_COMPLAIN_AND_ABORT("Not implemented");
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::unity_get_row_path(t_uindex idx) const {
|
||||
return get_row_path(idx);
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctx1::unity_get_column_path(t_uindex idx) const {
|
||||
return {};
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx1::unity_get_row_depth(t_uindex ridx) const {
|
||||
return m_traversal->get_depth(ridx);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx1::unity_get_column_depth(t_uindex cidx) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx1::unity_get_column_name(t_uindex idx) const {
|
||||
return m_config.unity_get_column_name(idx);
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctx1::unity_get_column_display_name(t_uindex idx) const {
|
||||
return m_config.unity_get_column_display_name(idx);
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctx1::unity_get_column_names() const {
|
||||
std::vector<std::string> rv;
|
||||
|
||||
for (t_uindex idx = 0, loop_end = unity_get_column_count(); idx < loop_end;
|
||||
++idx) {
|
||||
rv.push_back(unity_get_column_name(idx));
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctx1::unity_get_column_display_names() const {
|
||||
std::vector<std::string> rv;
|
||||
|
||||
for (t_uindex idx = 0, loop_end = unity_get_column_count(); idx < loop_end;
|
||||
++idx) {
|
||||
rv.push_back(unity_get_column_display_name(idx));
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx1::unity_get_column_count() const {
|
||||
return get_column_count() - 1;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctx1::unity_get_row_count() const {
|
||||
return get_row_count();
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1::unity_get_row_expanded(t_uindex idx) const {
|
||||
return m_traversal->get_node_expanded(idx);
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctx1::unity_get_column_expanded(t_uindex idx) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::clear_deltas() {
|
||||
m_tree->clear_deltas();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctx1::unity_init_load_step_end() {}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_ctx1::get_table() const {
|
||||
const t_schema& schema = m_tree->get_aggtable()->get_schema();
|
||||
const auto& pivots = m_config.get_row_pivots();
|
||||
auto tbl = std::make_shared<t_data_table>(schema, m_tree->size());
|
||||
tbl->init();
|
||||
tbl->extend(m_tree->size());
|
||||
|
||||
std::vector<t_column*> aggcols = tbl->get_columns();
|
||||
auto n_aggs = aggcols.size();
|
||||
std::vector<t_column*> pivcols;
|
||||
|
||||
std::stringstream ss;
|
||||
pivcols.reserve(pivots.size());
|
||||
for (const auto& c : pivots) {
|
||||
pivcols.push_back(
|
||||
tbl->add_column(c.colname(), m_schema.get_dtype(c.colname()), true)
|
||||
);
|
||||
}
|
||||
|
||||
auto idx = 0;
|
||||
for (auto nidx : m_tree->dfs()) {
|
||||
auto depth = m_tree->get_depth(nidx);
|
||||
if (depth > 0) {
|
||||
pivcols[depth - 1]->set_scalar(idx, m_tree->get_value(nidx));
|
||||
}
|
||||
for (t_uindex aggnum = 0; aggnum < n_aggs; ++aggnum) {
|
||||
auto aggscalar = m_tree->get_aggregate(nidx, aggnum);
|
||||
aggcols[aggnum]->set_scalar(idx, aggscalar);
|
||||
}
|
||||
++idx;
|
||||
}
|
||||
return tbl;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,497 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/context_base.h>
|
||||
#include <perspective/get_data_extents.h>
|
||||
#include <perspective/context_unit.h>
|
||||
#include <perspective/flat_traversal.h>
|
||||
#include <perspective/sym_table.h>
|
||||
|
||||
#include <perspective/filter_utils.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_ctxunit::t_ctxunit() = default;
|
||||
|
||||
t_ctxunit::t_ctxunit(const t_schema& schema, const t_config& config) :
|
||||
t_ctxbase<t_ctxunit>(schema, config),
|
||||
m_has_delta(false) {}
|
||||
|
||||
t_ctxunit::~t_ctxunit() = default;
|
||||
|
||||
void
|
||||
t_ctxunit::init() {
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::step_begin() {
|
||||
if (!m_init) {
|
||||
return;
|
||||
}
|
||||
|
||||
m_delta_pkeys.clear();
|
||||
m_rows_changed = false;
|
||||
m_columns_changed = false;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::step_end() {}
|
||||
|
||||
/**
|
||||
* @brief Notify the context with new data when the `t_gstate` master table is
|
||||
* not empty, and being updated with new data.
|
||||
*
|
||||
* @param flattened
|
||||
* @param delta
|
||||
* @param prev
|
||||
* @param curr
|
||||
* @param transitions
|
||||
* @param existed
|
||||
*/
|
||||
void
|
||||
t_ctxunit::notify(
|
||||
const t_data_table& flattened,
|
||||
const t_data_table& delta,
|
||||
const t_data_table& prev,
|
||||
const t_data_table& curr,
|
||||
const t_data_table& transitions,
|
||||
const t_data_table& existed
|
||||
) {
|
||||
t_uindex nrecs = flattened.size();
|
||||
|
||||
std::shared_ptr<const t_column> pkey_sptr =
|
||||
flattened.get_const_column("psp_pkey");
|
||||
std::shared_ptr<const t_column> op_sptr =
|
||||
flattened.get_const_column("psp_op");
|
||||
const t_column* pkey_col = pkey_sptr.get();
|
||||
const t_column* op_col = op_sptr.get();
|
||||
|
||||
bool delete_encountered = false;
|
||||
|
||||
for (t_uindex idx = 0; idx < nrecs; ++idx) {
|
||||
// pkeys are always integers >= 0 - no need to use internal
|
||||
// symtable to dereference.
|
||||
t_tscalar pkey = pkey_col->get_scalar(idx);
|
||||
std::uint8_t op_ = *(op_col->get_nth<std::uint8_t>(idx));
|
||||
t_op op = static_cast<t_op>(op_);
|
||||
|
||||
switch (op) {
|
||||
case OP_INSERT: {
|
||||
} break;
|
||||
case OP_DELETE: {
|
||||
delete_encountered = true;
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected OP");
|
||||
} break;
|
||||
}
|
||||
|
||||
// add the pkey for row delta
|
||||
add_delta_pkey(pkey);
|
||||
}
|
||||
|
||||
m_has_delta = !m_delta_pkeys.empty() || delete_encountered;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Notify the context with new data after the `t_gstate`'s master table
|
||||
* has been updated for the first time with data.
|
||||
*
|
||||
* @param flattened
|
||||
*/
|
||||
void
|
||||
t_ctxunit::notify(const t_data_table& flattened, bool is_registration) {
|
||||
t_uindex nrecs = flattened.size();
|
||||
std::shared_ptr<const t_column> pkey_sptr =
|
||||
flattened.get_const_column("psp_pkey");
|
||||
const t_column* pkey_col = pkey_sptr.get();
|
||||
|
||||
m_has_delta = true;
|
||||
|
||||
// During `_register_context`, no subscriber exists yet — skip the
|
||||
// hopscotch_set insertions no observer can see. A unit context has no
|
||||
// traversal/sort state to build either, so this is effectively O(1).
|
||||
if (is_registration) {
|
||||
return;
|
||||
}
|
||||
|
||||
// TODO: pkey and idx are equal, except idx is not a t_tscalar. I don't
|
||||
// think there is a difference between accessing the pkey column and
|
||||
// creating a brand new scalar, as get_scalar always returns a copy. We
|
||||
// could also simply store the row indices but there are some problems
|
||||
// with correctness that I couldn't nail down.
|
||||
for (t_uindex idx = 0; idx < nrecs; ++idx) {
|
||||
t_tscalar pkey = pkey_col->get_scalar(idx);
|
||||
|
||||
// Add primary key to track row delta
|
||||
add_delta_pkey(pkey);
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<t_tscalar, t_tscalar>
|
||||
t_ctxunit::get_min_max(const std::string& colname) const {
|
||||
auto col = m_gstate->get_table()->get_const_column(colname);
|
||||
auto rval = std::make_pair(mknone(), mknone());
|
||||
for (std::size_t i = 0; i < col->size(); i++) {
|
||||
t_tscalar val = col->get_scalar(i);
|
||||
if (!val.is_valid()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (rval.first.is_none() || (!val.is_none() && val < rval.first)) {
|
||||
rval.first = val;
|
||||
}
|
||||
|
||||
if (val > rval.second) {
|
||||
rval.second = val;
|
||||
}
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Given a start/end row and column, return the data for the subset.
|
||||
*
|
||||
* @param start_row
|
||||
* @param end_row
|
||||
* @param start_col
|
||||
* @param end_col
|
||||
* @return std::vector<t_tscalar>
|
||||
*/
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::get_data(
|
||||
t_index start_row, t_index end_row, t_index start_col, t_index end_col
|
||||
) const {
|
||||
t_uindex ctx_nrows = get_row_count();
|
||||
t_uindex ctx_ncols = get_column_count();
|
||||
|
||||
auto ext = sanitize_get_data_extents(
|
||||
ctx_nrows, ctx_ncols, start_row, end_row, start_col, end_col
|
||||
);
|
||||
|
||||
t_index num_rows = ext.m_erow - ext.m_srow;
|
||||
t_index stride = ext.m_ecol - ext.m_scol;
|
||||
std::vector<t_tscalar> values(num_rows * stride);
|
||||
|
||||
auto none = mknone();
|
||||
|
||||
const t_data_table& master_table = *(m_gstate->get_table());
|
||||
|
||||
for (t_index cidx = ext.m_scol; cidx < ext.m_ecol; ++cidx) {
|
||||
const std::string& colname = m_config.col_at(cidx);
|
||||
|
||||
std::vector<t_tscalar> out_data(num_rows);
|
||||
|
||||
// Read directly from the row indices on the table - they will
|
||||
// always correspond exactly.
|
||||
m_gstate->read_column(
|
||||
master_table, colname, start_row, end_row, out_data
|
||||
);
|
||||
|
||||
for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
|
||||
auto v = out_data[ridx - ext.m_srow];
|
||||
|
||||
// todo: fix null handling
|
||||
if (!v.is_valid()) {
|
||||
v.set(none);
|
||||
}
|
||||
|
||||
values[(ridx - ext.m_srow) * stride + (cidx - ext.m_scol)] = v;
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Given a vector of row indices, which may not be contiguous,
|
||||
* return the underlying data for these rows.
|
||||
*
|
||||
* @param rows a vector of row indices
|
||||
* @return std::vector<t_tscalar> a vector of scalars containing data
|
||||
*/
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::get_data(const std::vector<t_uindex>& rows) const {
|
||||
t_uindex stride = get_column_count();
|
||||
std::vector<t_tscalar> values(rows.size() * stride);
|
||||
|
||||
auto none = mknone();
|
||||
|
||||
const t_data_table& master_table = *(m_gstate->get_table());
|
||||
|
||||
for (t_uindex cidx = 0; cidx < stride; ++cidx) {
|
||||
std::vector<t_tscalar> out_data(rows.size());
|
||||
const std::string& colname = m_config.col_at(cidx);
|
||||
m_gstate->read_column(master_table, colname, rows, out_data);
|
||||
|
||||
for (t_uindex ridx = 0; ridx < rows.size(); ++ridx) {
|
||||
auto v = out_data[ridx];
|
||||
|
||||
if (!v.is_valid()) {
|
||||
v.set(none);
|
||||
}
|
||||
|
||||
values[(ridx)*stride + (cidx)] = v;
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::get_data(const std::vector<t_tscalar>& pkeys) const {
|
||||
t_uindex stride = get_column_count();
|
||||
std::vector<t_tscalar> values(pkeys.size() * stride);
|
||||
|
||||
auto none = mknone();
|
||||
|
||||
const t_data_table& master_table = *(m_gstate->get_table());
|
||||
|
||||
for (t_uindex cidx = 0; cidx < stride; ++cidx) {
|
||||
std::vector<t_tscalar> out_data(pkeys.size());
|
||||
const std::string& colname = m_config.col_at(cidx);
|
||||
|
||||
m_gstate->read_column(master_table, colname, pkeys, out_data);
|
||||
|
||||
for (t_uindex ridx = 0; ridx < pkeys.size(); ++ridx) {
|
||||
auto v = out_data[ridx];
|
||||
|
||||
if (!v.is_valid()) {
|
||||
v.set(none);
|
||||
}
|
||||
|
||||
values[(ridx)*stride + (cidx)] = v;
|
||||
}
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns a vector of primary keys for the specified cells,
|
||||
* reading from the gnode_state's master table instead of from a traversal.
|
||||
*
|
||||
* @param cells
|
||||
* @return std::vector<t_tscalar>
|
||||
*/
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
// Validate cells
|
||||
t_index num_rows = get_row_count();
|
||||
|
||||
for (const auto& cell : cells) {
|
||||
t_index ridx = cell.first;
|
||||
if (ridx >= num_rows) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
std::set<t_index> all_rows;
|
||||
|
||||
for (const auto& cell : cells) {
|
||||
all_rows.insert(cell.first);
|
||||
}
|
||||
|
||||
const t_data_table& master_table = *(m_gstate->get_table());
|
||||
std::shared_ptr<const t_column> pkey_sptr =
|
||||
master_table.get_const_column("psp_pkey");
|
||||
|
||||
std::vector<t_tscalar> rval(all_rows.size());
|
||||
|
||||
t_uindex i = 0;
|
||||
for (auto ridx : all_rows) {
|
||||
rval[i] = pkey_sptr->get_scalar(ridx);
|
||||
i++;
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns a string column name using the context's config.
|
||||
*
|
||||
* @param idx
|
||||
* @return t_tscalar
|
||||
*/
|
||||
t_tscalar
|
||||
t_ctxunit::get_column_name(t_index idx) {
|
||||
std::string empty;
|
||||
|
||||
if (idx >= get_column_count()) {
|
||||
return m_symtable.get_interned_tscalar(empty.c_str());
|
||||
}
|
||||
|
||||
return m_symtable.get_interned_tscalar(m_config.col_at(idx).c_str());
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Returns a `t_rowdelta` struct containing data from updated rows
|
||||
* and the updated row indices.
|
||||
*
|
||||
* @return t_rowdelta
|
||||
*/
|
||||
t_rowdelta
|
||||
t_ctxunit::get_row_delta() {
|
||||
bool rows_changed = m_rows_changed;
|
||||
std::vector<t_tscalar> pkey_vector(
|
||||
m_delta_pkeys.begin(), m_delta_pkeys.end()
|
||||
);
|
||||
|
||||
// Sort pkeys - they will always be integers >= 0, as the table has
|
||||
// no index set.
|
||||
std::sort(pkey_vector.begin(), pkey_vector.end());
|
||||
|
||||
std::vector<t_tscalar> data = get_data(pkey_vector);
|
||||
t_rowdelta rval(rows_changed, pkey_vector.size(), data);
|
||||
clear_deltas();
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
const tsl::hopscotch_set<t_tscalar>&
|
||||
t_ctxunit::get_delta_pkeys() const {
|
||||
return m_delta_pkeys;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctxunit::get_column_names() const {
|
||||
return m_schema.columns();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::reset() {
|
||||
m_has_delta = false;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctxunit::get_deltas_enabled() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::set_deltas_enabled(bool enabled_state) {}
|
||||
|
||||
t_index
|
||||
t_ctxunit::sidedness() const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctxunit::get_row_count() const {
|
||||
return m_gstate->num_rows();
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ctxunit::get_column_count() const {
|
||||
return m_config.get_num_columns();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::unity_get_row_data(t_uindex idx) const {
|
||||
return get_data(idx, idx + 1, 0, get_column_count());
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::unity_get_row_path(t_uindex idx) const {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ctxunit::unity_get_column_path(t_uindex idx) const {
|
||||
return {};
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctxunit::unity_get_row_depth(t_uindex ridx) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctxunit::unity_get_column_depth(t_uindex cidx) const {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::vector<std::string>
|
||||
t_ctxunit::unity_get_column_names() const {
|
||||
return get_column_names();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctxunit::unity_get_column_count() const {
|
||||
return get_column_count();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ctxunit::unity_get_row_count() const {
|
||||
return get_row_count();
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctxunit::unity_get_row_expanded(t_uindex idx) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctxunit::unity_get_column_expanded(t_uindex idx) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Mark a primary key as updated by adding it to the tracking set.
|
||||
*
|
||||
* @param pkey
|
||||
*/
|
||||
void
|
||||
t_ctxunit::add_delta_pkey(t_tscalar pkey) {
|
||||
m_delta_pkeys.insert(pkey);
|
||||
}
|
||||
|
||||
bool
|
||||
t_ctxunit::has_deltas() const {
|
||||
return m_has_delta;
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_ctxunit::get_column_dtype(t_uindex idx) const {
|
||||
if (idx >= static_cast<t_uindex>(get_column_count())) {
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
auto cname = m_config.col_at(idx);
|
||||
|
||||
if (!m_schema.has_column(cname)) {
|
||||
return DTYPE_NONE;
|
||||
}
|
||||
|
||||
return m_schema.get_dtype(cname);
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::clear_deltas() {
|
||||
m_has_delta = false;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_ctxunit::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_ctxunit<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
void
|
||||
t_ctxunit::pprint() const {}
|
||||
|
||||
} // end namespace perspective
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/data.h>
|
||||
|
||||
namespace perspective {
|
||||
t_data::t_data() = default;
|
||||
|
||||
t_data::t_data(const std::vector<t_tscalar>& data) : m_data(data) {}
|
||||
|
||||
const std::vector<t_tscalar>&
|
||||
t_data::data() const {
|
||||
return m_data;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>&
|
||||
t_data::data() {
|
||||
return m_data;
|
||||
}
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,201 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/data_slice.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
template <typename CTX_T>
|
||||
t_data_slice<CTX_T>::t_data_slice(
|
||||
std::shared_ptr<CTX_T> ctx,
|
||||
t_uindex start_row,
|
||||
t_uindex end_row,
|
||||
t_uindex start_col,
|
||||
t_uindex end_col,
|
||||
t_uindex row_offset,
|
||||
t_uindex col_offset,
|
||||
const std::vector<t_tscalar>& slice,
|
||||
const std::vector<std::vector<t_tscalar>>& column_names
|
||||
) :
|
||||
m_ctx(ctx),
|
||||
m_start_row(start_row),
|
||||
m_end_row(end_row),
|
||||
m_start_col(start_col),
|
||||
m_end_col(end_col),
|
||||
m_row_offset(row_offset),
|
||||
m_col_offset(col_offset),
|
||||
m_slice(slice),
|
||||
m_column_names(column_names) {
|
||||
m_stride = m_end_col - m_start_col;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_data_slice<CTX_T>::t_data_slice(
|
||||
std::shared_ptr<CTX_T> ctx,
|
||||
t_uindex start_row,
|
||||
t_uindex end_row,
|
||||
t_uindex start_col,
|
||||
t_uindex end_col,
|
||||
t_uindex row_offset,
|
||||
t_uindex col_offset,
|
||||
const std::vector<t_tscalar>& slice,
|
||||
const std::vector<std::vector<t_tscalar>>& column_names,
|
||||
const std::vector<t_uindex>& column_indices
|
||||
) :
|
||||
m_ctx(ctx),
|
||||
m_start_row(start_row),
|
||||
m_end_row(end_row),
|
||||
m_start_col(start_col),
|
||||
m_end_col(end_col),
|
||||
m_row_offset(row_offset),
|
||||
m_col_offset(col_offset),
|
||||
m_slice(slice),
|
||||
m_column_names(column_names),
|
||||
m_column_indices(column_indices) {
|
||||
m_stride = m_end_col - m_start_col;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_data_slice<CTX_T>::~t_data_slice() = default;
|
||||
|
||||
// Public API
|
||||
template <typename CTX_T>
|
||||
t_tscalar
|
||||
t_data_slice<CTX_T>::get(t_uindex ridx, t_uindex cidx) const {
|
||||
ridx += m_row_offset;
|
||||
t_uindex idx = get_slice_idx(ridx, cidx);
|
||||
t_tscalar rv;
|
||||
if (idx >= m_slice.size()) {
|
||||
rv.clear();
|
||||
} else {
|
||||
rv = m_slice.operator[](idx);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
std::vector<t_tscalar>
|
||||
t_data_slice<CTX_T>::get_pkeys(t_uindex ridx, t_uindex cidx) const {
|
||||
std::pair<t_uindex, t_uindex> pair{ridx, cidx};
|
||||
std::vector<std::pair<t_uindex, t_uindex>> vec{pair};
|
||||
return m_ctx->get_pkeys(vec);
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
std::vector<t_tscalar>
|
||||
t_data_slice<CTX_T>::get_column_slice(t_uindex cidx) const {
|
||||
std::vector<t_tscalar> column_data;
|
||||
column_data.reserve(m_end_row);
|
||||
|
||||
for (t_uindex ridx = 0; ridx < m_end_row; ++ridx) {
|
||||
ridx += m_row_offset;
|
||||
t_tscalar value = get(ridx, cidx);
|
||||
column_data.push_back(value);
|
||||
}
|
||||
|
||||
return column_data;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
std::vector<t_tscalar>
|
||||
t_data_slice<CTX_T>::get_row_path(t_uindex ridx) const {
|
||||
return m_ctx->unity_get_row_path(ridx);
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_uindex
|
||||
t_data_slice<CTX_T>::num_rows() const {
|
||||
return m_end_row - m_start_row;
|
||||
}
|
||||
|
||||
// Getters
|
||||
template <typename CTX_T>
|
||||
std::shared_ptr<CTX_T>
|
||||
t_data_slice<CTX_T>::get_context() const {
|
||||
return m_ctx;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
const std::vector<t_tscalar>&
|
||||
t_data_slice<CTX_T>::get_slice() const {
|
||||
return m_slice;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
const std::vector<std::vector<t_tscalar>>&
|
||||
t_data_slice<CTX_T>::get_column_names() const {
|
||||
return m_column_names;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
const std::vector<t_uindex>&
|
||||
t_data_slice<CTX_T>::get_column_indices() const {
|
||||
return m_column_indices;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
bool
|
||||
t_data_slice<CTX_T>::is_column_only() const {
|
||||
return false;
|
||||
}
|
||||
|
||||
template <>
|
||||
bool
|
||||
t_data_slice<t_ctx2>::is_column_only() const {
|
||||
auto config = m_ctx->get_config();
|
||||
return config.is_column_only();
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_uindex
|
||||
t_data_slice<CTX_T>::get_stride() const {
|
||||
return m_stride;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_uindex
|
||||
t_data_slice<CTX_T>::get_row_offset() const {
|
||||
return m_row_offset;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_uindex
|
||||
t_data_slice<CTX_T>::get_col_offset() const {
|
||||
return m_col_offset;
|
||||
}
|
||||
|
||||
template <typename CTX_T>
|
||||
t_get_data_extents
|
||||
t_data_slice<CTX_T>::get_data_extents() const {
|
||||
auto nrows = m_ctx->get_row_count();
|
||||
auto ncols = m_ctx->get_column_count();
|
||||
t_get_data_extents ext = sanitize_get_data_extents(
|
||||
nrows, ncols, m_start_row, m_end_row, m_start_col, m_end_col
|
||||
);
|
||||
return ext;
|
||||
}
|
||||
|
||||
// Private
|
||||
template <typename CTX_T>
|
||||
t_uindex
|
||||
t_data_slice<CTX_T>::get_slice_idx(t_uindex ridx, t_uindex cidx) const {
|
||||
t_uindex idx = (ridx - m_start_row) * m_stride + (cidx - m_start_col);
|
||||
return idx;
|
||||
}
|
||||
|
||||
// Explicitly instantiate data slice for each context
|
||||
template class t_data_slice<t_ctxunit>;
|
||||
template class t_data_slice<t_ctx0>;
|
||||
template class t_data_slice<t_ctx1>;
|
||||
template class t_data_slice<t_ctx2>;
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,936 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/raw_types.h>
|
||||
#include <perspective/data_table.h>
|
||||
#include <perspective/column.h>
|
||||
#include <perspective/storage.h>
|
||||
#include <perspective/scalar.h>
|
||||
#include <perspective/tracing.h>
|
||||
#include <perspective/utils.h>
|
||||
|
||||
#include <sstream>
|
||||
#include <utility>
|
||||
namespace perspective {
|
||||
|
||||
void
|
||||
t_data_table::set_capacity(t_uindex idx) {
|
||||
m_capacity = idx;
|
||||
#ifdef PSP_TABLE_VERIFY
|
||||
if (m_init) {
|
||||
for (auto c : m_columns) {
|
||||
c->verify_size();
|
||||
c->verify_size(m_capacity);
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
}
|
||||
|
||||
t_data_table::t_data_table(t_schema s, t_uindex init_cap) :
|
||||
m_schema(std::move(s)),
|
||||
m_size(0),
|
||||
m_backing_store(BACKING_STORE_MEMORY),
|
||||
m_init(false) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_CONSTRUCTOR("t_data_table");
|
||||
set_capacity(init_cap);
|
||||
}
|
||||
|
||||
t_data_table::t_data_table(
|
||||
std::string name,
|
||||
std::string dirname,
|
||||
t_schema s,
|
||||
t_uindex init_cap,
|
||||
t_backing_store backing_store
|
||||
) :
|
||||
m_name(std::move(name)),
|
||||
m_dirname(std::move(dirname)),
|
||||
m_schema(std::move(s)),
|
||||
m_size(0),
|
||||
m_backing_store(backing_store),
|
||||
m_init(false) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_CONSTRUCTOR("t_data_table");
|
||||
set_capacity(init_cap);
|
||||
}
|
||||
|
||||
// THIS CONSTRUCTOR INITS. Do not use in production.
|
||||
t_data_table::t_data_table(
|
||||
const t_schema& s, const std::vector<std::vector<t_tscalar>>& v
|
||||
) :
|
||||
m_schema(s),
|
||||
m_size(0),
|
||||
m_backing_store(BACKING_STORE_MEMORY),
|
||||
m_init(false) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_CONSTRUCTOR("t_data_table");
|
||||
auto ncols = s.size();
|
||||
PSP_VERBOSE_ASSERT(
|
||||
std::all_of(
|
||||
v.begin(),
|
||||
v.end(),
|
||||
[ncols](const std::vector<t_tscalar>& vec) {
|
||||
return vec.size() == ncols;
|
||||
}
|
||||
),
|
||||
"Mismatched row size found"
|
||||
);
|
||||
set_capacity(v.size());
|
||||
init();
|
||||
extend(v.size());
|
||||
std::vector<t_column*> cols = get_columns();
|
||||
for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
|
||||
auto* col = cols[cidx];
|
||||
for (t_uindex ridx = 0, loop_end = v.size(); ridx < loop_end; ++ridx) {
|
||||
col->set_scalar(ridx, v[ridx][cidx]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_data_table::~t_data_table() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_DESTRUCTOR("t_data_table");
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_data_table::name() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_name;
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::init(bool make_columns) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_INIT("t_data_table");
|
||||
m_columns = std::vector<std::shared_ptr<t_column>>(m_schema.size());
|
||||
|
||||
if (make_columns) {
|
||||
for (t_uindex idx = 0; idx < int(m_schema.size()); ++idx) {
|
||||
const std::string& colname = m_schema.m_columns[idx];
|
||||
t_dtype dtype = m_schema.m_types[idx];
|
||||
m_columns[idx] =
|
||||
make_column(colname, dtype, m_schema.m_status_enabled[idx]);
|
||||
m_columns[idx]->init();
|
||||
}
|
||||
}
|
||||
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::make_column(
|
||||
const std::string& colname, t_dtype dtype, bool status_enabled
|
||||
) {
|
||||
t_lstore_recipe a(
|
||||
m_dirname,
|
||||
m_name + std::string("_") + colname,
|
||||
m_capacity * get_dtype_size(dtype),
|
||||
m_backing_store
|
||||
);
|
||||
return std::make_shared<t_column>(dtype, status_enabled, a, m_capacity);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_data_table::num_columns() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_schema.size();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_data_table::num_rows() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_size;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_data_table::size() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return num_rows();
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_data_table::get_dtype(const std::string& colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_schema.get_dtype(colname);
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::get_column_safe(std::string_view colname) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx_safe(colname.data());
|
||||
if (idx == -1) {
|
||||
return nullptr;
|
||||
}
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::get_column(std::string_view colname) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx(colname.data());
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::get_column(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx(colname.data());
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::get_column_safe(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx_safe(colname.data());
|
||||
if (idx == -1) {
|
||||
return nullptr;
|
||||
}
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_data_table::get_const_column(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx(colname.data());
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_data_table::get_const_column_safe(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx_safe(colname.data());
|
||||
if (idx == -1) {
|
||||
return nullptr;
|
||||
}
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_data_table::get_const_column(t_uindex idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_data_table::get_const_column_safe(t_uindex idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
if (idx == -1) {
|
||||
return nullptr;
|
||||
}
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx];
|
||||
}
|
||||
|
||||
std::vector<t_column*>
|
||||
t_data_table::get_columns() {
|
||||
std::vector<t_column*> rval(m_columns.size());
|
||||
t_uindex idx = 0;
|
||||
for (const auto& c : m_columns) {
|
||||
ensure_col_resident(idx);
|
||||
rval[idx] = c.get();
|
||||
++idx;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<const t_column*>
|
||||
t_data_table::get_const_columns() const {
|
||||
std::vector<const t_column*> rval(m_columns.size());
|
||||
t_uindex idx = 0;
|
||||
for (const auto& c : m_columns) {
|
||||
ensure_col_resident(idx);
|
||||
rval[idx] = c.get();
|
||||
++idx;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::extend(t_uindex nelems) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
PSP_VERBOSE_ASSERT(m_init, "Table not inited");
|
||||
for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
|
||||
// Restore before reallocating: extend/reserve realloc `m_base`, which on
|
||||
// an evicted store would discard its disk-resident data.
|
||||
ensure_col_resident(idx);
|
||||
m_columns[idx]->extend_dtype(nelems);
|
||||
}
|
||||
m_size = std::max(nelems, m_size);
|
||||
set_capacity(std::max(nelems, m_capacity));
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::set_size(t_uindex size) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
|
||||
ensure_col_resident(idx);
|
||||
m_columns[idx]->set_size(size);
|
||||
}
|
||||
m_size = size;
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::set_table_size(t_uindex size) {
|
||||
m_size = size;
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::reserve(t_uindex capacity) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
|
||||
ensure_col_resident(idx);
|
||||
m_columns[idx]->reserve(capacity);
|
||||
}
|
||||
set_capacity(std::max(capacity, m_capacity));
|
||||
}
|
||||
|
||||
t_column*
|
||||
t_data_table::_get_column(std::string_view colname) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx(colname.data());
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx].get();
|
||||
}
|
||||
|
||||
const t_column*
|
||||
t_data_table::_get_const_column(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx(colname.data());
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx].get();
|
||||
}
|
||||
|
||||
const t_column*
|
||||
t_data_table::_get_const_column(t_uindex idx) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx].get();
|
||||
}
|
||||
|
||||
const t_column*
|
||||
t_data_table::_get_const_column_safe(std::string_view colname) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex idx = m_schema.get_colidx_safe(colname.data());
|
||||
if (idx == t_uindex(-1)) {
|
||||
return nullptr;
|
||||
}
|
||||
ensure_col_resident(idx);
|
||||
return m_columns[idx].get();
|
||||
}
|
||||
|
||||
const t_schema&
|
||||
t_data_table::get_schema() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_schema;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_data_table::flatten() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
PSP_VERBOSE_ASSERT(is_pkey_table(), "Not a pkeyed table");
|
||||
std::shared_ptr<t_data_table> flattened = std::make_shared<t_data_table>(
|
||||
"", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
flattened->init();
|
||||
flatten_body<std::shared_ptr<t_data_table>>(flattened);
|
||||
return flattened;
|
||||
}
|
||||
|
||||
bool
|
||||
t_data_table::is_pkey_table() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_schema.is_pkey();
|
||||
}
|
||||
|
||||
bool
|
||||
t_data_table::is_same_shape(t_data_table& tbl) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
return m_schema == tbl.m_schema;
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::pprint() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
pprint(size(), &std::cout);
|
||||
}
|
||||
|
||||
// void
|
||||
// t_data_table::pprint(const std::string& fname) const {
|
||||
|
||||
// PSP_TRACE_SENTINEL();
|
||||
// PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
// std::ofstream file;
|
||||
// file.open(fname);
|
||||
// pprint(size(), &file);
|
||||
// }
|
||||
|
||||
void
|
||||
t_data_table::pprint(t_uindex nrows, std::ostream* os) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (os == nullptr) {
|
||||
os = &std::cout;
|
||||
}
|
||||
|
||||
t_uindex nrows_ = nrows != 0U ? nrows : num_rows();
|
||||
nrows_ = std::min(nrows_, num_rows());
|
||||
|
||||
t_uindex ncols = num_columns();
|
||||
|
||||
std::vector<const t_column*> columns(ncols);
|
||||
for (t_uindex idx = 0; idx < ncols; ++idx) {
|
||||
columns[idx] = m_columns[idx].get();
|
||||
(*os) << m_schema.m_columns[idx] << ", ";
|
||||
}
|
||||
|
||||
(*os) << '\n';
|
||||
(*os) << "==========================" << '\n';
|
||||
|
||||
for (t_uindex ridx = 0; ridx < nrows_; ++ridx) {
|
||||
for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
|
||||
(*os) << columns[cidx]->get_scalar(ridx).to_string() << ", ";
|
||||
}
|
||||
(*os) << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::pprint(const std::vector<t_uindex>& vec) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_uindex nrows = vec.size();
|
||||
t_uindex ncols = num_columns();
|
||||
|
||||
std::vector<const t_column*> columns(ncols);
|
||||
for (t_uindex idx = 0; idx < ncols; ++idx) {
|
||||
columns[idx] = m_columns[idx].get();
|
||||
std::cout << m_schema.m_columns[idx] << ", ";
|
||||
}
|
||||
|
||||
std::cout << '\n';
|
||||
std::cout << "==========================" << '\n';
|
||||
|
||||
for (t_uindex ridx = 0; ridx < nrows; ++ridx) {
|
||||
for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
|
||||
std::cout << columns[cidx]->get_scalar(vec[ridx]) << ", ";
|
||||
}
|
||||
std::cout << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::append(const t_data_table& other) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
t_uindex cursize = size();
|
||||
|
||||
std::vector<const t_column*> src_cols;
|
||||
std::vector<t_column*> dst_cols;
|
||||
|
||||
src_cols.reserve(other.m_schema.m_columns.size());
|
||||
dst_cols.reserve(m_schema.m_columns.size());
|
||||
|
||||
std::set<std::string> incoming;
|
||||
|
||||
for (const auto& cname : other.m_schema.m_columns) {
|
||||
t_dtype col_dtype = get_column(cname)->get_dtype();
|
||||
t_dtype other_col_dtype = other.get_const_column(cname)->get_dtype();
|
||||
if (col_dtype != other_col_dtype) {
|
||||
std::stringstream ss;
|
||||
ss << "Mismatched dtypes for `" << cname
|
||||
<< "`: attempted to append column of dtype `"
|
||||
<< get_dtype_descr(other_col_dtype)
|
||||
<< "` to existing column of dtype `"
|
||||
<< get_dtype_descr(col_dtype) << "`" << '\n';
|
||||
std::cout << ss.str();
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str())
|
||||
}
|
||||
src_cols.push_back(other._get_const_column(cname));
|
||||
dst_cols.push_back(_get_column(cname));
|
||||
incoming.insert(cname);
|
||||
}
|
||||
t_uindex other_size = other.num_rows();
|
||||
|
||||
for (const auto& cname : m_schema.m_columns) {
|
||||
if (incoming.find(cname) == incoming.end()) {
|
||||
get_column(cname)->extend_dtype(cursize + other_size);
|
||||
}
|
||||
}
|
||||
|
||||
for (t_uindex colidx = 0; colidx < int(src_cols.size()); ++colidx) {
|
||||
dst_cols[colidx]->append(*(src_cols[colidx]));
|
||||
}
|
||||
|
||||
set_capacity(std::max(m_capacity, m_size + other.num_rows()));
|
||||
set_size(m_size + other.num_rows());
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::clear() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
for (const auto& m_column : m_columns) {
|
||||
m_column->clear();
|
||||
}
|
||||
m_size = 0;
|
||||
}
|
||||
|
||||
t_mask
|
||||
t_data_table::filter_cpp(
|
||||
t_filter_op combiner, const std::vector<t_fterm>& fterms_
|
||||
) const {
|
||||
auto* self = const_cast<t_data_table*>(this);
|
||||
auto fterms = fterms_;
|
||||
|
||||
t_mask mask(size());
|
||||
t_uindex fterm_size = fterms.size();
|
||||
std::vector<t_uindex> indices(fterm_size);
|
||||
std::vector<const t_column*> columns(fterm_size);
|
||||
|
||||
for (t_uindex idx = 0; idx < fterm_size; ++idx) {
|
||||
indices[idx] = m_schema.get_colidx(fterms[idx].m_colname);
|
||||
columns[idx] = _get_const_column(fterms[idx].m_colname);
|
||||
fterms[idx].coerce_numeric(columns[idx]->get_dtype());
|
||||
if (fterms[idx].m_use_interned) {
|
||||
t_tscalar& thr = fterms[idx].m_threshold;
|
||||
auto col = self->get_column(fterms[idx].m_colname);
|
||||
auto interned = col->get_interned(thr.get_char_ptr());
|
||||
thr.set(interned);
|
||||
}
|
||||
}
|
||||
|
||||
switch (combiner) {
|
||||
case FILTER_OP_AND: {
|
||||
t_tscalar cell_val;
|
||||
|
||||
for (t_uindex ridx = 0, rloop_end = size(); ridx < rloop_end;
|
||||
++ridx) {
|
||||
bool pass = true;
|
||||
|
||||
for (t_uindex cidx = 0; cidx < fterm_size; ++cidx) {
|
||||
if (!pass) {
|
||||
break;
|
||||
}
|
||||
|
||||
// TODO we can make this faster by not constructing these on
|
||||
// every iteration?
|
||||
|
||||
const auto& ft = fterms[cidx];
|
||||
bool tval;
|
||||
|
||||
if (ft.m_use_interned) {
|
||||
cell_val.set(*(columns[cidx]->get_nth<t_uindex>(ridx)));
|
||||
cell_val.set_status(*(columns[cidx]->get_nth_status(ridx
|
||||
)));
|
||||
} else {
|
||||
cell_val = columns[cidx]->get_scalar(ridx);
|
||||
}
|
||||
|
||||
tval = ft(cell_val);
|
||||
if (!tval) {
|
||||
pass = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
mask.set(ridx, pass);
|
||||
}
|
||||
} break;
|
||||
case FILTER_OP_OR: {
|
||||
for (t_uindex ridx = 0, rloop_end = size(); ridx < rloop_end;
|
||||
++ridx) {
|
||||
bool pass = false;
|
||||
for (t_uindex cidx = 0; cidx < fterm_size; ++cidx) {
|
||||
t_tscalar cell_val = columns[cidx]->get_scalar(ridx);
|
||||
if (fterms[cidx](cell_val)) {
|
||||
pass = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
mask.set(ridx, pass);
|
||||
}
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unknown filter op");
|
||||
} break;
|
||||
}
|
||||
|
||||
return mask;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_data_table::get_capacity() const {
|
||||
return m_capacity;
|
||||
}
|
||||
|
||||
t_data_table*
|
||||
t_data_table::clone_(const t_mask& mask) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_schema schema = m_schema;
|
||||
|
||||
auto* rval = new t_data_table("", "", schema, 5, BACKING_STORE_MEMORY);
|
||||
rval->init();
|
||||
|
||||
for (const auto& cname : schema.m_columns) {
|
||||
rval->set_column(cname, get_const_column(cname)->clone(mask));
|
||||
}
|
||||
|
||||
rval->set_size(mask.count());
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_data_table::clone(const t_mask& mask) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
auto* tbl = clone_(mask);
|
||||
return std::shared_ptr<t_data_table>(tbl);
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_data_table::clone() const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
t_schema schema = m_schema;
|
||||
auto rval =
|
||||
std::make_shared<t_data_table>("", "", schema, 5, BACKING_STORE_MEMORY);
|
||||
rval->init();
|
||||
|
||||
for (const auto& cname : schema.m_columns) {
|
||||
rval->set_column(cname, get_const_column(cname)->clone());
|
||||
}
|
||||
rval->set_size(size());
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_data_table::borrow(const std::vector<std::string>& columns) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
std::vector<t_dtype> dtypes;
|
||||
dtypes.reserve(columns.size());
|
||||
|
||||
for (const auto& col : columns) {
|
||||
dtypes.push_back(m_schema.get_dtype(col));
|
||||
}
|
||||
|
||||
t_schema borrowed_schema = t_schema(columns, dtypes);
|
||||
auto rval = std::make_shared<t_data_table>(
|
||||
"", "", borrowed_schema, 5, BACKING_STORE_MEMORY
|
||||
);
|
||||
rval->init();
|
||||
|
||||
for (const auto& cname : borrowed_schema.m_columns) {
|
||||
rval->set_column(cname, get_column(cname));
|
||||
}
|
||||
|
||||
rval->set_size(size());
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_data_table::join(const std::shared_ptr<t_data_table>& other_table) const {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (size() != other_table->size()) {
|
||||
std::stringstream ss;
|
||||
ss << "[t_data_table::join] Cannot join two tables of unequal sizes! "
|
||||
"Current size: "
|
||||
<< size() << ", size of other table: " << other_table->size()
|
||||
<< '\n';
|
||||
PSP_COMPLAIN_AND_ABORT(ss.str());
|
||||
}
|
||||
|
||||
// join both schemas
|
||||
t_schema schema = m_schema;
|
||||
const t_schema& other_schema = other_table->get_schema();
|
||||
std::vector<std::string> other_columns;
|
||||
|
||||
for (const std::string& column : other_schema.m_columns) {
|
||||
// Only append unique columns
|
||||
if (!schema.has_column(column)) {
|
||||
schema.add_column(column, other_schema.get_dtype(column));
|
||||
other_columns.push_back(column);
|
||||
}
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table> rval = std::make_shared<t_data_table>(
|
||||
"", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
|
||||
// init() without initializing the t_columns on the returned table, as
|
||||
// they will be immediately replaced by the columns from the tables
|
||||
// we are joining together.
|
||||
rval->init(false);
|
||||
|
||||
// borrow columns from the current table
|
||||
for (const std::string& column : m_schema.m_columns) {
|
||||
rval->set_column(column, get_column(column));
|
||||
}
|
||||
|
||||
// and the columns we need from the other table
|
||||
for (const std::string& column : other_columns) {
|
||||
rval->set_column(column, other_table->get_column(column));
|
||||
}
|
||||
|
||||
// Set size and capacity on the table only - don't mutate any of the
|
||||
// columns.
|
||||
rval->set_table_size(size());
|
||||
rval->set_capacity(std::max(get_capacity(), other_table->get_capacity()));
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::add_column_sptr(
|
||||
const std::string& name, t_dtype dtype, bool status_enabled
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (m_schema.has_column(name)) {
|
||||
return m_columns.at(m_schema.get_colidx(name));
|
||||
}
|
||||
m_schema.add_column(name, dtype);
|
||||
m_columns.push_back(make_column(name, dtype, status_enabled));
|
||||
m_columns.back()->init();
|
||||
m_columns.back()->reserve(
|
||||
std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
|
||||
);
|
||||
m_columns.back()->set_size(size());
|
||||
return m_columns.back();
|
||||
}
|
||||
|
||||
t_column*
|
||||
t_data_table::add_column(
|
||||
const std::string& name, t_dtype dtype, bool status_enabled
|
||||
) {
|
||||
return add_column_sptr(name, dtype, status_enabled).get();
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::drop_column(const std::string& name) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (m_schema.has_column(name)) {
|
||||
auto idx = m_schema.get_colidx(name);
|
||||
auto col = m_columns[idx];
|
||||
// FIXME: make sure that we can erase columns from m_schema eventually.
|
||||
m_columns[idx]->clear();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::promote_column(
|
||||
std::string_view col_name,
|
||||
t_dtype new_dtype,
|
||||
std::int32_t iter_limit,
|
||||
bool fill
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
const auto* name = col_name.data();
|
||||
|
||||
if (!m_schema.has_column(name)) {
|
||||
std::cout << "Cannot promote a column that does not exist." << '\n';
|
||||
return;
|
||||
}
|
||||
|
||||
t_dtype current_dtype = m_schema.get_dtype(name);
|
||||
if (current_dtype == new_dtype) {
|
||||
return;
|
||||
}
|
||||
|
||||
t_uindex idx = m_schema.get_colidx(name);
|
||||
std::shared_ptr<t_column> current_col = m_columns[idx];
|
||||
|
||||
// create the new column and copy data
|
||||
std::shared_ptr<t_column> promoted_col =
|
||||
make_column(name, new_dtype, current_col->is_status_enabled());
|
||||
promoted_col->init();
|
||||
promoted_col->reserve(
|
||||
std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
|
||||
);
|
||||
promoted_col->set_size(size());
|
||||
|
||||
if (fill) {
|
||||
for (auto i = 0; i < iter_limit; ++i) {
|
||||
switch (new_dtype) {
|
||||
case DTYPE_INT64: {
|
||||
auto* val = current_col->get_nth<std::int32_t>(i);
|
||||
auto fval = static_cast<std::int64_t>(*val);
|
||||
promoted_col->set_nth(i, fval);
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
auto* val = current_col->get_nth<std::int32_t>(i);
|
||||
auto fval = static_cast<double>(*val);
|
||||
promoted_col->set_nth(i, fval);
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
auto* val = current_col->get_nth<std::int32_t>(i);
|
||||
std::string fval = std::to_string(*val);
|
||||
promoted_col->set_nth(i, fval);
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT(
|
||||
"Columns can only be promoted to "
|
||||
"int64, float64, or string type."
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// finally, mutate schema and columns
|
||||
m_schema.retype_column(name, new_dtype);
|
||||
set_column(idx, promoted_col);
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::set_column(t_uindex idx, std::shared_ptr<t_column> col) {
|
||||
m_columns[idx] = std::move(col);
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::set_column(
|
||||
const std::string& name, std::shared_ptr<t_column> col
|
||||
) {
|
||||
t_uindex idx = m_schema.get_colidx(name);
|
||||
set_column(idx, std::move(col));
|
||||
}
|
||||
|
||||
t_column*
|
||||
t_data_table::clone_column(
|
||||
const std::string& existing_col, const std::string& new_colname
|
||||
) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
|
||||
|
||||
if (!m_schema.has_column(existing_col)) {
|
||||
std::cout << "Cannot clone non existing column: " << existing_col
|
||||
<< '\n';
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
t_uindex idx = m_schema.get_colidx(existing_col);
|
||||
|
||||
m_schema.add_column(new_colname, m_columns[idx]->get_dtype());
|
||||
m_columns.push_back(m_columns[idx]->clone());
|
||||
m_columns.back()->reserve(
|
||||
std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
|
||||
);
|
||||
m_columns.back()->set_size(size());
|
||||
return m_columns.back().get();
|
||||
}
|
||||
|
||||
std::string
|
||||
t_data_table::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_data_table<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::verify() const {
|
||||
for (const auto& c : m_columns) {
|
||||
c->verify_size(m_capacity);
|
||||
c->verify();
|
||||
}
|
||||
|
||||
for (const auto& c : m_columns) {
|
||||
PSP_VERBOSE_ASSERT(
|
||||
c, || (size() == c->size()), "Ragged table encountered"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_data_table::reset() {
|
||||
for (const auto& m_column : m_columns) {
|
||||
if (m_column->get_dtype() == DTYPE_OBJECT) {
|
||||
m_column->clear_objects();
|
||||
}
|
||||
m_column->clear();
|
||||
}
|
||||
m_size = 0;
|
||||
m_capacity = DEFAULT_EMPTY_CAPACITY;
|
||||
init();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_data_table::get_scalvec() const {
|
||||
auto nrows = size();
|
||||
auto cols = get_const_columns();
|
||||
auto ncols = cols.size();
|
||||
std::vector<t_tscalar> rv;
|
||||
rv.reserve(nrows * ncols);
|
||||
for (t_uindex idx = 0; idx < nrows; ++idx) {
|
||||
for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
|
||||
rv.push_back(cols[cidx]->get_scalar(idx));
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_column>
|
||||
t_data_table::operator[](const std::string& name) {
|
||||
if (!m_schema.has_column(name)) {
|
||||
return {nullptr};
|
||||
}
|
||||
return m_columns[m_schema.get_colidx(name)];
|
||||
}
|
||||
|
||||
bool
|
||||
operator==(const t_data_table& lhs, const t_data_table& rhs) {
|
||||
return lhs.get_scalvec() == rhs.get_scalvec();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,172 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/date.h>
|
||||
#include <perspective/time.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_date::t_date() : m_storage(0) {}
|
||||
|
||||
void
|
||||
t_date::set_psp_date(t_uindex dt)
|
||||
|
||||
{
|
||||
m_storage = t_rawtype(dt);
|
||||
}
|
||||
|
||||
t_date::t_date(std::int16_t year, std::int8_t month, std::int8_t day) {
|
||||
set_year_month_day(year, month, day);
|
||||
}
|
||||
|
||||
void
|
||||
t_date::set_year_month_day(
|
||||
std::int16_t year, std::int8_t month, std::int8_t day
|
||||
) {
|
||||
set_year(year);
|
||||
set_month(month);
|
||||
set_day(day);
|
||||
}
|
||||
|
||||
void
|
||||
t_date::set_year(std::int16_t year) {
|
||||
m_storage = (m_storage & ~YEAR_MASK) | (year << YEAR_SHIFT);
|
||||
}
|
||||
void
|
||||
t_date::set_month(std::int8_t month) {
|
||||
m_storage = (m_storage & ~MONTH_MASK) | (month << MONTH_SHIFT);
|
||||
}
|
||||
void
|
||||
t_date::set_day(std::int8_t day) {
|
||||
m_storage = (m_storage & ~DAY_MASK) | (day << DAY_SHIFT);
|
||||
}
|
||||
|
||||
t_date::t_date(std::uint32_t raw_val) : m_storage(raw_val) {}
|
||||
|
||||
std::uint32_t
|
||||
t_date::raw_value() const {
|
||||
return m_storage;
|
||||
}
|
||||
// Index such that
|
||||
// a.consecutive_day_idx()-b.consecutive_day_idx() is
|
||||
// number of days a is after b.
|
||||
//(Start point is unspecified, may not be stable and
|
||||
// only works for dates after 1900AD.)
|
||||
std::int32_t
|
||||
t_date::consecutive_day_idx() const {
|
||||
std::int32_t m = month();
|
||||
std::int32_t y = year();
|
||||
std::int32_t yP = y - 1;
|
||||
std::int32_t leap_selector =
|
||||
(y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 1 : 0;
|
||||
return day() + 365 * y + yP / 4 - yP / 100 + yP / 400
|
||||
+ CUMULATIVE_DAYS[leap_selector][m - 1];
|
||||
}
|
||||
|
||||
std::int64_t
|
||||
t_date::as_epoch_ms() const {
|
||||
// `to_gmtime` months are [1-12], `t_date` months are [0-11]
|
||||
return to_gmtime(year(), month() + 1, day(), 0, 0, 0) * 1000;
|
||||
}
|
||||
|
||||
t_date
|
||||
t_date::from_epoch_ms(std::int64_t ms) {
|
||||
std::tm t = gmtime_from_epoch_ms(ms);
|
||||
return {
|
||||
static_cast<std::int16_t>(t.tm_year + 1900),
|
||||
static_cast<std::int8_t>(t.tm_mon),
|
||||
static_cast<std::int8_t>(t.tm_mday)
|
||||
};
|
||||
}
|
||||
|
||||
t_date
|
||||
from_consecutive_day_idx(std::int32_t idx) {
|
||||
auto y = static_cast<std::int32_t>(idx / 365.2425);
|
||||
std::int32_t yP = y - 1;
|
||||
std::int32_t idx_year_removed =
|
||||
idx - (y * 365 + yP / 4 - yP / 100 + yP / 400);
|
||||
std::int32_t tgt_substraction =
|
||||
(y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 366 : 365;
|
||||
if (idx_year_removed > tgt_substraction) {
|
||||
idx_year_removed -= tgt_substraction;
|
||||
y += 1;
|
||||
}
|
||||
std::int32_t yearkind =
|
||||
(y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 1 : 0;
|
||||
const std::int32_t* const pos = std::lower_bound(
|
||||
CUMULATIVE_DAYS[yearkind],
|
||||
CUMULATIVE_DAYS[yearkind] + 13,
|
||||
idx_year_removed
|
||||
)
|
||||
- 1;
|
||||
// NOLINTNEXTLINE
|
||||
return t_date(
|
||||
y,
|
||||
std::distance(CUMULATIVE_DAYS[yearkind], pos) + 1,
|
||||
idx_year_removed - *pos /*+1*/
|
||||
);
|
||||
}
|
||||
|
||||
bool
|
||||
operator<(const t_date& a, const t_date& b) {
|
||||
return a.m_storage < b.m_storage;
|
||||
}
|
||||
bool
|
||||
operator<=(const t_date& a, const t_date& b) {
|
||||
return a.m_storage <= b.m_storage;
|
||||
}
|
||||
bool
|
||||
operator>(const t_date& a, const t_date& b) {
|
||||
return a.m_storage > b.m_storage;
|
||||
}
|
||||
bool
|
||||
operator>=(const t_date& a, const t_date& b) {
|
||||
return a.m_storage >= b.m_storage;
|
||||
}
|
||||
bool
|
||||
operator==(const t_date& a, const t_date& b) {
|
||||
return a.m_storage == b.m_storage;
|
||||
}
|
||||
bool
|
||||
operator!=(const t_date& a, const t_date& b) {
|
||||
return a.m_storage != b.m_storage;
|
||||
}
|
||||
|
||||
std::int32_t
|
||||
t_date::year() const {
|
||||
return ((m_storage & YEAR_MASK) >> YEAR_SHIFT);
|
||||
}
|
||||
std::int32_t
|
||||
t_date::month() const {
|
||||
return ((m_storage & MONTH_MASK) >> MONTH_SHIFT);
|
||||
}
|
||||
std::int32_t
|
||||
t_date::day() const {
|
||||
return ((m_storage & DAY_MASK) >> DAY_SHIFT);
|
||||
}
|
||||
std::string
|
||||
t_date::str() const {
|
||||
std::stringstream ss;
|
||||
ss << year() << "-" << str_(month() + 1) << "-" << str_(day());
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
namespace std {
|
||||
std::ostream&
|
||||
operator<<(std::ostream& os, const perspective::t_date& t) {
|
||||
os << t.str();
|
||||
return os;
|
||||
}
|
||||
} // namespace std
|
||||
@@ -0,0 +1,54 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <sstream>
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/date_parser.h>
|
||||
#include <locale>
|
||||
#include <iomanip>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
// Milliseconds & timezones are not currently handled
|
||||
const std::string t_date_parser::VALID_FORMATS[12] = {
|
||||
"%Y%m%dT%H%M%S", // ISO "%Y%m%dT%H%M%S%F%q"
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S", // ISO extended
|
||||
"%A, %d %b %Y %H:%M:%S", // RFC 0822
|
||||
"%Y-%m-%d\\%H:%M:%S"
|
||||
"%m-%d-%Y",
|
||||
"%m/%d/%Y",
|
||||
"%m-%d-%Y",
|
||||
"%m %d %Y",
|
||||
"%m/%d/%Y",
|
||||
"%m/%d/%y",
|
||||
"%d %m %Y"
|
||||
};
|
||||
|
||||
t_date_parser::t_date_parser() {}
|
||||
|
||||
bool
|
||||
t_date_parser::is_valid(std::string const& datestring) {
|
||||
for (const std::string& fmt : VALID_FORMATS) {
|
||||
if (fmt != "") {
|
||||
std::tm t = {};
|
||||
std::stringstream ss(datestring);
|
||||
ss.imbue(std::locale::classic());
|
||||
ss >> std::get_time(&t, fmt.c_str());
|
||||
if (!ss.fail()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,46 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/dense_nodes.h>
|
||||
|
||||
namespace std {
|
||||
|
||||
std::ostream&
|
||||
operator<<(std::ostream& os, const perspective::t_dense_tnode& s) {
|
||||
std::cout << "t_dtnode<idx:" << s.m_idx << " pidx:" << s.m_pidx
|
||||
<< " fcidx:" << s.m_fcidx << " nchild:" << s.m_nchild
|
||||
<< " flidx:" << s.m_flidx << " nleaves:" << s.m_nleaves << ">";
|
||||
return os;
|
||||
}
|
||||
} // namespace std
|
||||
namespace perspective {
|
||||
|
||||
void
|
||||
fill_dense_tnode(
|
||||
t_dense_tnode* node,
|
||||
t_uindex idx,
|
||||
t_uindex pidx,
|
||||
t_uindex fcidx,
|
||||
t_uindex nchild,
|
||||
t_uindex flidx,
|
||||
t_uindex nleaves
|
||||
) {
|
||||
node->m_idx = idx;
|
||||
node->m_pidx = pidx;
|
||||
node->m_fcidx = fcidx;
|
||||
node->m_nchild = nchild;
|
||||
node->m_flidx = flidx;
|
||||
node->m_nleaves = nleaves;
|
||||
}
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,503 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/dense_tree.h>
|
||||
#include <perspective/filter.h>
|
||||
#include <perspective/node_processor.h>
|
||||
#include <perspective/comparators.h>
|
||||
#include <perspective/sort_specification.h>
|
||||
#include <perspective/data_table.h>
|
||||
|
||||
#include <numeric>
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_dtree::t_dtree(
|
||||
t_dssptr ds,
|
||||
const std::vector<t_pivot>& pivots,
|
||||
const std::vector<std::pair<std::string, std::string>>& sortby_colvec
|
||||
) :
|
||||
m_levels_pivoted(0),
|
||||
m_ds(std::move(ds)),
|
||||
m_pivots(pivots),
|
||||
m_nidx(0),
|
||||
m_backing_store(BACKING_STORE_MEMORY),
|
||||
m_init(false),
|
||||
m_sortby_colvec(sortby_colvec) {}
|
||||
|
||||
t_uindex
|
||||
t_dtree::size() const {
|
||||
return m_nodes.size();
|
||||
}
|
||||
|
||||
std::pair<t_index, t_index>
|
||||
t_dtree::get_level_markers(t_uindex idx) const {
|
||||
PSP_VERBOSE_ASSERT(idx < m_levels.size(), "Unexpected lvlidx");
|
||||
return m_levels[idx];
|
||||
}
|
||||
|
||||
t_dtree::t_dtree(
|
||||
std::string dirname,
|
||||
t_dssptr ds,
|
||||
const std::vector<t_pivot>& pivots,
|
||||
t_backing_store backing_store,
|
||||
const std::vector<std::pair<std::string, std::string>>& sortby_colvec
|
||||
) :
|
||||
m_dirname(std::move(dirname)),
|
||||
m_levels_pivoted(0),
|
||||
m_ds(std::move(ds)),
|
||||
m_pivots(pivots),
|
||||
m_nidx(0),
|
||||
m_backing_store(backing_store),
|
||||
m_init(false),
|
||||
m_sortby_colvec(sortby_colvec) {}
|
||||
|
||||
void
|
||||
t_dtree::init() {
|
||||
t_lstore_recipe leaf_args(
|
||||
m_dirname, leaves_colname(), DEFAULT_CAPACITY, m_backing_store
|
||||
);
|
||||
m_leaves = t_column(DTYPE_UINT64, false, leaf_args, DEFAULT_CAPACITY);
|
||||
m_leaves.init();
|
||||
|
||||
t_lstore_recipe node_args(
|
||||
m_dirname, nodes_colname(), DEFAULT_CAPACITY, m_backing_store
|
||||
);
|
||||
|
||||
m_values = std::vector<t_column>(m_pivots.size() + 1);
|
||||
|
||||
m_has_sortby = std::vector<bool>(m_values.size());
|
||||
m_has_sortby[0] = false;
|
||||
|
||||
m_sortby_columns.clear();
|
||||
for (auto& sidx : m_sortby_colvec) {
|
||||
m_sortby_columns[sidx.first] = sidx.second;
|
||||
}
|
||||
|
||||
t_lstore_recipe root_args(
|
||||
m_dirname, values_colname("_root_"), DEFAULT_CAPACITY, m_backing_store
|
||||
);
|
||||
|
||||
m_values[0] = t_column(DTYPE_STR, true, leaf_args, DEFAULT_CAPACITY);
|
||||
m_values[0].init();
|
||||
|
||||
m_sortby_dpthcol.emplace_back("");
|
||||
|
||||
for (t_uindex idx = 0, loop_end = m_pivots.size(); idx < loop_end; ++idx) {
|
||||
auto colname = m_pivots[idx].colname();
|
||||
t_lstore_recipe leaf_args(
|
||||
m_dirname,
|
||||
values_colname(colname),
|
||||
DEFAULT_CAPACITY,
|
||||
m_backing_store
|
||||
);
|
||||
|
||||
auto siter = m_sortby_columns.find(colname);
|
||||
bool has_sortby =
|
||||
siter != m_sortby_columns.end() && siter->second != colname;
|
||||
m_has_sortby[idx + 1] = has_sortby;
|
||||
std::string sortby_column = has_sortby ? siter->second : colname;
|
||||
m_sortby_dpthcol.push_back(sortby_column);
|
||||
t_dtype dtype = m_ds->get_dtype(colname);
|
||||
m_values[idx + 1] = t_column(dtype, true, leaf_args, DEFAULT_CAPACITY);
|
||||
m_values[idx + 1].init();
|
||||
}
|
||||
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
std::string
|
||||
t_dtree::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << m_ds->name() << "_tree_" << this;
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::string
|
||||
t_dtree::leaves_colname() const {
|
||||
return repr() + std::string("_leaves");
|
||||
}
|
||||
|
||||
std::string
|
||||
t_dtree::nodes_colname() const {
|
||||
return repr() + std::string("_nodes");
|
||||
}
|
||||
|
||||
std::string
|
||||
t_dtree::values_colname(const std::string& tbl_colname) const {
|
||||
return repr() + std::string("_valuespan_") + tbl_colname;
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree::check_pivot(const t_filter& filter, t_uindex level) {
|
||||
if (level <= m_levels_pivoted) {
|
||||
return;
|
||||
}
|
||||
|
||||
PSP_VERBOSE_ASSERT(
|
||||
level <= m_pivots.size() + 1, "Erroneous level passed in"
|
||||
);
|
||||
|
||||
pivot(filter, level);
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree::pivot(const t_filter& filter, t_uindex level) {
|
||||
if (level <= m_levels_pivoted) {
|
||||
return;
|
||||
}
|
||||
|
||||
PSP_VERBOSE_ASSERT(
|
||||
level <= m_pivots.size() + 1, "Erroneous level passed in"
|
||||
);
|
||||
|
||||
t_uindex ncols = m_pivots.size();
|
||||
|
||||
t_uindex nidx = m_nidx;
|
||||
|
||||
t_uindex nrows;
|
||||
const t_mask* mask = nullptr;
|
||||
|
||||
if (ncols > 0) {
|
||||
if (filter.has_filter()) {
|
||||
nrows = filter.count();
|
||||
mask = filter.cmask().get();
|
||||
} else {
|
||||
nrows = m_ds->num_rows();
|
||||
}
|
||||
} else {
|
||||
nrows = m_pivots.empty() ? m_ds->num_rows() : 1;
|
||||
}
|
||||
|
||||
t_uindex nbidx;
|
||||
t_uindex neidx;
|
||||
|
||||
if (m_levels_pivoted == 0) {
|
||||
m_leaves.extend<t_uindex>(nrows);
|
||||
auto* leaves = m_leaves.get_nth<t_uindex>(0);
|
||||
std::iota(leaves, leaves + nrows, t_uindex(0));
|
||||
|
||||
nbidx = 0;
|
||||
neidx = 1;
|
||||
} else {
|
||||
nbidx = m_levels[m_levels_pivoted].first;
|
||||
neidx = m_levels[m_levels_pivoted].second;
|
||||
}
|
||||
|
||||
for (t_uindex pidx = m_levels_pivoted; pidx < level; pidx++) {
|
||||
const t_column* pivcol;
|
||||
if (pidx == 0) {
|
||||
m_nodes.emplace_back();
|
||||
t_tnode* root = &m_nodes.back();
|
||||
fill_dense_tnode(root, nidx, nidx, 1, 0, 0, nrows);
|
||||
nidx++;
|
||||
m_values[0].push_back(std::string("Grand Aggregate"));
|
||||
m_levels.emplace_back(nbidx, neidx);
|
||||
} else {
|
||||
const t_pivot& pivot = m_pivots[pidx - 1];
|
||||
std::string pivot_colname = pivot.colname();
|
||||
pivcol = m_ds->_get_const_column(pivot_colname);
|
||||
t_dtype piv_dtype = pivcol->get_dtype();
|
||||
|
||||
t_uindex next_neidx = 0;
|
||||
|
||||
switch (piv_dtype) {
|
||||
case DTYPE_STR: {
|
||||
next_neidx = t_pivot_processor<DTYPE_STR>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT64: {
|
||||
next_neidx = t_pivot_processor<DTYPE_INT64>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
next_neidx = t_pivot_processor<DTYPE_INT32>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
next_neidx = t_pivot_processor<DTYPE_INT16>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
next_neidx = t_pivot_processor<DTYPE_INT8>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
next_neidx = t_pivot_processor<DTYPE_UINT64>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
next_neidx = t_pivot_processor<DTYPE_UINT32>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
next_neidx = t_pivot_processor<DTYPE_UINT16>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
next_neidx = t_pivot_processor<DTYPE_UINT8>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
next_neidx = t_pivot_processor<DTYPE_FLOAT64>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
next_neidx = t_pivot_processor<DTYPE_FLOAT32>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
|
||||
next_neidx = t_pivot_processor<DTYPE_BOOL>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_TIME: {
|
||||
next_neidx = t_pivot_processor<DTYPE_INT64>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
case DTYPE_DATE: {
|
||||
next_neidx = t_pivot_processor<DTYPE_UINT32>()(
|
||||
pivcol,
|
||||
&m_nodes,
|
||||
&(m_values[pidx]),
|
||||
&m_leaves,
|
||||
nbidx,
|
||||
neidx,
|
||||
mask
|
||||
);
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Not supported yet");
|
||||
} break;
|
||||
}
|
||||
nbidx = neidx;
|
||||
neidx = next_neidx;
|
||||
m_levels.emplace_back(nbidx, neidx);
|
||||
}
|
||||
|
||||
m_levels_pivoted = pidx;
|
||||
}
|
||||
|
||||
m_nidx = neidx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_dtree::get_depth(t_index idx) const {
|
||||
return get_span_index(idx).first;
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree::pprint(const t_filter& filter) const {
|
||||
std::string indent(" ");
|
||||
|
||||
for (auto idx : dfs()) {
|
||||
t_uindex depth = get_depth(idx);
|
||||
|
||||
for (t_uindex didx = 0; didx < depth; ++didx) {
|
||||
std::cout << indent;
|
||||
}
|
||||
|
||||
const t_dense_tnode* node = get_node_ptr(idx);
|
||||
|
||||
std::cout << get_value(filter, idx) << " idx => " << node->m_idx
|
||||
<< " pidx => " << node->m_pidx << " fcidx => "
|
||||
<< node->m_fcidx << " nchild => " << node->m_nchild
|
||||
<< " flidx => " << node->m_flidx << " nleaves => "
|
||||
<< node->m_nleaves << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_dtree::last_level() const {
|
||||
return m_pivots.size();
|
||||
}
|
||||
|
||||
const t_dtree::t_tnode*
|
||||
t_dtree::get_node_ptr(t_index nidx) const {
|
||||
return &m_nodes.at(nidx);
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_dtree::_get_value(const t_filter& filter, t_index nidx, bool sort_value)
|
||||
const {
|
||||
|
||||
std::pair<t_uindex, t_uindex> spi = get_span_index(nidx);
|
||||
t_index dpth = spi.first;
|
||||
t_uindex scalar_idx = spi.second;
|
||||
|
||||
if (sort_value || nidx == 0) {
|
||||
const t_column& col = m_values[dpth];
|
||||
return col.get_scalar(scalar_idx);
|
||||
}
|
||||
const std::string& colname = m_sortby_dpthcol[dpth];
|
||||
const t_column& col = *(m_ds->get_const_column(colname));
|
||||
const auto* node = get_node_ptr(nidx);
|
||||
t_uindex lfidx = *(m_leaves.get_nth<t_uindex>(node->m_flidx));
|
||||
return col.get_scalar(lfidx);
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_dtree::get_value(const t_filter& filter, t_index nidx) const {
|
||||
return _get_value(filter, nidx, true);
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_dtree::get_sortby_value(const t_filter& filter, t_index nidx) const {
|
||||
return _get_value(filter, nidx, false);
|
||||
}
|
||||
|
||||
std::pair<t_uindex, t_uindex>
|
||||
t_dtree::get_span_index(t_index idx) const {
|
||||
for (t_uindex i = 0, loop_end = m_levels.size(); i < loop_end; i++) {
|
||||
t_index bidx = m_levels[i].first;
|
||||
t_index eidx = m_levels[i].second;
|
||||
|
||||
if ((idx >= bidx) && (idx < eidx)) {
|
||||
return std::pair<t_uindex, t_uindex>(i, idx - bidx);
|
||||
}
|
||||
}
|
||||
|
||||
PSP_COMPLAIN_AND_ABORT("Reached unreachable.");
|
||||
return std::pair<t_uindex, t_uindex>(0, 0);
|
||||
}
|
||||
|
||||
const t_column*
|
||||
t_dtree::get_leaf_cptr() const {
|
||||
return &m_leaves;
|
||||
}
|
||||
|
||||
t_bfs_iter<t_dtree>
|
||||
t_dtree::bfs() const {
|
||||
return {this};
|
||||
}
|
||||
|
||||
t_dfs_iter<t_dtree>
|
||||
t_dtree::dfs() const {
|
||||
return {this};
|
||||
}
|
||||
|
||||
t_index
|
||||
t_dtree::get_parent(t_index idx) const {
|
||||
const t_tnode* n = get_node_ptr(idx);
|
||||
return n->m_pidx;
|
||||
}
|
||||
|
||||
const std::vector<t_pivot>&
|
||||
t_dtree::get_pivots() const {
|
||||
return m_pivots;
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree::get_child_indices(t_index nidx, std::vector<t_index>& v) const {
|
||||
const auto* nptr = get_node_ptr(nidx);
|
||||
for (t_index idx = nptr->m_fcidx + nptr->m_nchild - 1,
|
||||
loop_end = nptr->m_fcidx;
|
||||
idx >= loop_end;
|
||||
--idx) {
|
||||
v.push_back(idx);
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,290 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <iomanip>
|
||||
#include <utility>
|
||||
#include <perspective/dense_tree_context.h>
|
||||
#include <perspective/dependency.h>
|
||||
#include <perspective/schema.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_dtree_ctx::t_dtree_ctx(
|
||||
std::shared_ptr<const t_data_table> strands,
|
||||
std::shared_ptr<const t_data_table> strand_deltas,
|
||||
const t_dtree& tree,
|
||||
const std::vector<t_aggspec>& aggspecs
|
||||
) :
|
||||
m_strands(std::move(std::move(strands))),
|
||||
m_strand_deltas(std::move(std::move(strand_deltas))),
|
||||
m_tree(tree),
|
||||
m_aggspecs(aggspecs),
|
||||
m_init(false) {
|
||||
std::vector<t_dep> depvec = {t_dep("psp_strand_count", DEPTYPE_COLUMN)};
|
||||
|
||||
m_aggspecs.emplace_back("psp_strand_count_sum", AGGTYPE_SUM, depvec);
|
||||
|
||||
t_uindex aggidx = 0;
|
||||
for (const auto& spec : m_aggspecs) {
|
||||
m_aggspecmap[spec.name()] = aggidx;
|
||||
++aggidx;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree_ctx::init() {
|
||||
build_aggregates();
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_dtree_ctx::get_num_aggcols() const {
|
||||
return m_aggspecs.size();
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree_ctx::build_aggregates() {
|
||||
std::vector<std::string> columns;
|
||||
std::vector<t_dtype> dtypes;
|
||||
|
||||
const t_schema& delta_schema = m_strand_deltas->get_schema();
|
||||
|
||||
for (const auto& spec : m_aggspecs) {
|
||||
auto cinfo = spec.get_output_specs(delta_schema);
|
||||
for (const auto& ci : cinfo) {
|
||||
PSP_VERBOSE_ASSERT(
|
||||
ci.m_type != DTYPE_NONE, "NULL type encountered"
|
||||
);
|
||||
columns.push_back(ci.m_name);
|
||||
dtypes.push_back(ci.m_type);
|
||||
}
|
||||
}
|
||||
|
||||
t_schema aggschema(columns, dtypes);
|
||||
|
||||
m_aggregates = std::make_shared<t_data_table>(aggschema, m_tree.size());
|
||||
|
||||
m_aggregates->init();
|
||||
m_aggregates->set_size(m_tree.size());
|
||||
|
||||
for (const auto& aggspec : m_aggspecs) {
|
||||
const std::vector<t_dep>& deps = aggspec.get_dependencies();
|
||||
|
||||
const t_data_table* tbl =
|
||||
aggspec.is_non_delta() ? m_strands.get() : m_strand_deltas.get();
|
||||
|
||||
std::vector<std::shared_ptr<const t_column>> icolumns;
|
||||
icolumns.reserve(deps.size());
|
||||
for (const auto& d : deps) {
|
||||
icolumns.push_back(tbl->get_const_column(d.name()));
|
||||
}
|
||||
|
||||
auto output_col = m_aggregates->get_column(aggspec.name());
|
||||
|
||||
t_aggregate agg(m_tree, aggspec.agg(), icolumns, output_col);
|
||||
agg.init();
|
||||
}
|
||||
}
|
||||
|
||||
const t_data_table&
|
||||
t_dtree_ctx::get_aggtable() const {
|
||||
return *(m_aggregates);
|
||||
}
|
||||
|
||||
const t_dtree&
|
||||
t_dtree_ctx::get_tree() const {
|
||||
return m_tree;
|
||||
}
|
||||
|
||||
const std::vector<t_aggspec>&
|
||||
t_dtree_ctx::get_aggspecs() const {
|
||||
return m_aggspecs;
|
||||
}
|
||||
|
||||
const t_aggspec&
|
||||
t_dtree_ctx::get_aggspec(const std::string& aggname) const {
|
||||
auto iter = m_aggspecmap.find(aggname);
|
||||
PSP_VERBOSE_ASSERT(iter != m_aggspecmap.end(), "Failed to find aggspec");
|
||||
PSP_VERBOSE_ASSERT(
|
||||
static_cast<t_uindex>(iter->second) < m_aggspecs.size(),
|
||||
"Invalid aggspec index"
|
||||
);
|
||||
return m_aggspecs[iter->second];
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree_ctx::pprint(const t_filter& fltr) const {
|
||||
std::vector<const t_column*> aggcols;
|
||||
|
||||
t_uindex ncols = 0;
|
||||
for (const std::string& colname : m_aggregates->get_schema().m_columns) {
|
||||
aggcols.push_back(m_aggregates->_get_const_column(colname));
|
||||
std::cout << colname << ", ";
|
||||
++ncols;
|
||||
}
|
||||
std::cout << "\n====================================\n";
|
||||
|
||||
for (auto idx : m_tree.dfs()) {
|
||||
auto depth = m_tree.get_depth(idx);
|
||||
for (t_uindex spidx = 0; spidx < static_cast<t_uindex>(depth);
|
||||
++spidx) {
|
||||
std::cout << "\t";
|
||||
}
|
||||
|
||||
auto value = m_tree.get_value(fltr, idx);
|
||||
std::cout << "(" << idx << "). " << value << " => ";
|
||||
|
||||
for (t_uindex aggidx = 0; aggidx < ncols; ++aggidx) {
|
||||
std::cout << aggcols[aggidx]->get_scalar(idx) << ", ";
|
||||
}
|
||||
std::cout << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<const t_uindex*, const t_uindex*>
|
||||
t_dtree_ctx::get_leaf_iterators(t_index idx) const {
|
||||
const t_dense_tnode* node = m_tree.get_node_ptr(idx);
|
||||
const auto* lbaseptr = m_tree.get_leaf_cptr()->get_nth<t_uindex>(0);
|
||||
return std::pair<const t_uindex*, const t_uindex*>(
|
||||
lbaseptr + node->m_flidx, lbaseptr + node->m_flidx + node->m_nleaves
|
||||
);
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_dtree_ctx::get_pkey_col() const {
|
||||
return m_strands->get_const_column("psp_pkey");
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_column>
|
||||
t_dtree_ctx::get_strand_count_col() const {
|
||||
return m_strand_deltas->get_const_column("psp_strand_count");
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_data_table>
|
||||
t_dtree_ctx::get_strands() const {
|
||||
return m_strands;
|
||||
}
|
||||
|
||||
std::shared_ptr<const t_data_table>
|
||||
t_dtree_ctx::get_strand_deltas() const {
|
||||
return m_strand_deltas;
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree_ctx::pprint_strands() const {
|
||||
std::vector<const t_column*> columns;
|
||||
const auto* scount_col =
|
||||
m_strand_deltas->_get_const_column("psp_strand_count");
|
||||
const auto* pkey_col = m_strands->_get_const_column("psp_pkey");
|
||||
auto strand_schema = m_strands->get_schema();
|
||||
|
||||
t_uindex width = 18;
|
||||
|
||||
std::vector<std::string> colnames = {"psp_pkey", "psp_strand_count"};
|
||||
|
||||
for (const auto& colname : strand_schema.m_columns) {
|
||||
const auto* col = m_strands->_get_const_column(colname);
|
||||
if (col != pkey_col) {
|
||||
columns.push_back(col);
|
||||
colnames.push_back(colname);
|
||||
}
|
||||
}
|
||||
|
||||
auto strand_delta_schema = m_strand_deltas->get_schema();
|
||||
for (const auto& colname : strand_delta_schema.m_columns) {
|
||||
const auto* col = m_strand_deltas->_get_const_column(colname);
|
||||
if (col != scount_col) {
|
||||
columns.push_back(col);
|
||||
std::stringstream ss;
|
||||
ss << "delta(" << colname << ")";
|
||||
colnames.push_back(ss.str());
|
||||
}
|
||||
}
|
||||
|
||||
for (const auto& c : colnames) {
|
||||
std::cout << std::setw(width) << c;
|
||||
}
|
||||
|
||||
std::cout << "\n====================================\n";
|
||||
|
||||
for (t_uindex idx = 0, loop_end = m_strands->size(); idx < loop_end;
|
||||
++idx) {
|
||||
|
||||
std::vector<t_tscalar> vec;
|
||||
vec.push_back(pkey_col->get_scalar(idx));
|
||||
t_tscalar sc;
|
||||
sc.set(t_index(*(scount_col->get<std::int8_t>(idx))));
|
||||
vec.push_back(sc);
|
||||
|
||||
for (const auto* col : columns) {
|
||||
vec.push_back(col->get_scalar(idx));
|
||||
}
|
||||
|
||||
std::cout << idx << ".";
|
||||
for (const auto& val : vec) {
|
||||
std::cout << std::setw(width) << val;
|
||||
}
|
||||
|
||||
std::cout << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_dtree_ctx::pprint_strands_tree() const {
|
||||
typedef std::pair<std::string, const t_column*> t_colname_cptr_pair;
|
||||
|
||||
std::vector<t_colname_cptr_pair> columns;
|
||||
|
||||
columns.emplace_back(
|
||||
"psp_pkey", m_strands->_get_const_column("psp_pkey")
|
||||
);
|
||||
|
||||
columns.emplace_back(
|
||||
"psp_strand_count",
|
||||
m_strand_deltas->_get_const_column("psp_strand_count")
|
||||
);
|
||||
|
||||
for (const auto& piv : m_tree.get_pivots()) {
|
||||
columns.emplace_back(
|
||||
piv.colname(), m_strands->_get_const_column(piv.colname())
|
||||
);
|
||||
}
|
||||
|
||||
for (auto dptidx : m_tree.dfs()) {
|
||||
std::cout << "nidx(" << dptidx << ") => " << '\n';
|
||||
|
||||
t_depth ndepth = m_tree.get_depth(dptidx);
|
||||
|
||||
auto liters = get_leaf_iterators(dptidx);
|
||||
|
||||
for (const auto* lfiter = liters.first; lfiter != liters.second;
|
||||
++lfiter) {
|
||||
for (t_uindex idx = 0; idx < ndepth; ++idx) {
|
||||
std::cout << "\t";
|
||||
}
|
||||
|
||||
std::cout << "\tleaf# " << *lfiter << "\n";
|
||||
|
||||
for (const auto& colinfo : columns) {
|
||||
for (t_uindex idx = 0; idx < ndepth + 1; ++idx) {
|
||||
std::cout << "\t";
|
||||
}
|
||||
std::cout << " " << colinfo.first << ": "
|
||||
<< colinfo.second->get_scalar(*lfiter) << "\n";
|
||||
}
|
||||
}
|
||||
|
||||
std::cout << '\n';
|
||||
}
|
||||
}
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,82 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/dependency.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_dep::t_dep(const t_dep_recipe& v) :
|
||||
m_name(v.m_name),
|
||||
m_disp_name(v.m_disp_name),
|
||||
m_type(v.m_type),
|
||||
m_imm(v.m_imm),
|
||||
m_dtype(v.m_dtype) {}
|
||||
|
||||
t_dep::t_dep(const std::string& name, t_deptype type) :
|
||||
m_name(name),
|
||||
m_disp_name(name),
|
||||
m_type(type),
|
||||
m_dtype(DTYPE_NONE) {}
|
||||
|
||||
t_dep::t_dep(t_tscalar imm) :
|
||||
m_type(DEPTYPE_SCALAR),
|
||||
m_imm(imm),
|
||||
m_dtype(DTYPE_NONE) {}
|
||||
|
||||
t_dep::t_dep(
|
||||
std::string name, std::string disp_name, t_deptype type, t_dtype dtype
|
||||
) :
|
||||
m_name(std::move(name)),
|
||||
m_disp_name(std::move(disp_name)),
|
||||
m_type(type),
|
||||
m_dtype(dtype) {}
|
||||
|
||||
const std::string&
|
||||
t_dep::name() const {
|
||||
return m_name;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_dep::disp_name() const {
|
||||
return m_disp_name;
|
||||
}
|
||||
|
||||
t_deptype
|
||||
t_dep::type() const {
|
||||
return m_type;
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_dep::imm() const {
|
||||
return m_imm;
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_dep::dtype() const {
|
||||
return m_dtype;
|
||||
}
|
||||
|
||||
t_dep_recipe
|
||||
t_dep::get_recipe() const {
|
||||
t_dep_recipe rv;
|
||||
rv.m_name = m_name;
|
||||
rv.m_disp_name = m_disp_name;
|
||||
rv.m_type = m_type;
|
||||
rv.m_imm = m_imm;
|
||||
rv.m_dtype = m_dtype;
|
||||
return rv;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,198 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/expression_tables.h>
|
||||
#include <perspective/utils.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_expression_tables::t_expression_tables(
|
||||
const std::vector<std::shared_ptr<t_computed_expression>>& expressions,
|
||||
t_backing_store backing_store
|
||||
) {
|
||||
t_schema schema;
|
||||
t_schema transitions_schema;
|
||||
|
||||
for (const auto& expr : expressions) {
|
||||
const std::string& alias = expr->get_expression_alias();
|
||||
schema.add_column(alias, expr->get_dtype());
|
||||
transitions_schema.add_column(alias, DTYPE_UINT8);
|
||||
}
|
||||
|
||||
// Only the persistent `m_master` table honors on-disk backing; the
|
||||
// transitional tables are per-update scratch (cleared every update, sized
|
||||
// to the update batch) so disk-backing them is pure I/O churn with no
|
||||
// memory-relief benefit, and they stay in memory.
|
||||
std::string master_dirname;
|
||||
if (backing_store == BACKING_STORE_DISK && !expressions.empty()) {
|
||||
master_dirname = create_backing_store_dir("perspective_expr_");
|
||||
}
|
||||
|
||||
m_master = std::make_shared<t_data_table>(
|
||||
"", master_dirname, schema, DEFAULT_EMPTY_CAPACITY, backing_store
|
||||
);
|
||||
m_flattened = std::make_shared<t_data_table>(
|
||||
"", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_prev = std::make_shared<t_data_table>(
|
||||
"", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_current = std::make_shared<t_data_table>(
|
||||
"", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_delta = std::make_shared<t_data_table>(
|
||||
"", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_transitions = std::make_shared<t_data_table>(
|
||||
"", "", transitions_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
|
||||
m_master->init();
|
||||
m_flattened->init();
|
||||
m_prev->init();
|
||||
m_current->init();
|
||||
m_delta->init();
|
||||
m_transitions->init();
|
||||
}
|
||||
|
||||
t_data_table*
|
||||
t_expression_tables::get_table() const {
|
||||
return m_master.get();
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::set_flattened(
|
||||
const std::shared_ptr<t_data_table>& flattened
|
||||
) const {
|
||||
t_uindex flattened_num_rows = flattened->size();
|
||||
reserve_transitional_table_size(flattened_num_rows);
|
||||
set_transitional_table_size(flattened_num_rows);
|
||||
const t_schema& schema = m_flattened->get_schema();
|
||||
const std::vector<std::string>& column_names = schema.m_columns;
|
||||
for (const auto& colname : column_names) {
|
||||
// Deep-copy into the (in-memory) transitional column in place rather
|
||||
// than `clone()`-ing the source. `clone()` inherits the source's
|
||||
// backing store via its recipe, and for an on-disk expression
|
||||
// `m_master` that produces a broken disk clone which aliases the
|
||||
// master's backing file. `copy_from` preserves `m_flattened`'s own
|
||||
// (memory) backing store.
|
||||
m_flattened->get_column(colname)->copy_from(
|
||||
*flattened->get_column(colname)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::calculate_transitions(
|
||||
const std::shared_ptr<t_data_table>& existed
|
||||
) {
|
||||
const t_schema& schema = m_transitions->get_schema();
|
||||
const std::vector<std::string>& column_names = schema.m_columns;
|
||||
const t_column& existed_column =
|
||||
*(existed->get_const_column("psp_existed"));
|
||||
|
||||
auto num_cols = column_names.size();
|
||||
|
||||
parallel_for(
|
||||
int(num_cols),
|
||||
[&column_names, &existed_column, this](int cidx) {
|
||||
const std::string& cname = column_names[cidx];
|
||||
const t_column& prev_column = *(m_prev->get_const_column(cname));
|
||||
const t_column& current_column =
|
||||
*(m_current->get_const_column(cname));
|
||||
std::shared_ptr<t_column> transition_column =
|
||||
m_transitions->get_column(cname);
|
||||
|
||||
for (t_uindex ridx = 0; ridx < transition_column->size(); ++ridx) {
|
||||
bool row_existed =
|
||||
existed_column.get_nth<bool>(ridx) != nullptr;
|
||||
|
||||
t_tscalar prev_value = prev_column.get_scalar(ridx);
|
||||
t_tscalar curr_value = current_column.get_scalar(ridx);
|
||||
|
||||
bool prev_valid = prev_column.is_valid(ridx);
|
||||
bool curr_valid = current_column.is_valid(ridx);
|
||||
bool prev_curr_eq =
|
||||
prev_valid && curr_valid && (prev_value == curr_value);
|
||||
|
||||
t_value_transition transition;
|
||||
|
||||
// Use a small subset of `t_value_transitions` that are
|
||||
// relevant - I have not implemented the code paths in
|
||||
// `calc_transitions` that are not referenced elsewhere, i.e.
|
||||
// by a context or by a tree implementation.
|
||||
if (row_existed) {
|
||||
if (prev_curr_eq) {
|
||||
// Row existed before, and the current value is
|
||||
// the same as the previous value.
|
||||
transition = VALUE_TRANSITION_EQ_TT;
|
||||
} else {
|
||||
if (!prev_valid && curr_valid) {
|
||||
// Previous value was a null, new value is valid.
|
||||
transition = VALUE_TRANSITION_NEQ_FT;
|
||||
} else {
|
||||
// Previous value was not null, new value is
|
||||
// not null, and previous value != new value
|
||||
transition = VALUE_TRANSITION_NEQ_TT;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Row did not exist before and was added
|
||||
transition = VALUE_TRANSITION_NEQ_FT;
|
||||
}
|
||||
|
||||
transition_column->set_nth<std::uint8_t>(ridx, transition);
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::reserve_transitional_table_size(t_uindex size) const {
|
||||
m_flattened->reserve(size);
|
||||
m_prev->reserve(size);
|
||||
m_current->reserve(size);
|
||||
m_delta->reserve(size);
|
||||
m_transitions->reserve(size);
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::set_transitional_table_size(t_uindex size) const {
|
||||
m_flattened->set_size(size);
|
||||
m_prev->set_size(size);
|
||||
m_current->set_size(size);
|
||||
m_delta->set_size(size);
|
||||
m_transitions->set_size(size);
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::clear_transitional_tables() const {
|
||||
m_flattened->clear();
|
||||
m_prev->clear();
|
||||
m_current->clear();
|
||||
m_delta->clear();
|
||||
m_transitions->clear();
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_tables::reset() const {
|
||||
m_master->reset();
|
||||
m_flattened->reset();
|
||||
m_prev->reset();
|
||||
m_current->reset();
|
||||
m_delta->reset();
|
||||
m_transitions->reset();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,81 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/expression_vocab.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_expression_vocab::t_expression_vocab() {
|
||||
// Pre-reserve each page to its full size so that `intern()` never
|
||||
// triggers a realloc inside `t_vocab`; previously-returned `const char*`
|
||||
// pointers must remain valid for the lifetime of the vocab.
|
||||
m_max_vocab_size = 16 * 64 * 64;
|
||||
|
||||
// Always start with one vocab
|
||||
allocate_new_vocab();
|
||||
}
|
||||
|
||||
const char*
|
||||
t_expression_vocab::intern(const char* str) {
|
||||
std::size_t bytelength = strlen(str);
|
||||
std::size_t hash = boost::hash_range(str, str + bytelength);
|
||||
t_uindex existing_idx;
|
||||
|
||||
for (auto& current_vocab : m_vocabs) {
|
||||
if (current_vocab.string_exists(str, hash, existing_idx)) {
|
||||
return current_vocab.unintern_c(existing_idx);
|
||||
}
|
||||
}
|
||||
|
||||
if (m_current_vocab_size + bytelength + 1 > m_max_vocab_size) {
|
||||
allocate_new_vocab();
|
||||
}
|
||||
|
||||
t_vocab& current_vocab = m_vocabs.front();
|
||||
m_current_vocab_size += bytelength + 1;
|
||||
t_uindex interned_idx = current_vocab.get_interned(str, bytelength, hash);
|
||||
return current_vocab.unintern_c(interned_idx);
|
||||
}
|
||||
|
||||
const char*
|
||||
t_expression_vocab::intern(const std::string& str) {
|
||||
return intern(str.c_str());
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_vocab::clear() {
|
||||
m_vocabs.clear();
|
||||
allocate_new_vocab();
|
||||
}
|
||||
|
||||
const char*
|
||||
t_expression_vocab::get_empty_string() const {
|
||||
return m_empty_string.c_str();
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_vocab::pprint() const {
|
||||
for (const auto& vocab : m_vocabs) {
|
||||
vocab.pprint_vocabulary();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_expression_vocab::allocate_new_vocab() {
|
||||
t_vocab vocab;
|
||||
vocab.init(false);
|
||||
vocab.reserve(m_max_vocab_size, m_max_vocab_size / 64);
|
||||
m_vocabs.insert(m_vocabs.begin(), std::move(vocab));
|
||||
m_current_vocab_size = 0;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,124 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/extract_aggregate.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_tscalar
|
||||
extract_aggregate(
|
||||
const t_aggspec& aggspec,
|
||||
const t_column* aggcol,
|
||||
t_uindex ridx,
|
||||
t_index pridx
|
||||
) {
|
||||
|
||||
switch (aggspec.agg()) {
|
||||
case AGGTYPE_PCT_SUM_PARENT: {
|
||||
t_tscalar cv = aggcol->get_scalar(ridx);
|
||||
|
||||
if (pridx == INVALID_INDEX) {
|
||||
return mktscalar<double>(100.0);
|
||||
}
|
||||
|
||||
double pv = aggcol->get_scalar(pridx).to_double();
|
||||
t_tscalar ret;
|
||||
if (pv == 0) {
|
||||
ret.set(t_none());
|
||||
} else {
|
||||
ret.set(100.0 * (cv.to_double() / pv));
|
||||
}
|
||||
return ret;
|
||||
} break;
|
||||
case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
|
||||
t_tscalar cv = aggcol->get_scalar(ridx);
|
||||
double pv = aggcol->get_scalar(ROOT_AGGIDX).to_double();
|
||||
t_tscalar ret;
|
||||
if (pv == 0) {
|
||||
ret.set(t_none());
|
||||
} else {
|
||||
ret.set(100.0 * (cv.to_double() / pv));
|
||||
}
|
||||
return ret;
|
||||
} break;
|
||||
case AGGTYPE_SUM:
|
||||
case AGGTYPE_SUM_ABS:
|
||||
case AGGTYPE_ABS_SUM:
|
||||
case AGGTYPE_GMV:
|
||||
case AGGTYPE_SUM_NOT_NULL:
|
||||
case AGGTYPE_MUL:
|
||||
case AGGTYPE_COUNT:
|
||||
case AGGTYPE_ANY:
|
||||
case AGGTYPE_DOMINANT:
|
||||
case AGGTYPE_Q1:
|
||||
case AGGTYPE_Q3:
|
||||
case AGGTYPE_MEDIAN:
|
||||
case AGGTYPE_FIRST:
|
||||
case AGGTYPE_AND:
|
||||
case AGGTYPE_OR:
|
||||
case AGGTYPE_LAST_BY_INDEX:
|
||||
case AGGTYPE_LAST_VALUE:
|
||||
case AGGTYPE_LAST_MINUS_FIRST:
|
||||
case AGGTYPE_MAX:
|
||||
case AGGTYPE_MAX_BY:
|
||||
case AGGTYPE_MIN_BY:
|
||||
case AGGTYPE_MIN:
|
||||
case AGGTYPE_HIGH_WATER_MARK:
|
||||
case AGGTYPE_LOW_WATER_MARK:
|
||||
case AGGTYPE_HIGH_MINUS_LOW:
|
||||
case AGGTYPE_SCALED_DIV:
|
||||
case AGGTYPE_SCALED_ADD:
|
||||
case AGGTYPE_SCALED_MUL:
|
||||
case AGGTYPE_UDF_COMBINER:
|
||||
case AGGTYPE_UDF_REDUCER:
|
||||
case AGGTYPE_JOIN:
|
||||
case AGGTYPE_IDENTITY:
|
||||
case AGGTYPE_DISTINCT_COUNT:
|
||||
case AGGTYPE_DISTINCT_LEAF:
|
||||
case AGGTYPE_VARIANCE:
|
||||
case AGGTYPE_STANDARD_DEVIATION: {
|
||||
t_tscalar rval = aggcol->get_scalar(ridx);
|
||||
return rval;
|
||||
} break;
|
||||
case AGGTYPE_UNIQUE: {
|
||||
t_tscalar rval = aggcol->get_scalar(ridx);
|
||||
if (!rval.is_valid()) {
|
||||
t_tscalar rv;
|
||||
rv.set(t_none());
|
||||
return rv;
|
||||
}
|
||||
return rval;
|
||||
} break;
|
||||
case AGGTYPE_MEAN_BY_COUNT:
|
||||
case AGGTYPE_WEIGHTED_MEAN:
|
||||
case AGGTYPE_MEAN: {
|
||||
const auto* pair = aggcol->get_nth<std::pair<double, double>>(ridx);
|
||||
t_tscalar rval;
|
||||
double second = pair->second;
|
||||
if (second != 0) {
|
||||
double mean = pair->first / second;
|
||||
rval.set(mean);
|
||||
} else {
|
||||
rval.set(t_none());
|
||||
}
|
||||
return rval;
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected agg type");
|
||||
}
|
||||
}
|
||||
|
||||
return mknone();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,168 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/filter.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_fterm::t_fterm() = default;
|
||||
|
||||
t_fterm::t_fterm(
|
||||
std::string colname,
|
||||
t_filter_op op,
|
||||
t_tscalar threshold,
|
||||
const std::vector<t_tscalar>& bag,
|
||||
bool negated,
|
||||
bool is_primary
|
||||
) :
|
||||
m_colname(std::move(colname)),
|
||||
m_op(op),
|
||||
m_threshold(threshold),
|
||||
m_bag(bag),
|
||||
m_negated(negated),
|
||||
m_is_primary(is_primary) {
|
||||
m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE)
|
||||
&& threshold.m_type == DTYPE_STR;
|
||||
}
|
||||
|
||||
t_fterm::t_fterm(
|
||||
std::string colname,
|
||||
t_filter_op op,
|
||||
t_tscalar threshold,
|
||||
const std::vector<t_tscalar>& bag
|
||||
) :
|
||||
m_colname(std::move(colname)),
|
||||
m_op(op),
|
||||
m_threshold(threshold),
|
||||
m_bag(bag),
|
||||
m_negated(false),
|
||||
m_is_primary(false) {
|
||||
m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE)
|
||||
&& threshold.m_type == DTYPE_STR;
|
||||
}
|
||||
|
||||
void
|
||||
t_fterm::coerce_numeric(t_dtype dtype) {
|
||||
m_threshold.set(m_threshold.coerce_numeric_dtype(dtype));
|
||||
for (auto& f : m_bag) {
|
||||
f.set(f.coerce_numeric_dtype(dtype));
|
||||
}
|
||||
}
|
||||
|
||||
std::string
|
||||
t_fterm::get_expr() const {
|
||||
std::stringstream ss;
|
||||
|
||||
ss << m_colname << " ";
|
||||
|
||||
switch (m_op) {
|
||||
case FILTER_OP_LT:
|
||||
case FILTER_OP_LTEQ:
|
||||
case FILTER_OP_GT:
|
||||
case FILTER_OP_GTEQ:
|
||||
case FILTER_OP_EQ:
|
||||
case FILTER_OP_NE:
|
||||
case FILTER_OP_CONTAINS: {
|
||||
ss << filter_op_to_str(m_op) << " ";
|
||||
ss << m_threshold.to_string(true);
|
||||
} break;
|
||||
case FILTER_OP_NOT_IN:
|
||||
case FILTER_OP_IN: {
|
||||
ss << " " << filter_op_to_str(m_op) << " (";
|
||||
for (auto v : m_bag) {
|
||||
ss << v.to_string(true) << ", ";
|
||||
}
|
||||
ss << " )";
|
||||
} break;
|
||||
case FILTER_OP_BEGINS_WITH:
|
||||
case FILTER_OP_ENDS_WITH: {
|
||||
ss << "." << filter_op_to_str(m_op) << "( "
|
||||
<< m_threshold.to_string(true) << " )";
|
||||
} break;
|
||||
default: {
|
||||
ss << " is failed_compilation";
|
||||
}
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
t_filter::t_filter() : m_mode(SELECT_MODE_ALL) {}
|
||||
|
||||
t_filter::t_filter(const std::vector<std::string>& columns) :
|
||||
m_mode(SELECT_MODE_ALL),
|
||||
m_columns(columns) {}
|
||||
|
||||
t_filter::t_filter(
|
||||
const std::vector<std::string>& columns, t_uindex bidx, t_uindex eidx
|
||||
) :
|
||||
m_mode(SELECT_MODE_RANGE),
|
||||
m_bidx(bidx),
|
||||
m_eidx(eidx),
|
||||
m_columns(columns) {}
|
||||
|
||||
t_filter::t_filter(
|
||||
const std::vector<std::string>& columns, t_uindex mask_size
|
||||
) :
|
||||
m_mode(SELECT_MODE_MASK),
|
||||
m_columns(columns),
|
||||
m_mask(std::make_shared<t_mask>(mask_size)) {}
|
||||
|
||||
t_uindex
|
||||
t_filter::num_cols() const {
|
||||
return m_columns.size();
|
||||
}
|
||||
|
||||
bool
|
||||
t_filter::has_filter() const {
|
||||
return m_mode != SELECT_MODE_ALL;
|
||||
}
|
||||
|
||||
const std::vector<std::string>&
|
||||
t_filter::columns() const {
|
||||
return m_columns;
|
||||
}
|
||||
|
||||
t_select_mode
|
||||
t_filter::mode() const {
|
||||
return m_mode;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_filter::bidx() const {
|
||||
return m_bidx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_filter::eidx() const {
|
||||
return m_eidx;
|
||||
}
|
||||
|
||||
t_maskcsptr
|
||||
t_filter::cmask() const {
|
||||
return t_maskcsptr(m_mask);
|
||||
}
|
||||
|
||||
t_masksptr
|
||||
t_filter::mask() const {
|
||||
return t_masksptr(m_mask);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_filter::count() const {
|
||||
return m_mask->count();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,497 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/config.h>
|
||||
#include <perspective/flat_traversal.h>
|
||||
#include <perspective/scalar.h>
|
||||
#include <perspective/schema.h>
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_ftrav::t_ftrav() : m_step_deletes(0), m_step_inserts(0) {
|
||||
m_index = std::make_shared<std::vector<t_mselem>>();
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::init() {
|
||||
m_index = std::make_shared<std::vector<t_mselem>>();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ftrav::get_all_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
// assumes the code calling this has already validated
|
||||
// cells
|
||||
std::vector<t_tscalar> rval;
|
||||
rval.reserve(cells.size());
|
||||
std::vector<t_mselem>* index = m_index.get();
|
||||
for (const auto& cell : cells) {
|
||||
rval.push_back((*index)[cell.first].m_pkey);
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ftrav::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
tsl::hopscotch_set<t_tscalar> all_pkeys;
|
||||
|
||||
std::set<t_index> all_rows;
|
||||
|
||||
for (const auto& cell : cells) {
|
||||
all_rows.insert(cell.first);
|
||||
}
|
||||
|
||||
std::vector<t_tscalar> rval(all_rows.size());
|
||||
std::set<t_index>::iterator it;
|
||||
t_index count = 0;
|
||||
for (it = all_rows.begin(); it != all_rows.end(); ++it) {
|
||||
rval[count] = (*m_index)[*it].m_pkey;
|
||||
++count;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ftrav::get_pkeys(t_index begin_row, t_index end_row) const {
|
||||
t_index index_size = m_index->size();
|
||||
end_row = std::min(end_row, index_size);
|
||||
std::vector<t_tscalar> rval(end_row - begin_row);
|
||||
for (t_index ridx = begin_row; ridx < end_row; ++ridx) {
|
||||
rval[ridx - begin_row] = (*m_index)[ridx].m_pkey;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ftrav::get_pkeys(const std::vector<t_uindex>& rows) const {
|
||||
std::vector<t_tscalar> rval;
|
||||
rval.reserve(rows.size());
|
||||
for (unsigned long long ridx : rows) {
|
||||
rval.push_back((*m_index)[ridx].m_pkey);
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_ftrav::get_pkeys() const {
|
||||
return get_pkeys(0, size());
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_ftrav::get_pkey(t_index idx) const {
|
||||
return (*m_index)[idx].m_pkey;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::fill_sort_elem(
|
||||
const t_gstate& gstate,
|
||||
const t_data_table& expression_master_table,
|
||||
const t_config& config,
|
||||
t_tscalar pkey,
|
||||
t_mselem& out_elem
|
||||
) {
|
||||
t_index sortby_size = m_sortby.size();
|
||||
out_elem.m_row.reserve(sortby_size);
|
||||
out_elem.m_pkey = pkey;
|
||||
|
||||
for (const t_sortspec& sort : m_sortby) {
|
||||
// maintain backwards compatibility
|
||||
std::string colname;
|
||||
|
||||
if (!sort.m_colname.empty()) {
|
||||
colname = config.get_sort_by(sort.m_colname);
|
||||
} else {
|
||||
colname = config.col_at(sort.m_agg_index);
|
||||
}
|
||||
|
||||
const std::string& sortby_colname = config.get_sort_by(colname);
|
||||
|
||||
out_elem.m_row.push_back(
|
||||
m_symtable.get_interned_tscalar(get_from_gstate(
|
||||
gstate, expression_master_table, sortby_colname, pkey
|
||||
))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::fill_sort_elem(
|
||||
const t_gstate& gstate,
|
||||
const t_config& config,
|
||||
const std::vector<t_tscalar>& row,
|
||||
t_mselem& out_elem
|
||||
) const {
|
||||
t_index sortby_size = m_sortby.size();
|
||||
out_elem.m_row.reserve(sortby_size);
|
||||
out_elem.m_pkey = mknone();
|
||||
|
||||
for (const t_sortspec& sort : m_sortby) {
|
||||
std::string colname;
|
||||
|
||||
if (!sort.m_colname.empty()) {
|
||||
colname = config.get_sort_by(sort.m_colname);
|
||||
} else {
|
||||
colname = config.col_at(sort.m_agg_index);
|
||||
}
|
||||
|
||||
const std::string& sortby_colname = config.get_sort_by(colname);
|
||||
|
||||
out_elem.m_row.push_back(
|
||||
get_interned_tscalar(row.at(config.get_colidx(sortby_colname)))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::sort_by(
|
||||
const t_gstate& gstate,
|
||||
const t_data_table& expression_master_table,
|
||||
const t_config& config,
|
||||
const std::vector<t_sortspec>& sortby
|
||||
) {
|
||||
if (sortby.empty()) {
|
||||
return;
|
||||
}
|
||||
t_multisorter sorter(get_sort_orders(sortby));
|
||||
t_index size = m_index->size();
|
||||
auto sort_elems =
|
||||
std::make_shared<std::vector<t_mselem>>(static_cast<size_t>(size));
|
||||
m_sortby = sortby;
|
||||
|
||||
for (t_index idx = 0; idx < size; ++idx) {
|
||||
t_mselem& elem = (*sort_elems)[idx];
|
||||
t_tscalar pkey = (*m_index)[idx].m_pkey;
|
||||
fill_sort_elem(gstate, expression_master_table, config, pkey, elem);
|
||||
}
|
||||
|
||||
std::swap(m_index, sort_elems);
|
||||
std::sort(m_index->begin(), m_index->end(), sorter);
|
||||
m_pkeyidx.clear();
|
||||
for (t_index idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) {
|
||||
m_pkeyidx[(*m_index)[idx].m_pkey] = idx;
|
||||
}
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ftrav::size() const {
|
||||
return m_index->size();
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::get_row_indices(
|
||||
const tsl::hopscotch_set<t_tscalar>& pkeys,
|
||||
tsl::hopscotch_map<t_tscalar, t_index>& out_map
|
||||
) const {
|
||||
for (t_index idx = 0, loop_end = size(); idx < loop_end; ++idx) {
|
||||
const t_tscalar& pkey = (*m_index)[idx].m_pkey;
|
||||
if (pkeys.find(pkey) != pkeys.end()) {
|
||||
out_map[pkey] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::get_row_indices(
|
||||
t_index bidx,
|
||||
t_index eidx,
|
||||
const tsl::hopscotch_set<t_tscalar>& pkeys,
|
||||
tsl::hopscotch_map<t_tscalar, t_index>& out_map
|
||||
) const {
|
||||
for (t_index idx = bidx; idx < eidx; ++idx) {
|
||||
const t_tscalar& pkey = (*m_index)[idx].m_pkey;
|
||||
if (pkeys.find(pkey) != pkeys.end()) {
|
||||
out_map[pkey] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Given a set of primary keys, return the corresponding row indices.
|
||||
*
|
||||
* @param pkeys
|
||||
* @return std::vector<t_index>
|
||||
*/
|
||||
std::vector<t_uindex>
|
||||
t_ftrav::get_row_indices(const tsl::hopscotch_set<t_tscalar>& pkeys) const {
|
||||
std::vector<t_uindex> rows;
|
||||
for (t_uindex idx = 0, loop_end = size(); idx < loop_end; ++idx) {
|
||||
const t_tscalar& pkey = (*m_index)[idx].m_pkey;
|
||||
if (pkeys.find(pkey) != pkeys.end()) {
|
||||
rows.push_back(idx);
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::reset() {
|
||||
if (m_index != nullptr) {
|
||||
m_index->clear();
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::check_size() {
|
||||
tsl::hopscotch_set<t_tscalar> pkey_set;
|
||||
for (t_index idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) {
|
||||
if (pkey_set.find((*m_index)[idx].m_pkey) != pkey_set.end()) {
|
||||
std::cout << "Duplicate entry for " << (*m_index)[idx].m_pkey
|
||||
<< '\n';
|
||||
PSP_COMPLAIN_AND_ABORT("Exiting");
|
||||
}
|
||||
|
||||
pkey_set.insert((*m_index)[idx].m_pkey);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_ftrav::validate_cells(const std::vector<std::pair<t_uindex, t_uindex>>& cells
|
||||
) const {
|
||||
t_index trav_size = size();
|
||||
|
||||
for (const auto& cell : cells) {
|
||||
t_index ridx = cell.first;
|
||||
if (ridx >= trav_size) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::step_begin() {
|
||||
m_step_deletes = 0;
|
||||
m_step_inserts = 0;
|
||||
m_new_elems.clear();
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::step_end() {
|
||||
// Fast path: if no incremental work happened this step, `m_index` and
|
||||
// `m_pkeyidx` are already in their final shape (either unchanged, or
|
||||
// populated directly via `bulk_load_append`). Skip the O(N) rebuild.
|
||||
if (m_step_inserts == 0 && m_step_deletes == 0 && m_new_elems.empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// The new number of rows in this traversal
|
||||
t_index new_size = m_index->size() + m_step_inserts - m_step_deletes;
|
||||
|
||||
auto new_index = std::make_shared<std::vector<t_mselem>>();
|
||||
new_index->reserve(new_size);
|
||||
|
||||
t_uindex i = 0;
|
||||
t_multisorter sorter(get_sort_orders(m_sortby));
|
||||
|
||||
std::vector<t_mselem> new_rows;
|
||||
new_rows.reserve(m_new_elems.size());
|
||||
|
||||
for (tsl::hopscotch_map<t_tscalar, t_mselem>::const_iterator pkelem_iter =
|
||||
m_new_elems.begin();
|
||||
pkelem_iter != m_new_elems.end();
|
||||
++pkelem_iter) {
|
||||
new_rows.push_back(pkelem_iter->second);
|
||||
}
|
||||
|
||||
// TODO: int/float/date/datetime pkeys are already sorted here, so if
|
||||
// there was a way to assert that `psp_pkey` is a string typed column,
|
||||
// we can conditional the sort on whether m_sortby.size() > 0 or if
|
||||
// psp_pkey is a string column.
|
||||
std::sort(new_rows.begin(), new_rows.end(), sorter);
|
||||
|
||||
for (auto& new_elem : new_rows) {
|
||||
while (i < m_index->size()) {
|
||||
const t_mselem& old_elem = (*m_index)[i];
|
||||
|
||||
if (old_elem.m_deleted) {
|
||||
i++;
|
||||
m_pkeyidx.erase(old_elem.m_pkey);
|
||||
} else if (old_elem.m_updated) {
|
||||
i++;
|
||||
} else if (sorter(old_elem, new_elem)) {
|
||||
m_pkeyidx[old_elem.m_pkey] = new_index->size();
|
||||
new_index->push_back(old_elem);
|
||||
i++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
m_pkeyidx[new_elem.m_pkey] = new_index->size();
|
||||
new_index->push_back(new_elem);
|
||||
}
|
||||
|
||||
// reconcile old rows that are marked as removed or updated.
|
||||
while (i < m_index->size()) {
|
||||
const t_mselem& old_elem = (*m_index)[i++];
|
||||
|
||||
if (old_elem.m_deleted) {
|
||||
m_pkeyidx.erase(old_elem.m_pkey);
|
||||
} else if (!old_elem.m_updated) {
|
||||
// Add back cells that have not changed during this step.
|
||||
m_pkeyidx[old_elem.m_pkey] = new_index->size();
|
||||
new_index->push_back(old_elem);
|
||||
}
|
||||
}
|
||||
|
||||
std::swap(new_index, m_index);
|
||||
m_new_elems.clear();
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::add_row(
|
||||
const t_gstate& gstate,
|
||||
const t_data_table& expression_master_table,
|
||||
const t_config& config,
|
||||
t_tscalar pkey
|
||||
) {
|
||||
t_mselem mselem;
|
||||
fill_sort_elem(gstate, expression_master_table, config, pkey, mselem);
|
||||
m_new_elems[pkey] = mselem;
|
||||
++m_step_inserts;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::update_row(
|
||||
const t_gstate& gstate,
|
||||
const t_data_table& expression_master_table,
|
||||
const t_config& config,
|
||||
t_tscalar pkey
|
||||
) {
|
||||
if (m_sortby.empty()) {
|
||||
return;
|
||||
}
|
||||
auto pkiter = m_pkeyidx.find(pkey);
|
||||
if (pkiter == m_pkeyidx.end()) {
|
||||
add_row(gstate, expression_master_table, config, pkey);
|
||||
return;
|
||||
}
|
||||
t_mselem mselem;
|
||||
fill_sort_elem(gstate, expression_master_table, config, pkey, mselem);
|
||||
(*m_index)[pkiter->second].m_updated = true;
|
||||
m_new_elems[pkey] = mselem;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::delete_row(t_tscalar pkey) {
|
||||
auto pkiter = m_pkeyidx.find(pkey);
|
||||
if (pkiter == m_pkeyidx.end()) {
|
||||
return;
|
||||
}
|
||||
(*m_index)[pkiter->second].m_deleted = true;
|
||||
m_new_elems.erase(pkey);
|
||||
++m_step_deletes;
|
||||
}
|
||||
|
||||
std::vector<t_sortspec>
|
||||
t_ftrav::get_sort_by() const {
|
||||
return m_sortby;
|
||||
}
|
||||
|
||||
bool
|
||||
t_ftrav::empty_sort_by() const {
|
||||
return m_sortby.empty();
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::reset_step_state() {
|
||||
m_step_deletes = 0;
|
||||
m_step_inserts = 0;
|
||||
m_new_elems.clear();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_ftrav::lower_bound_row_idx(
|
||||
const t_gstate& gstate,
|
||||
const t_config& config,
|
||||
const std::vector<t_tscalar>& row
|
||||
) const {
|
||||
t_multisorter sorter(get_sort_orders(m_sortby));
|
||||
t_mselem target_val;
|
||||
|
||||
fill_sort_elem(gstate, config, row, target_val);
|
||||
|
||||
auto iter =
|
||||
std::lower_bound(m_index->begin(), m_index->end(), target_val, sorter);
|
||||
|
||||
return std::distance(m_index->begin(), iter);
|
||||
}
|
||||
|
||||
t_index
|
||||
t_ftrav::get_row_idx(t_tscalar pkey) const {
|
||||
auto pkiter = m_pkeyidx.find(pkey);
|
||||
if (pkiter == m_pkeyidx.end()) {
|
||||
return -1;
|
||||
}
|
||||
return pkiter->second;
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_ftrav::get_from_gstate(
|
||||
const t_gstate& gstate,
|
||||
const t_data_table& expression_master_table,
|
||||
const std::string& colname,
|
||||
t_tscalar pkey
|
||||
) const {
|
||||
const t_schema& expression_schema = expression_master_table.get_schema();
|
||||
|
||||
if (expression_schema.has_column(colname)) {
|
||||
return gstate.get(expression_master_table, colname, pkey);
|
||||
}
|
||||
std::shared_ptr<t_data_table> master_table = gstate.get_table();
|
||||
return gstate.get(*master_table, colname, pkey);
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::bulk_load_reserve(t_uindex n) {
|
||||
m_index->reserve(n);
|
||||
m_pkeyidx.reserve(n);
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::bulk_load_append(t_tscalar pkey) {
|
||||
// Pre-condition: `empty_sort_by()` and the caller has ownership of
|
||||
// step framing (i.e. we're inside a `step_begin`/`step_end` pair for
|
||||
// an initial registration). `m_step_inserts` is intentionally NOT
|
||||
// incremented so that `step_end` takes its short-circuit path.
|
||||
// `m_pkeyidx` is populated in `bulk_load_finalize` after sorting,
|
||||
// not here — the pkey-to-index mapping only has meaning once the
|
||||
// final sort order is determined.
|
||||
m_index->emplace_back();
|
||||
m_index->back().m_pkey = pkey;
|
||||
}
|
||||
|
||||
void
|
||||
t_ftrav::bulk_load_finalize() {
|
||||
// Match the order the existing `add_row`/`step_end` path would
|
||||
// produce for an empty sort spec: `cmp_mselem` with zero sort
|
||||
// columns falls through to `a.m_pkey < b.m_pkey`, so sort
|
||||
// `m_index` by pkey and then rebuild `m_pkeyidx` against the final
|
||||
// row positions.
|
||||
std::sort(
|
||||
m_index->begin(),
|
||||
m_index->end(),
|
||||
[](const t_mselem& a, const t_mselem& b) {
|
||||
return a.m_pkey < b.m_pkey;
|
||||
}
|
||||
);
|
||||
for (t_uindex i = 0, loop_end = m_index->size(); i < loop_end; ++i) {
|
||||
m_pkeyidx[(*m_index)[i].m_pkey] = i;
|
||||
}
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,48 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/get_data_extents.h>
|
||||
|
||||
namespace perspective {
|
||||
t_get_data_extents
|
||||
sanitize_get_data_extents(
|
||||
t_index nrows,
|
||||
t_index ncols,
|
||||
t_index start_row,
|
||||
t_index end_row,
|
||||
t_index start_col,
|
||||
t_index end_col
|
||||
) {
|
||||
start_row = std::min(start_row, nrows);
|
||||
end_row = std::min(end_row, nrows);
|
||||
|
||||
start_row = std::max(t_index(0), start_row);
|
||||
end_row = std::max(t_index(0), end_row);
|
||||
end_row = std::max(start_row, end_row);
|
||||
|
||||
start_col = std::min(start_col, ncols);
|
||||
end_col = std::min(end_col, ncols);
|
||||
|
||||
start_col = std::max(t_index(0), start_col);
|
||||
end_col = std::max(t_index(0), end_col);
|
||||
end_col = std::max(start_col, end_col);
|
||||
|
||||
t_get_data_extents rval;
|
||||
rval.m_srow = start_row;
|
||||
rval.m_erow = end_row;
|
||||
rval.m_scol = start_col;
|
||||
rval.m_ecol = end_col;
|
||||
|
||||
return rval;
|
||||
}
|
||||
} // namespace perspective
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,886 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/context_unit.h>
|
||||
#include <perspective/context_zero.h>
|
||||
#include <perspective/context_one.h>
|
||||
#include <perspective/context_two.h>
|
||||
#include <perspective/gnode_state.h>
|
||||
#include <perspective/mask.h>
|
||||
#include <perspective/sym_table.h>
|
||||
#include <perspective/parallel_for.h>
|
||||
#include <perspective/utils.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_gstate::t_gstate(
|
||||
t_schema input_schema,
|
||||
t_schema output_schema,
|
||||
t_backing_store backing_store
|
||||
) :
|
||||
m_input_schema(std::move(input_schema)),
|
||||
m_output_schema(std::move(output_schema)),
|
||||
m_backing_store(backing_store),
|
||||
m_init(false) {
|
||||
LOG_CONSTRUCTOR("t_gstate");
|
||||
}
|
||||
|
||||
t_gstate::~t_gstate() { LOG_DESTRUCTOR("t_gstate"); }
|
||||
|
||||
void
|
||||
t_gstate::init() {
|
||||
// The master `t_data_table` is the canonical copy of the dataset and is the
|
||||
// only table eligible for on-disk backing. When `BACKING_STORE_DISK` is
|
||||
// requested, the column files live in a unique per-table directory under
|
||||
// the OS temp directory (native only; WASM/OPFS is handled separately).
|
||||
std::string dirname;
|
||||
if (m_backing_store == BACKING_STORE_DISK) {
|
||||
dirname = create_backing_store_dir("perspective_");
|
||||
}
|
||||
m_table = std::make_shared<t_data_table>(
|
||||
"", dirname, m_input_schema, DEFAULT_EMPTY_CAPACITY, m_backing_store
|
||||
);
|
||||
m_table->init();
|
||||
m_pkcol = m_table->get_column("psp_pkey");
|
||||
m_opcol = m_table->get_column("psp_op");
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
t_rlookup
|
||||
t_gstate::lookup(t_tscalar pkey) const {
|
||||
t_rlookup rval(0, false);
|
||||
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
|
||||
if (iter == m_mapping.end()) {
|
||||
return rval;
|
||||
}
|
||||
|
||||
rval.m_idx = iter->second;
|
||||
rval.m_exists = true;
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::_mark_deleted(t_uindex idx) {
|
||||
m_free.insert(idx);
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::erase(const t_tscalar& pkey) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
|
||||
if (iter == m_mapping.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto columns = m_table->get_columns();
|
||||
|
||||
t_uindex idx = iter->second;
|
||||
|
||||
for (auto* c : columns) {
|
||||
c->clear(idx);
|
||||
}
|
||||
|
||||
m_mapping.erase(iter);
|
||||
_mark_deleted(idx);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_gstate::lookup_or_create(const t_tscalar& pkey) {
|
||||
auto pkey_ = m_symtable.get_interned_tscalar(pkey);
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey_);
|
||||
|
||||
if (iter != m_mapping.end()) {
|
||||
return iter->second;
|
||||
}
|
||||
|
||||
if (!m_free.empty()) {
|
||||
t_free_items::const_iterator iter = m_free.begin();
|
||||
t_uindex idx = *iter;
|
||||
m_free.erase(iter);
|
||||
m_mapping.emplace(pkey_, idx);
|
||||
return idx;
|
||||
}
|
||||
|
||||
t_uindex nrows = m_table->num_rows();
|
||||
if (nrows >= m_table->get_capacity() - 1) {
|
||||
m_table->reserve(std::max(
|
||||
nrows + 1,
|
||||
static_cast<t_uindex>(
|
||||
m_table->get_capacity() * PSP_TABLE_GROW_RATIO
|
||||
)
|
||||
));
|
||||
}
|
||||
|
||||
m_table->set_size(nrows + 1);
|
||||
m_opcol->set_nth<std::uint8_t>(nrows, OP_INSERT);
|
||||
m_pkcol->set_scalar(nrows, pkey);
|
||||
m_mapping.emplace(pkey_, nrows);
|
||||
return nrows;
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::fill_master_table(const std::shared_ptr<t_data_table>& flattened) {
|
||||
// insert into empty `m_table`
|
||||
m_free.clear();
|
||||
m_mapping.clear();
|
||||
|
||||
const t_schema& master_table_schema = m_table->get_schema();
|
||||
|
||||
const t_column* flattened_pkey_col =
|
||||
flattened->_get_const_column("psp_pkey");
|
||||
const t_column* flattened_op_col =
|
||||
flattened->_get_const_column("psp_op");
|
||||
|
||||
t_uindex ncols = m_table->num_columns();
|
||||
auto* master_table = m_table.get();
|
||||
const bool page_to_disk = m_backing_store == BACKING_STORE_DISK;
|
||||
|
||||
parallel_for(
|
||||
int(ncols),
|
||||
[&master_table, &master_table_schema, &flattened, page_to_disk](int idx) {
|
||||
const std::string& column_name = master_table_schema.m_columns[idx];
|
||||
auto flattened_column = flattened->get_column_safe(column_name);
|
||||
if (!flattened_column) {
|
||||
return;
|
||||
}
|
||||
if (page_to_disk) {
|
||||
// The master table is on-disk; preserve its disk-backed column
|
||||
// and deep-copy the flattened (in-memory) column's data into
|
||||
// it rather than aliasing the in-memory `shared_ptr`.
|
||||
master_table->_get_column(column_name)
|
||||
->copy_from(*flattened_column);
|
||||
} else {
|
||||
// Alias each column `shared_ptr` from flattened into `m_table`
|
||||
// rather than deep-cloning.
|
||||
master_table->set_column(idx, std::move(flattened_column));
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
m_pkcol = master_table->get_column("psp_pkey");
|
||||
m_opcol = master_table->get_column("psp_op");
|
||||
|
||||
master_table->set_capacity(flattened->get_capacity());
|
||||
master_table->set_size(flattened->size());
|
||||
|
||||
const std::uint8_t* flattened_op_base =
|
||||
flattened_op_col->get_nth_base<std::uint8_t>();
|
||||
for (t_uindex idx = 0, loop_end = flattened->num_rows(); idx < loop_end;
|
||||
++idx) {
|
||||
t_tscalar pkey = flattened_pkey_col->get_scalar(idx);
|
||||
t_op op = static_cast<t_op>(flattened_op_base[idx]);
|
||||
|
||||
switch (op) {
|
||||
case OP_INSERT: {
|
||||
// Write new primary keys into `m_mapping`
|
||||
m_mapping.emplace(
|
||||
m_symtable.get_interned_tscalar(pkey), idx
|
||||
);
|
||||
m_opcol->set_nth<std::uint8_t>(idx, OP_INSERT);
|
||||
m_pkcol->set_scalar(idx, pkey);
|
||||
} break;
|
||||
case OP_DELETE: {
|
||||
_mark_deleted(idx);
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected OP");
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef PSP_TABLE_VERIFY
|
||||
master_table->verify();
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::init_from_table(const std::shared_ptr<t_data_table>& source) {
|
||||
// Bulk-init the master table directly from `source`. Assumes:
|
||||
// - master table is empty
|
||||
// - all rows are `OP_INSERT`
|
||||
// - `psp_pkey` has no duplicates
|
||||
//
|
||||
// Column `shared_ptr`s are aliased rather than deep-cloned.
|
||||
m_free.clear();
|
||||
m_mapping.clear();
|
||||
|
||||
const t_schema& master_table_schema = m_table->get_schema();
|
||||
t_uindex ncols = m_table->num_columns();
|
||||
auto* master_table = m_table.get();
|
||||
const bool page_to_disk = m_backing_store == BACKING_STORE_DISK;
|
||||
|
||||
parallel_for(
|
||||
int(ncols),
|
||||
[&master_table, &master_table_schema, &source, page_to_disk](int idx) {
|
||||
const std::string& column_name = master_table_schema.m_columns[idx];
|
||||
auto src_col = source->get_column_safe(column_name);
|
||||
if (!src_col) {
|
||||
return;
|
||||
}
|
||||
if (page_to_disk) {
|
||||
// Preserve the disk-backed master column; deep-copy rather than
|
||||
// aliasing the in-memory source column.
|
||||
master_table->_get_column(column_name)->copy_from(*src_col);
|
||||
} else {
|
||||
master_table->set_column(idx, std::move(src_col));
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
m_pkcol = master_table->get_column("psp_pkey");
|
||||
m_opcol = master_table->get_column("psp_op");
|
||||
|
||||
master_table->set_capacity(source->get_capacity());
|
||||
master_table->set_size(source->size());
|
||||
|
||||
const t_column* pkey_col = master_table->_get_const_column("psp_pkey");
|
||||
for (t_uindex idx = 0, loop_end = source->num_rows(); idx < loop_end;
|
||||
++idx) {
|
||||
t_tscalar pkey = pkey_col->get_scalar(idx);
|
||||
m_mapping.emplace(m_symtable.get_interned_tscalar(pkey), idx);
|
||||
}
|
||||
|
||||
#ifdef PSP_TABLE_VERIFY
|
||||
master_table->verify();
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::update_master_table(
|
||||
const std::shared_ptr<t_data_table>& flattened
|
||||
) {
|
||||
if (num_rows() == 0) {
|
||||
fill_master_table(flattened);
|
||||
return;
|
||||
}
|
||||
|
||||
// Update existing `m_table`
|
||||
const t_column* flattened_pkey_col =
|
||||
flattened->_get_const_column("psp_pkey");
|
||||
const t_column* flattened_op_col =
|
||||
flattened->_get_const_column("psp_op");
|
||||
|
||||
t_data_table* master_table = m_table.get();
|
||||
std::vector<t_uindex> master_table_indexes(flattened->num_rows());
|
||||
const std::uint8_t* flattened_op_base =
|
||||
flattened_op_col->get_nth_base<std::uint8_t>();
|
||||
|
||||
for (t_uindex idx = 0, loop_end = flattened->num_rows(); idx < loop_end;
|
||||
++idx) {
|
||||
t_tscalar pkey = flattened_pkey_col->get_scalar(idx);
|
||||
t_op op = static_cast<t_op>(flattened_op_base[idx]);
|
||||
|
||||
switch (op) {
|
||||
case OP_INSERT: {
|
||||
// Lookup/create the row index in `m_table` based on pkey
|
||||
master_table_indexes[idx] = lookup_or_create(pkey);
|
||||
|
||||
// Write the op and pkey to `m_table`
|
||||
m_opcol->set_nth<std::uint8_t>(
|
||||
master_table_indexes[idx], OP_INSERT
|
||||
);
|
||||
m_pkcol->set_scalar(master_table_indexes[idx], pkey);
|
||||
} break;
|
||||
case OP_DELETE: {
|
||||
// Erase the pkey from the master table, but this does not
|
||||
// change the size as the row isn't removed, just cleared out.
|
||||
erase(pkey);
|
||||
} break;
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected OP");
|
||||
} break;
|
||||
}
|
||||
}
|
||||
|
||||
const t_schema& master_schema = m_table->get_schema();
|
||||
t_uindex ncols = master_table->num_columns();
|
||||
|
||||
parallel_for(
|
||||
int(ncols),
|
||||
[flattened,
|
||||
flattened_op_col,
|
||||
&master_schema,
|
||||
&master_table,
|
||||
&master_table_indexes,
|
||||
this](int idx) {
|
||||
const std::string& column_name = master_schema.m_columns[idx];
|
||||
t_column* master_column =
|
||||
master_table->_get_column(column_name);
|
||||
const t_column* flattened_column =
|
||||
flattened->_get_const_column_safe(column_name);
|
||||
if (!flattened_column) {
|
||||
return;
|
||||
}
|
||||
update_master_column(
|
||||
master_column,
|
||||
flattened_column,
|
||||
flattened_op_col,
|
||||
master_table_indexes,
|
||||
flattened->num_rows()
|
||||
);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::update_master_column(
|
||||
t_column* master_column,
|
||||
const t_column* flattened_column,
|
||||
const t_column* op_column,
|
||||
const std::vector<t_uindex>& master_table_indexes,
|
||||
t_uindex num_rows
|
||||
) {
|
||||
const std::uint8_t* op_base = op_column->get_nth_base<std::uint8_t>();
|
||||
for (t_uindex idx = 0, loop_end = num_rows; idx < loop_end; ++idx) {
|
||||
bool is_valid = flattened_column->is_valid(idx);
|
||||
t_uindex master_table_idx = master_table_indexes[idx];
|
||||
|
||||
if (!is_valid) {
|
||||
bool is_cleared = flattened_column->is_cleared(idx);
|
||||
if (is_cleared) {
|
||||
master_column->clear(master_table_idx);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
t_op op = static_cast<t_op>(op_base[idx]);
|
||||
|
||||
if (op == OP_DELETE) {
|
||||
continue;
|
||||
}
|
||||
|
||||
switch (flattened_column->get_dtype()) {
|
||||
case DTYPE_NONE: {
|
||||
} break;
|
||||
case DTYPE_INT64: {
|
||||
master_column->set_nth<std::int64_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::int64_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT32: {
|
||||
master_column->set_nth<std::int32_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::int32_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT16: {
|
||||
master_column->set_nth<std::int16_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::int16_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_INT8: {
|
||||
master_column->set_nth<std::int8_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::int8_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT64: {
|
||||
master_column->set_nth<std::uint64_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint64_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT32: {
|
||||
master_column->set_nth<std::uint32_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint32_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT16: {
|
||||
master_column->set_nth<std::uint16_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint16_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_UINT8: {
|
||||
master_column->set_nth<std::uint8_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint8_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_FLOAT64: {
|
||||
master_column->set_nth<double>(
|
||||
master_table_idx, *(flattened_column->get_nth<double>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_FLOAT32: {
|
||||
master_column->set_nth<float>(
|
||||
master_table_idx, *(flattened_column->get_nth<float>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_BOOL: {
|
||||
master_column->set_nth<std::uint8_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint8_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_TIME: {
|
||||
master_column->set_nth<std::int64_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::int64_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_DATE: {
|
||||
master_column->set_nth<std::uint32_t>(
|
||||
master_table_idx,
|
||||
*(flattened_column->get_nth<std::uint32_t>(idx))
|
||||
);
|
||||
} break;
|
||||
case DTYPE_STR: {
|
||||
master_column->set_nth<const char*>(
|
||||
master_table_idx, flattened_column->get_nth<const char>(idx)
|
||||
);
|
||||
} break;
|
||||
case DTYPE_OBJECT:
|
||||
default: {
|
||||
PSP_COMPLAIN_AND_ABORT("Unexpected type");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::pprint() const {
|
||||
std::vector<t_uindex> indices(m_mapping.size());
|
||||
t_uindex idx = 0;
|
||||
for (const auto& iter : m_mapping) {
|
||||
indices[idx] = iter.second;
|
||||
++idx;
|
||||
}
|
||||
m_table->pprint(indices);
|
||||
}
|
||||
|
||||
t_mask
|
||||
t_gstate::get_cpp_mask() const {
|
||||
t_uindex sz = m_table->size();
|
||||
t_mask msk(sz);
|
||||
for (const auto& iter : m_mapping) {
|
||||
msk.set(iter.second, true);
|
||||
}
|
||||
return msk;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_gstate::get_table() const {
|
||||
return m_table;
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_gstate::read_by_pkey(
|
||||
const t_data_table& table, const std::string& colname, t_tscalar& pkey
|
||||
) const {
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
if (iter != m_mapping.end()) {
|
||||
return col_->get_scalar(iter->second);
|
||||
}
|
||||
PSP_COMPLAIN_AND_ABORT("Called without pkey");
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::resolve_pkeys(
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
std::vector<t_uindex>& out_row_indices
|
||||
) const {
|
||||
t_uindex num_rows = pkeys.size();
|
||||
out_row_indices.resize(num_rows);
|
||||
constexpr t_uindex INVALID_ROW = t_uindex(-1);
|
||||
for (t_uindex idx = 0; idx < num_rows; ++idx) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
|
||||
if (iter != m_mapping.end()) {
|
||||
out_row_indices[idx] = iter->second;
|
||||
} else {
|
||||
out_row_indices[idx] = INVALID_ROW;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::read_column(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
std::vector<t_tscalar>& out_data
|
||||
) const {
|
||||
t_index num_rows = pkeys.size();
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
out_data.resize(num_rows);
|
||||
|
||||
for (t_index idx = 0; idx < num_rows; ++idx) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
|
||||
if (iter != m_mapping.end()) {
|
||||
out_data[idx].set(col_->get_scalar(iter->second));
|
||||
} else {
|
||||
out_data[idx] = t_tscalar();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::read_column(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
std::vector<double>& out_data
|
||||
) const {
|
||||
read_column(table, colname, pkeys, out_data, true);
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::read_column(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
std::vector<double>& out_data,
|
||||
bool include_nones
|
||||
) const {
|
||||
t_index num_rows = pkeys.size();
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
|
||||
out_data.clear();
|
||||
out_data.reserve(num_rows);
|
||||
for (t_index idx = 0; idx < num_rows; ++idx) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
|
||||
if (iter != m_mapping.end()) {
|
||||
auto tscalar = col_->get_scalar(iter->second);
|
||||
if (include_nones || tscalar.is_valid()) {
|
||||
out_data.push_back(tscalar.to_double());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::read_column(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
t_uindex start_idx,
|
||||
t_uindex end_idx,
|
||||
std::vector<t_tscalar>& out_data
|
||||
) const {
|
||||
t_index num_rows = end_idx - start_idx;
|
||||
|
||||
// Don't read invalid row indices.
|
||||
if (num_rows <= 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
|
||||
out_data.resize(num_rows);
|
||||
|
||||
t_uindex i = 0;
|
||||
for (t_uindex idx = start_idx; idx < end_idx; ++idx) {
|
||||
out_data[i] = col_->get_scalar(idx);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::read_column(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_uindex>& row_indices,
|
||||
std::vector<t_tscalar>& out_data
|
||||
) const {
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
|
||||
t_index num_rows = row_indices.size();
|
||||
out_data.resize(num_rows);
|
||||
|
||||
t_uindex i = 0;
|
||||
for (auto idx : row_indices) {
|
||||
out_data[i] = col_->get_scalar(idx);
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_gstate::get(
|
||||
const t_data_table& table, const std::string& colname, t_tscalar pkey
|
||||
) const {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
if (iter != m_mapping.end()) {
|
||||
return table._get_const_column(colname)->get_scalar(iter->second);
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
t_tscalar
|
||||
t_gstate::get_value(
|
||||
const t_data_table& table, const std::string& colname, const t_tscalar& pkey
|
||||
) const {
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
t_tscalar rval = mknone();
|
||||
|
||||
auto iter = m_mapping.find(pkey);
|
||||
if (iter != m_mapping.end()) {
|
||||
rval.set(col_->get_scalar(iter->second));
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_gstate::is_unique(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
t_tscalar& value
|
||||
) const {
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
value = mknone();
|
||||
|
||||
for (const auto& pkey : pkeys) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
if (iter != m_mapping.end()) {
|
||||
auto tmp = col_->get_scalar(iter->second);
|
||||
if (!value.is_none() && value != tmp) {
|
||||
return false;
|
||||
}
|
||||
value = tmp;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
t_gstate::apply(
|
||||
const t_data_table& table,
|
||||
const std::string& colname,
|
||||
const std::vector<t_tscalar>& pkeys,
|
||||
t_tscalar& value,
|
||||
const std::function<bool(const t_tscalar&, t_tscalar&)>& fn
|
||||
) const {
|
||||
const t_column* col_ = table._get_const_column(colname);
|
||||
|
||||
value = mknone();
|
||||
|
||||
for (const auto& pkey : pkeys) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
if (iter != m_mapping.end()) {
|
||||
auto tmp = col_->get_scalar(iter->second);
|
||||
bool done = fn(tmp, value);
|
||||
if (done) {
|
||||
value = tmp;
|
||||
return done;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
const t_schema&
|
||||
t_gstate::get_output_schema() const {
|
||||
return m_output_schema;
|
||||
}
|
||||
|
||||
const t_gstate::t_mapping&
|
||||
t_gstate::get_pkey_map() const {
|
||||
return m_mapping;
|
||||
}
|
||||
|
||||
t_dtype
|
||||
t_gstate::get_pkey_dtype() const {
|
||||
if (m_mapping.empty()) {
|
||||
return DTYPE_STR;
|
||||
}
|
||||
auto iter = m_mapping.begin();
|
||||
return iter->first.get_dtype();
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_gstate::get_pkeyed_table() const {
|
||||
return get_pkeyed_table(m_input_schema, m_table);
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_gstate::get_pkeyed_table(
|
||||
const t_schema& schema, const std::shared_ptr<t_data_table>& table
|
||||
) const {
|
||||
// If there are no removes, just return the gstate table. Removes would
|
||||
// cause m_mapping to be smaller than m_table.
|
||||
if (m_mapping.size() == table->size()) {
|
||||
return table;
|
||||
}
|
||||
|
||||
// Otherwise mask out the removed rows and return the table.
|
||||
auto mask = get_cpp_mask();
|
||||
|
||||
// count = total number of rows - number of removed rows
|
||||
t_uindex table_size = mask.count();
|
||||
|
||||
const auto& schema_columns = schema.m_columns;
|
||||
t_uindex num_columns = schema_columns.size();
|
||||
|
||||
std::shared_ptr<t_data_table> rval =
|
||||
std::make_shared<t_data_table>(schema, table_size);
|
||||
rval->init();
|
||||
rval->set_size(table_size);
|
||||
|
||||
parallel_for(
|
||||
int(num_columns),
|
||||
[&schema_columns, rval, table, &mask](int colidx) {
|
||||
const std::string& colname = schema_columns[colidx];
|
||||
rval->set_column(
|
||||
colname, table->_get_const_column(colname)->clone(mask)
|
||||
);
|
||||
}
|
||||
|
||||
);
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_gstate::num_rows() const {
|
||||
return m_table->num_rows();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_gstate::num_columns() const {
|
||||
return m_table->num_columns();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_gstate::get_row_data_pkeys(const std::vector<t_tscalar>& pkeys) const {
|
||||
t_uindex ncols = m_table->num_columns();
|
||||
const t_schema& schema = m_table->get_schema();
|
||||
std::vector<t_tscalar> rval;
|
||||
rval.reserve(pkeys.size() * ncols);
|
||||
|
||||
std::vector<const t_column*> columns(ncols);
|
||||
for (t_uindex idx = 0, loop_end = schema.size(); idx < loop_end; ++idx) {
|
||||
const std::string& cname = schema.m_columns[idx];
|
||||
columns[idx] = m_table->_get_const_column(cname);
|
||||
}
|
||||
|
||||
auto none = mknone();
|
||||
|
||||
for (const auto& pkey : pkeys) {
|
||||
t_mapping::const_iterator iter = m_mapping.find(pkey);
|
||||
if (iter == m_mapping.end()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
|
||||
auto v = columns[cidx]->get_scalar(iter->second);
|
||||
if (v.is_valid()) {
|
||||
rval.push_back(v);
|
||||
} else {
|
||||
rval.push_back(none);
|
||||
}
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_gstate::has_pkey(t_tscalar pkey) const {
|
||||
return m_mapping.find(pkey) != m_mapping.end();
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_gstate::has_pkeys(const std::vector<t_tscalar>& pkeys) const {
|
||||
if (pkeys.empty()) {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<t_tscalar> rval(pkeys.size());
|
||||
t_uindex idx = 0;
|
||||
|
||||
for (const auto& p : pkeys) {
|
||||
t_tscalar tval;
|
||||
tval.set(m_mapping.find(p) != m_mapping.end());
|
||||
rval[idx].set(tval);
|
||||
++idx;
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_gstate::get_pkeys() const {
|
||||
std::vector<t_tscalar> rval(m_mapping.size());
|
||||
t_uindex idx = 0;
|
||||
for (const auto& kv : m_mapping) {
|
||||
rval[idx].set(kv.first);
|
||||
++idx;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
std::pair<t_tscalar, t_tscalar>
|
||||
get_vec_min_max(const std::vector<t_tscalar>& vec) {
|
||||
t_tscalar min = mknone();
|
||||
t_tscalar max = mknone();
|
||||
|
||||
for (const auto& v : vec) {
|
||||
if (min.is_none()) {
|
||||
min = v;
|
||||
} else {
|
||||
min = std::min(v, min);
|
||||
}
|
||||
|
||||
if (max.is_none()) {
|
||||
max = v;
|
||||
} else {
|
||||
max = std::max(v, max);
|
||||
}
|
||||
}
|
||||
|
||||
return std::pair<t_tscalar, t_tscalar>(min, max);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_gstate::mapping_size() const {
|
||||
return m_mapping.size();
|
||||
}
|
||||
|
||||
void
|
||||
t_gstate::reset() {
|
||||
m_table->reset();
|
||||
m_mapping.clear();
|
||||
m_free.clear();
|
||||
}
|
||||
|
||||
const t_schema&
|
||||
t_gstate::get_input_schema() const {
|
||||
return m_input_schema;
|
||||
}
|
||||
|
||||
std::vector<t_uindex>
|
||||
t_gstate::get_pkeys_idx(const std::vector<t_tscalar>& pkeys) const {
|
||||
std::vector<t_uindex> rv;
|
||||
rv.reserve(pkeys.size());
|
||||
|
||||
for (const auto& p : pkeys) {
|
||||
auto lk = lookup(p);
|
||||
std::cout << "pkey " << p << " exists " << lk.m_exists << '\n';
|
||||
if (lk.m_exists) {
|
||||
rv.push_back(lk.m_idx);
|
||||
}
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,234 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include "perspective/base.h"
|
||||
#include "perspective/heap_instruments.h"
|
||||
#include <cstddef>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <emscripten/emscripten.h>
|
||||
#include <emscripten/heap.h>
|
||||
#include <emscripten/em_asm.h>
|
||||
#include <emscripten/stack.h>
|
||||
#include <string>
|
||||
|
||||
static std::uint64_t USED_MEMORY = 0;
|
||||
|
||||
static constexpr std::uint64_t MIN_RELEVANT_SIZE = 5 * 1024 * 1024;
|
||||
|
||||
extern "C" void
|
||||
psp_print_used_memory() {
|
||||
printf("Used memory: %llu\n", USED_MEMORY);
|
||||
}
|
||||
|
||||
struct Header {
|
||||
std::uint64_t size;
|
||||
};
|
||||
|
||||
using UnderlyingString =
|
||||
std::basic_string<char, std::char_traits<char>, UnderlyingAllocator<char>>;
|
||||
|
||||
using UnderlyingIStringStream = std::basic_istringstream<
|
||||
char,
|
||||
std::char_traits<char>,
|
||||
UnderlyingAllocator<char>>;
|
||||
|
||||
struct AllocMeta {
|
||||
Header header;
|
||||
const UnderlyingString* trace;
|
||||
std::uint64_t size;
|
||||
};
|
||||
|
||||
static std::unordered_map<
|
||||
UnderlyingString,
|
||||
AllocMeta,
|
||||
std::hash<UnderlyingString>,
|
||||
std::equal_to<>,
|
||||
UnderlyingAllocator<std::pair<UnderlyingString const, AllocMeta>>>
|
||||
stack_traces;
|
||||
|
||||
static UnderlyingString IRRELEVANT = "irrelevant";
|
||||
|
||||
static inline void
|
||||
record_stack_trace(Header* header, std::uint64_t size) {
|
||||
if (size >= MIN_RELEVANT_SIZE) {
|
||||
const char* stack_c_str = perspective::psp_stack_trace();
|
||||
UnderlyingIStringStream stack(stack_c_str);
|
||||
UnderlyingString line;
|
||||
UnderlyingString out;
|
||||
|
||||
while (std::getline(stack, line)) {
|
||||
line = line.substr(0, line.find_last_of(" ("));
|
||||
out += line + "\n";
|
||||
}
|
||||
|
||||
emscripten_builtin_free(const_cast<char*>(stack_c_str));
|
||||
|
||||
// stack_traces[ptr] = {.header = *header, .trace = out};
|
||||
if (stack_traces.find(out) == stack_traces.end()) {
|
||||
stack_traces[out] =
|
||||
AllocMeta{.header = *header, .trace = nullptr, .size = size};
|
||||
|
||||
stack_traces[out].trace = &stack_traces.find(out)->first;
|
||||
} else {
|
||||
stack_traces[out].size += size;
|
||||
}
|
||||
} else {
|
||||
if (stack_traces.find(IRRELEVANT) == stack_traces.end()) {
|
||||
stack_traces[IRRELEVANT] = AllocMeta{
|
||||
.header = *header, .trace = &IRRELEVANT, .size = size
|
||||
};
|
||||
} else {
|
||||
stack_traces[IRRELEVANT].size += size;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void
|
||||
psp_dump_stack_traces() {
|
||||
std::vector<AllocMeta, UnderlyingAllocator<AllocMeta>> metas;
|
||||
metas.reserve(stack_traces.size());
|
||||
for (const auto& [_, meta] : stack_traces) {
|
||||
metas.push_back(meta);
|
||||
}
|
||||
std::sort(
|
||||
metas.begin(),
|
||||
metas.end(),
|
||||
[](const AllocMeta& a, const AllocMeta& b) {
|
||||
return a.header.size > b.header.size;
|
||||
}
|
||||
);
|
||||
for (const auto& meta : metas) {
|
||||
printf("Allocated %llu bytes\n", meta.header.size);
|
||||
printf("Stacktrace:\n%s\n", meta.trace->c_str());
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" void
|
||||
psp_clear_stack_traces() {
|
||||
stack_traces.clear();
|
||||
}
|
||||
|
||||
void*
|
||||
malloc(size_t size) {
|
||||
if (size > MIN_RELEVANT_SIZE) {
|
||||
printf("Allocating %zu bytes\n", size);
|
||||
}
|
||||
USED_MEMORY += size;
|
||||
const size_t total_size = size + sizeof(Header);
|
||||
auto* header = static_cast<Header*>(emscripten_builtin_malloc(total_size));
|
||||
if (header == nullptr) {
|
||||
fprintf(stderr, "Failed to allocate %zu bytes\n", size);
|
||||
}
|
||||
header->size = size;
|
||||
record_stack_trace(header, size);
|
||||
return header + 1;
|
||||
}
|
||||
|
||||
void*
|
||||
calloc(size_t nmemb, size_t size) {
|
||||
// printf("Allocating array: %zu elements of size %zu\n", nmemb, size);
|
||||
USED_MEMORY += nmemb * size;
|
||||
// return emscripten_builtin_calloc(nmemb, size);
|
||||
const size_t total_size = (nmemb * size) + sizeof(Header);
|
||||
auto* header = static_cast<Header*>(emscripten_builtin_malloc(total_size));
|
||||
if (header == nullptr) {
|
||||
fprintf(stderr, "Failed to allocate %zu bytes\n", size);
|
||||
}
|
||||
header->size = nmemb * size;
|
||||
memset(header + 1, 0, nmemb * size);
|
||||
record_stack_trace(header, size);
|
||||
return header + 1;
|
||||
}
|
||||
|
||||
void
|
||||
free(void* ptr) {
|
||||
// printf("Freeing memory at %p\n", ptr);
|
||||
|
||||
if (ptr == nullptr) {
|
||||
emscripten_builtin_free(ptr);
|
||||
} else {
|
||||
auto* header = static_cast<Header*>(ptr) - 1;
|
||||
auto old_memory = USED_MEMORY;
|
||||
USED_MEMORY -= header->size;
|
||||
if (USED_MEMORY > old_memory) {
|
||||
std::abort();
|
||||
}
|
||||
emscripten_builtin_free(header);
|
||||
}
|
||||
}
|
||||
|
||||
void*
|
||||
memalign(size_t alignment, size_t size) {
|
||||
const size_t total_size = size + sizeof(Header);
|
||||
auto* header =
|
||||
static_cast<Header*>(emscripten_builtin_memalign(alignment, total_size)
|
||||
);
|
||||
if (header == nullptr) {
|
||||
fprintf(stderr, "Failed to allocate %zu bytes\n", size);
|
||||
}
|
||||
header->size = size;
|
||||
record_stack_trace(header, size);
|
||||
return header + 1;
|
||||
}
|
||||
|
||||
int
|
||||
posix_memalign(void** memptr, size_t alignment, size_t size) {
|
||||
auto* header = static_cast<Header*>(
|
||||
emscripten_builtin_memalign(alignment, size + sizeof(Header))
|
||||
);
|
||||
if (header == nullptr) {
|
||||
fprintf(stderr, "Failed to allocate %zu bytes\n", size);
|
||||
}
|
||||
header->size = size;
|
||||
USED_MEMORY += size;
|
||||
record_stack_trace(header, size);
|
||||
*memptr = header + 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void*
|
||||
realloc(void* ptr, size_t new_size) {
|
||||
if (ptr == nullptr) {
|
||||
// If ptr is nullptr, realloc behaves like malloc
|
||||
return malloc(new_size);
|
||||
}
|
||||
|
||||
if (new_size == 0) {
|
||||
// If new_size is 0, realloc behaves like free
|
||||
free(ptr);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
auto* header = static_cast<Header*>(ptr) - 1;
|
||||
const size_t old_size = header->size;
|
||||
|
||||
if (new_size <= old_size) {
|
||||
USED_MEMORY -= old_size - new_size;
|
||||
// If the new size is smaller or equal, we can potentially shrink the
|
||||
// block in place. For simplicity, we don't actually shrink the block
|
||||
// here.
|
||||
header->size = new_size; // Update the size in the header
|
||||
return ptr; // Return the same pointer
|
||||
}
|
||||
|
||||
// If the new size is larger, allocate a new block
|
||||
void* new_ptr = malloc(new_size);
|
||||
if (new_ptr == nullptr) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
memcpy(new_ptr, ptr, old_size);
|
||||
free(ptr);
|
||||
|
||||
return new_ptr;
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/init.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
void
|
||||
perspective_init() {}
|
||||
|
||||
void
|
||||
perspective_finalize() {}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,451 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include "perspective/join_engine.h"
|
||||
#include "perspective/column.h"
|
||||
#include "perspective/data_table.h"
|
||||
#include "perspective/gnode.h"
|
||||
#include "perspective/scalar.h"
|
||||
#include <algorithm>
|
||||
#include <sstream>
|
||||
#include <tsl/hopscotch_map.h>
|
||||
#include <tsl/hopscotch_set.h>
|
||||
|
||||
namespace perspective::server {
|
||||
|
||||
namespace {
|
||||
|
||||
// Typed column copy for fixed-size types.
|
||||
template <typename T>
|
||||
void
|
||||
copy_column_typed(
|
||||
t_column* dst,
|
||||
const t_column* src,
|
||||
const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
|
||||
t_uindex num_matched,
|
||||
bool use_first
|
||||
) {
|
||||
for (t_uindex i = 0; i < num_matched; ++i) {
|
||||
auto src_idx = use_first ? matched_rows[i].first : matched_rows[i].second;
|
||||
if (src_idx == static_cast<t_uindex>(-1)) {
|
||||
dst->clear(i);
|
||||
} else {
|
||||
const T* val = src->get_nth<T>(src_idx);
|
||||
const t_status* st =
|
||||
src->is_status_enabled() ? src->get_nth_status(src_idx)
|
||||
: nullptr;
|
||||
dst->set_nth<T>(i, *val, st ? *st : STATUS_VALID);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
copy_column_str(
|
||||
t_column* dst,
|
||||
const t_column* src,
|
||||
const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
|
||||
t_uindex num_matched,
|
||||
bool use_first
|
||||
) {
|
||||
for (t_uindex i = 0; i < num_matched; ++i) {
|
||||
auto src_idx = use_first ? matched_rows[i].first : matched_rows[i].second;
|
||||
if (src_idx == static_cast<t_uindex>(-1)) {
|
||||
dst->clear(i);
|
||||
} else {
|
||||
dst->set_scalar(i, src->get_scalar(src_idx));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
copy_column_dispatch(
|
||||
t_column* dst,
|
||||
const t_column* src,
|
||||
const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
|
||||
t_uindex num_matched,
|
||||
bool use_first
|
||||
) {
|
||||
switch (dst->get_dtype()) {
|
||||
case DTYPE_INT64:
|
||||
copy_column_typed<std::int64_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_INT32:
|
||||
copy_column_typed<std::int32_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_INT16:
|
||||
copy_column_typed<std::int16_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_INT8:
|
||||
copy_column_typed<std::int8_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_UINT64:
|
||||
copy_column_typed<std::uint64_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_UINT32:
|
||||
copy_column_typed<std::uint32_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_UINT16:
|
||||
copy_column_typed<std::uint16_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_UINT8:
|
||||
copy_column_typed<std::uint8_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_FLOAT64:
|
||||
copy_column_typed<double>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_FLOAT32:
|
||||
copy_column_typed<float>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_BOOL:
|
||||
copy_column_typed<bool>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_TIME:
|
||||
copy_column_typed<std::int64_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_DATE:
|
||||
copy_column_typed<std::int32_t>(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
case DTYPE_STR:
|
||||
copy_column_str(
|
||||
dst, src, matched_rows, num_matched, use_first
|
||||
);
|
||||
break;
|
||||
default:
|
||||
// Fallback for any other types (OBJECT, F64PAIR, etc.)
|
||||
for (t_uindex i = 0; i < num_matched; ++i) {
|
||||
auto src_idx = use_first ? matched_rows[i].first
|
||||
: matched_rows[i].second;
|
||||
if (src_idx == static_cast<t_uindex>(-1)) {
|
||||
dst->clear(i);
|
||||
} else {
|
||||
dst->set_scalar(i, src->get_scalar(src_idx));
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Copy the join key column for OUTER join rows where the left side has no
|
||||
// match but the right side does. Uses the right key column as the source
|
||||
// for just those rows; all other rows are already filled from the left.
|
||||
void
|
||||
copy_join_key_fallback(
|
||||
t_column* dst,
|
||||
const t_column* right_key_col,
|
||||
const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
|
||||
t_uindex num_matched
|
||||
) {
|
||||
for (t_uindex i = 0; i < num_matched; ++i) {
|
||||
if (matched_rows[i].first == static_cast<t_uindex>(-1)
|
||||
&& matched_rows[i].second != static_cast<t_uindex>(-1)) {
|
||||
dst->set_scalar(
|
||||
i, right_key_col->get_scalar(matched_rows[i].second)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void
|
||||
JoinEngine::register_join(
|
||||
const t_id& join_table_id,
|
||||
const t_id& left_table_id,
|
||||
const t_id& right_table_id,
|
||||
const std::string& on_column,
|
||||
const std::string& right_on_column,
|
||||
proto::JoinType join_type
|
||||
) {
|
||||
auto effective_right_on = right_on_column.empty() ? on_column : right_on_column;
|
||||
JoinDef def{left_table_id, right_table_id, on_column, effective_right_on, join_type};
|
||||
m_join_defs.emplace(join_table_id, def);
|
||||
m_table_to_join_tables.emplace(left_table_id, join_table_id);
|
||||
m_table_to_join_tables.emplace(right_table_id, join_table_id);
|
||||
}
|
||||
|
||||
void
|
||||
JoinEngine::unregister_join(const t_id& join_table_id) {
|
||||
auto it = m_join_defs.find(join_table_id);
|
||||
if (it == m_join_defs.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto& def = it->second;
|
||||
|
||||
for (const auto& source_id : {def.left_table_id, def.right_table_id}) {
|
||||
auto range = m_table_to_join_tables.equal_range(source_id);
|
||||
for (auto jt = range.first; jt != range.second;) {
|
||||
if (jt->second == join_table_id) {
|
||||
jt = m_table_to_join_tables.erase(jt);
|
||||
} else {
|
||||
++jt;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
m_join_defs.erase(it);
|
||||
m_caches.erase(join_table_id);
|
||||
}
|
||||
|
||||
bool
|
||||
JoinEngine::is_join_table(const t_id& id) const {
|
||||
return m_join_defs.contains(id);
|
||||
}
|
||||
|
||||
std::vector<JoinEngine::t_id>
|
||||
JoinEngine::get_dependent_join_tables(const t_id& source_table_id) const {
|
||||
std::vector<t_id> result;
|
||||
auto range = m_table_to_join_tables.equal_range(source_table_id);
|
||||
for (auto it = range.first; it != range.second; ++it) {
|
||||
result.push_back(it->second);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
const JoinDef&
|
||||
JoinEngine::get_join_def(const t_id& join_table_id) const {
|
||||
return m_join_defs.at(join_table_id);
|
||||
}
|
||||
|
||||
MakeJoinResult
|
||||
JoinEngine::make_join_table(
|
||||
const std::string& on_column,
|
||||
const std::string& right_on_column,
|
||||
proto::JoinType join_type,
|
||||
const std::shared_ptr<Table>& left_table,
|
||||
const std::shared_ptr<Table>& right_table
|
||||
) {
|
||||
auto effective_right_on = right_on_column.empty() ? on_column : right_on_column;
|
||||
auto left_schema = left_table->get_schema();
|
||||
auto right_schema = right_table->get_schema();
|
||||
if (!left_schema.has_column(on_column)) {
|
||||
std::stringstream ss;
|
||||
ss << "Column \"" << on_column << "\" not found in left table";
|
||||
return {nullptr, ss.str()};
|
||||
}
|
||||
|
||||
if (!right_schema.has_column(effective_right_on)) {
|
||||
std::stringstream ss;
|
||||
ss << "Column \"" << effective_right_on << "\" not found in right table";
|
||||
return {nullptr, ss.str()};
|
||||
}
|
||||
|
||||
if (left_schema.get_dtype(on_column) != right_schema.get_dtype(effective_right_on)) {
|
||||
return {nullptr, "Join column type mismatch"};
|
||||
}
|
||||
|
||||
for (const auto& rcol : right_schema.columns()) {
|
||||
if (rcol == effective_right_on) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (left_schema.has_column(rcol)) {
|
||||
std::stringstream ss;
|
||||
ss << "Column \"" << rcol << "\" exists in both tables";
|
||||
return {nullptr, ss.str()};
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::string> merged_columns;
|
||||
std::vector<t_dtype> merged_types;
|
||||
for (t_uindex i = 0; i < left_schema.columns().size(); ++i) {
|
||||
merged_columns.push_back(left_schema.columns()[i]);
|
||||
merged_types.push_back(left_schema.types()[i]);
|
||||
}
|
||||
|
||||
for (t_uindex i = 0; i < right_schema.columns().size(); ++i) {
|
||||
if (right_schema.columns()[i] == effective_right_on) {
|
||||
continue;
|
||||
}
|
||||
|
||||
merged_columns.push_back(right_schema.columns()[i]);
|
||||
merged_types.push_back(right_schema.types()[i]);
|
||||
}
|
||||
|
||||
t_schema merged_schema(merged_columns, merged_types);
|
||||
auto join_table = Table::from_schema("", merged_schema);
|
||||
return {join_table, ""};
|
||||
}
|
||||
|
||||
void
|
||||
JoinEngine::build_right_index(
|
||||
JoinCache& cache,
|
||||
const std::shared_ptr<Table>& right_table,
|
||||
const std::string& on_column
|
||||
) {
|
||||
auto right_data = right_table->get_gnode()->get_table_sptr();
|
||||
const auto& right_pkey_map = right_table->get_gnode()->get_pkey_map();
|
||||
auto right_key_col = right_data->get_column(on_column);
|
||||
cache.right_entries.assign(right_pkey_map.begin(), right_pkey_map.end());
|
||||
std::sort(
|
||||
cache.right_entries.begin(),
|
||||
cache.right_entries.end(),
|
||||
[](const auto& a, const auto& b) { return a.first < b.first; }
|
||||
);
|
||||
|
||||
cache.right_index.clear();
|
||||
cache.right_index.reserve(right_pkey_map.size());
|
||||
for (const auto& [pkey, row_idx] : cache.right_entries) {
|
||||
auto join_key = right_key_col->get_scalar(row_idx);
|
||||
if (!join_key.is_none()) {
|
||||
cache.right_index[join_key].push_back(row_idx);
|
||||
}
|
||||
}
|
||||
|
||||
cache.valid = true;
|
||||
}
|
||||
|
||||
void
|
||||
JoinEngine::recompute(
|
||||
const t_id& join_table_id,
|
||||
const std::shared_ptr<Table>& left_table,
|
||||
const std::shared_ptr<Table>& right_table,
|
||||
const std::shared_ptr<Table>& join_table,
|
||||
bool left_changed,
|
||||
bool right_changed
|
||||
) {
|
||||
const auto& def = m_join_defs.at(join_table_id);
|
||||
auto& cache = m_caches[join_table_id];
|
||||
auto left_data = left_table->get_gnode()->get_table_sptr();
|
||||
auto right_data = right_table->get_gnode()->get_table_sptr();
|
||||
const auto& left_pkey_map = left_table->get_gnode()->get_pkey_map();
|
||||
auto left_key_col = left_data->get_column(def.on_column);
|
||||
auto right_key_col = right_data->get_column(def.right_on_column);
|
||||
|
||||
// Rebuild right-side index only when the right table has changed,
|
||||
// or on the first recompute when no cache exists yet.
|
||||
if (right_changed || !cache.valid) {
|
||||
build_right_index(cache, right_table, def.right_on_column);
|
||||
}
|
||||
|
||||
const auto& right_join_key_to_rows = cache.right_index;
|
||||
const auto& right_entries = cache.right_entries;
|
||||
|
||||
// Sort left pkey entries so the join result preserves left-table
|
||||
// insertion order.
|
||||
std::vector<std::pair<t_tscalar, t_uindex>> left_entries(
|
||||
left_pkey_map.begin(), left_pkey_map.end()
|
||||
);
|
||||
|
||||
std::sort(
|
||||
left_entries.begin(),
|
||||
left_entries.end(),
|
||||
[](const auto& a, const auto& b) { return a.first < b.first; }
|
||||
);
|
||||
|
||||
const t_uindex NO_MATCH = static_cast<t_uindex>(-1);
|
||||
std::vector<std::pair<t_uindex, t_uindex>> matched_rows;
|
||||
matched_rows.reserve(left_entries.size());
|
||||
tsl::hopscotch_set<t_uindex> matched_right_rows;
|
||||
for (const auto& [pkey, row_idx] : left_entries) {
|
||||
auto join_key = left_key_col->get_scalar(row_idx);
|
||||
if (join_key.is_none()) {
|
||||
if (def.join_type == proto::LEFT
|
||||
|| def.join_type == proto::OUTER) {
|
||||
matched_rows.emplace_back(row_idx, NO_MATCH);
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
auto it = right_join_key_to_rows.find(join_key);
|
||||
if (it != right_join_key_to_rows.end()) {
|
||||
for (auto right_row_idx : it->second) {
|
||||
matched_rows.emplace_back(row_idx, right_row_idx);
|
||||
if (def.join_type == proto::OUTER) {
|
||||
matched_right_rows.insert(right_row_idx);
|
||||
}
|
||||
}
|
||||
} else if (def.join_type == proto::LEFT
|
||||
|| def.join_type == proto::OUTER) {
|
||||
matched_rows.emplace_back(row_idx, NO_MATCH);
|
||||
}
|
||||
}
|
||||
|
||||
if (def.join_type == proto::OUTER) {
|
||||
for (const auto& [pkey, row_idx] : right_entries) {
|
||||
if (matched_right_rows.find(row_idx)
|
||||
== matched_right_rows.end()) {
|
||||
matched_rows.emplace_back(NO_MATCH, row_idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_uindex num_matched = matched_rows.size();
|
||||
auto join_schema = join_table->get_schema();
|
||||
t_data_table joined_data(join_schema);
|
||||
joined_data.init();
|
||||
joined_data.extend(num_matched);
|
||||
auto left_schema = left_table->get_schema();
|
||||
auto right_schema = right_table->get_schema();
|
||||
for (const auto& col_name : join_schema.columns()) {
|
||||
auto dst_col = joined_data.get_column(col_name);
|
||||
bool is_join_col = (col_name == def.on_column);
|
||||
if (left_schema.has_column(col_name)) {
|
||||
auto left_src_col = left_data->get_column(col_name);
|
||||
copy_column_dispatch(
|
||||
dst_col.get(), left_src_col.get(), matched_rows, num_matched, true
|
||||
);
|
||||
|
||||
if (is_join_col) {
|
||||
copy_join_key_fallback(
|
||||
dst_col.get(), right_key_col.get(), matched_rows, num_matched
|
||||
);
|
||||
}
|
||||
} else if (right_schema.has_column(col_name)) {
|
||||
auto src_col = right_data->get_column(col_name);
|
||||
copy_column_dispatch(
|
||||
dst_col.get(), src_col.get(), matched_rows, num_matched, false
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
joined_data.set_size(num_matched);
|
||||
auto* pkey_col = joined_data.add_column("psp_pkey", DTYPE_INT32, true);
|
||||
auto* okey_col = joined_data.add_column("psp_okey", DTYPE_INT32, true);
|
||||
for (t_uindex i = 0; i < num_matched; ++i) {
|
||||
pkey_col->set_nth<std::int32_t>(i, static_cast<std::int32_t>(i), STATUS_VALID);
|
||||
okey_col->set_nth<std::int32_t>(i, static_cast<std::int32_t>(i), STATUS_VALID);
|
||||
}
|
||||
|
||||
join_table->clear();
|
||||
join_table->init(joined_data, num_matched, t_op::OP_INSERT, 0);
|
||||
}
|
||||
|
||||
} // namespace perspective::server
|
||||
@@ -0,0 +1,24 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/kernel_engine.h>
|
||||
|
||||
namespace perspective {
|
||||
t_kernel_evaluator::t_kernel_evaluator() {}
|
||||
|
||||
t_kernel_evaluator*
|
||||
get_evaluator() {
|
||||
static t_kernel_evaluator* evaluator = new t_kernel_evaluator();
|
||||
return evaluator;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,20 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
/**
|
||||
* Main
|
||||
*/
|
||||
int
|
||||
main(int argc, char** argv) {
|
||||
std::cout << "Perspective initialized successfully"
|
||||
<< "\n";
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/mask.h>
|
||||
#include <perspective/raii.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_mask::t_mask() { LOG_CONSTRUCTOR("t_mask"); }
|
||||
|
||||
t_mask::t_mask(t_uindex size) : m_bitmap(t_msize(size)) {
|
||||
LOG_CONSTRUCTOR("t_mask");
|
||||
}
|
||||
|
||||
t_mask::t_mask(const t_simple_bitmask& m) {
|
||||
m_bitmap = boost::dynamic_bitset<>(static_cast<size_t>(m.size()));
|
||||
|
||||
for (t_uindex idx = 0, loop_end = m.size(); idx < loop_end; ++idx) {
|
||||
set(idx, m.is_set(idx));
|
||||
}
|
||||
}
|
||||
|
||||
t_mask::~t_mask() { LOG_DESTRUCTOR("t_mask"); }
|
||||
|
||||
void
|
||||
t_mask::clear() {
|
||||
m_bitmap.clear();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask::count() const {
|
||||
return m_bitmap.count();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask::size() const {
|
||||
return m_bitmap.size();
|
||||
}
|
||||
|
||||
bool
|
||||
t_mask::get(t_uindex idx) const {
|
||||
return m_bitmap.test(t_msize(idx));
|
||||
}
|
||||
|
||||
void
|
||||
t_mask::set(t_uindex idx, bool v) {
|
||||
m_bitmap.set(t_msize(idx), v);
|
||||
}
|
||||
|
||||
void
|
||||
t_mask::set(t_uindex idx) {
|
||||
m_bitmap.set(t_msize(idx), true);
|
||||
}
|
||||
|
||||
t_mask&
|
||||
t_mask::operator&=(const t_mask& b) {
|
||||
m_bitmap &= b.m_bitmap;
|
||||
return *this;
|
||||
}
|
||||
|
||||
t_mask&
|
||||
t_mask::operator|=(const t_mask& b) {
|
||||
m_bitmap |= b.m_bitmap;
|
||||
return *this;
|
||||
}
|
||||
|
||||
t_mask&
|
||||
t_mask::operator^=(const t_mask& b) {
|
||||
m_bitmap ^= b.m_bitmap;
|
||||
return *this;
|
||||
}
|
||||
|
||||
t_mask&
|
||||
t_mask::operator-=(const t_mask& b) {
|
||||
m_bitmap -= b.m_bitmap;
|
||||
return *this;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask::find_first() const {
|
||||
return m_bitmap.find_first();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask::find_next(t_uindex pos) const {
|
||||
return m_bitmap.find_next(t_msize(pos));
|
||||
}
|
||||
|
||||
void
|
||||
t_mask::pprint() const {
|
||||
std::cout << *this << '\n';
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask_iterator::next() {
|
||||
t_uindex rval = m_pos;
|
||||
m_pos = m_mask->find_next(rval);
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_mask_iterator::has_next() const {
|
||||
return m_pos != m_end;
|
||||
}
|
||||
|
||||
t_mask_iterator::t_mask_iterator(t_maskcsptr m) :
|
||||
m_mask(std::move(std::move(m))),
|
||||
m_pos(m_mask->find_first()) {
|
||||
LOG_CONSTRUCTOR("t_mask_iterator");
|
||||
}
|
||||
|
||||
t_mask_iterator::t_mask_iterator() { LOG_CONSTRUCTOR("t_mask_iterator"); }
|
||||
|
||||
t_uindex
|
||||
t_mask_iterator::size() const {
|
||||
return m_mask->size();
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mask_iterator::count() const {
|
||||
return m_mask->count();
|
||||
}
|
||||
|
||||
t_mask_iterator::~t_mask_iterator() { LOG_DESTRUCTOR("t_mask_iterator"); }
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
namespace std {
|
||||
std::ostream&
|
||||
operator<<(std::ostream& os, const perspective::t_mask& mask) {
|
||||
std::cout << "t_mask<\n";
|
||||
for (perspective::t_uindex idx = 0, loop_end = mask.size(); idx < loop_end;
|
||||
++idx) {
|
||||
std::cout << "\t" << idx << ". " << mask.get(idx) << '\n';
|
||||
}
|
||||
std::cout << ">\n";
|
||||
return os;
|
||||
}
|
||||
} // namespace std
|
||||
@@ -0,0 +1,204 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <boost/math/special_functions/fpclassify.hpp>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/multi_sort.h>
|
||||
#include <perspective/scalar.h>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_mselem::t_mselem() :
|
||||
m_pkey(mknone()),
|
||||
m_order(0),
|
||||
m_deleted(false),
|
||||
m_updated(false) {}
|
||||
|
||||
t_mselem::t_mselem(const std::vector<t_tscalar>& row) :
|
||||
m_row(row),
|
||||
m_pkey(mknone()),
|
||||
m_order(0),
|
||||
m_deleted(false),
|
||||
m_updated(false) {}
|
||||
|
||||
t_mselem::t_mselem(const std::vector<t_tscalar>& row, t_uindex order) :
|
||||
m_row(row),
|
||||
m_pkey(mknone()),
|
||||
m_order(order),
|
||||
m_deleted(false),
|
||||
m_updated(false) {}
|
||||
|
||||
t_mselem::t_mselem(const t_tscalar& pkey, const std::vector<t_tscalar>& row) :
|
||||
m_row(row),
|
||||
m_pkey(pkey),
|
||||
m_order(0),
|
||||
m_deleted(false),
|
||||
m_updated(false) {}
|
||||
|
||||
t_minmax_idx::t_minmax_idx(t_index mn, t_index mx) : m_min(mn), m_max(mx) {}
|
||||
|
||||
// Given a vector return the indices of the
|
||||
// minimum and maximum elements in it.
|
||||
t_minmax_idx
|
||||
get_minmax_idx(const std::vector<t_tscalar>& vec, t_sorttype stype) {
|
||||
t_minmax_idx rval(-1, -1);
|
||||
|
||||
if (vec.empty()) {
|
||||
return rval;
|
||||
}
|
||||
|
||||
// min, max
|
||||
std::pair<t_tscalar, t_tscalar> min_max;
|
||||
min_max.first = *(vec.begin());
|
||||
min_max.second = *(vec.begin());
|
||||
|
||||
switch (stype) {
|
||||
case SORTTYPE_DESCENDING:
|
||||
case SORTTYPE_ASCENDING: {
|
||||
for (t_index idx = 0, loop_end = vec.size(); idx < loop_end;
|
||||
++idx) {
|
||||
const t_tscalar& val = vec[idx];
|
||||
if (val <= min_max.first) {
|
||||
min_max.first = val;
|
||||
rval.m_min = idx;
|
||||
}
|
||||
|
||||
if (val >= min_max.second) {
|
||||
min_max.second = val;
|
||||
rval.m_max = idx;
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
} break;
|
||||
case SORTTYPE_ASCENDING_ABS:
|
||||
case SORTTYPE_DESCENDING_ABS: {
|
||||
double mindbl = std::abs(vec[0].to_double());
|
||||
double maxdbl = mindbl;
|
||||
rval.m_min = 0;
|
||||
rval.m_max = 0;
|
||||
for (t_index idx = 1, loop_end = vec.size(); idx < loop_end;
|
||||
++idx) {
|
||||
double val = std::abs(vec[idx].to_double());
|
||||
|
||||
if (val <= mindbl) {
|
||||
mindbl = val;
|
||||
rval.m_min = idx;
|
||||
}
|
||||
|
||||
if (val >= maxdbl) {
|
||||
maxdbl = val;
|
||||
rval.m_max = idx;
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
} break;
|
||||
case SORTTYPE_NONE: {
|
||||
rval.m_min = 0;
|
||||
rval.m_max = 0;
|
||||
return rval;
|
||||
} break;
|
||||
default: {
|
||||
return rval;
|
||||
}
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
double
|
||||
to_double(const t_tscalar& c) {
|
||||
return c.to_double();
|
||||
}
|
||||
|
||||
t_nancmp::t_nancmp() : m_active(false), m_cmpval(CMP_OP_EQ) {}
|
||||
|
||||
t_nancmp
|
||||
nan_compare(t_sorttype order, const t_tscalar& a, const t_tscalar& b) {
|
||||
t_nancmp rval;
|
||||
bool a_fp = a.is_floating_point();
|
||||
bool b_fp = b.is_floating_point();
|
||||
|
||||
if (!a_fp && !b_fp) {
|
||||
return rval;
|
||||
}
|
||||
|
||||
double a_dbl = a.to_double();
|
||||
double b_dbl = b.to_double();
|
||||
|
||||
bool a_nan = std::isnan(a_dbl);
|
||||
bool b_nan = std::isnan(b_dbl);
|
||||
|
||||
rval.m_active = a_nan || b_nan;
|
||||
|
||||
if (!rval.m_active) {
|
||||
return rval;
|
||||
}
|
||||
|
||||
if (a_nan && b_nan) {
|
||||
rval.m_cmpval = CMP_OP_EQ;
|
||||
return rval;
|
||||
}
|
||||
|
||||
if (a_nan) {
|
||||
switch (order) {
|
||||
case SORTTYPE_NONE:
|
||||
case SORTTYPE_ASCENDING:
|
||||
case SORTTYPE_ASCENDING_ABS: {
|
||||
rval.m_cmpval = CMP_OP_LT;
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING_ABS:
|
||||
case SORTTYPE_DESCENDING: {
|
||||
rval.m_cmpval = CMP_OP_GT;
|
||||
} break;
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
switch (order) {
|
||||
case SORTTYPE_NONE:
|
||||
case SORTTYPE_ASCENDING:
|
||||
case SORTTYPE_ASCENDING_ABS: {
|
||||
rval.m_cmpval = CMP_OP_GT;
|
||||
} break;
|
||||
case SORTTYPE_DESCENDING_ABS:
|
||||
case SORTTYPE_DESCENDING: {
|
||||
rval.m_cmpval = CMP_OP_LT;
|
||||
} break;
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
t_multisorter::t_multisorter(const std::vector<t_sorttype>& order) :
|
||||
m_sort_order(order) {}
|
||||
|
||||
t_multisorter::t_multisorter(
|
||||
std::shared_ptr<const std::vector<t_mselem>> elems,
|
||||
const std::vector<t_sorttype>& order
|
||||
) :
|
||||
m_sort_order(order),
|
||||
m_elems(std::move(elems)) {}
|
||||
|
||||
bool
|
||||
t_multisorter::operator()(const t_mselem& a, const t_mselem& b) const {
|
||||
return cmp_mselem(a, b, m_sort_order);
|
||||
}
|
||||
|
||||
bool
|
||||
t_multisorter::operator()(t_index a, t_index b) const {
|
||||
return this->operator()((*m_elems)[a], (*m_elems)[b]);
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,59 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/none.h>
|
||||
|
||||
#include <boost/functional/hash.hpp>
|
||||
namespace perspective {
|
||||
|
||||
t_none::t_none() = default;
|
||||
|
||||
bool
|
||||
t_none::operator==(const t_none& /*unused*/) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
t_none::operator<(const t_none& /*unused*/) const {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool
|
||||
t_none::operator<=(const t_none& /*unused*/) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
t_none::operator>(const t_none& /*unused*/) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool
|
||||
t_none::operator>=(const t_none& /*unused*/) const {
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t
|
||||
hash_value(const t_none& none) {
|
||||
boost::hash<long> hasher;
|
||||
return hasher(static_cast<long>(-1));
|
||||
}
|
||||
} // namespace perspective
|
||||
|
||||
namespace std {
|
||||
std::ostream&
|
||||
operator<<(std::ostream& os, const perspective::t_none& dt) {
|
||||
os << "<t_none>";
|
||||
return os;
|
||||
}
|
||||
} // namespace std
|
||||
@@ -0,0 +1,31 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/path.h>
|
||||
|
||||
namespace perspective {
|
||||
t_path::t_path() = default;
|
||||
|
||||
t_path::t_path(const std::vector<t_tscalar>& path) : m_path(path) {}
|
||||
|
||||
const std::vector<t_tscalar>&
|
||||
t_path::path() const {
|
||||
return m_path;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>&
|
||||
t_path::path() {
|
||||
return m_path;
|
||||
}
|
||||
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,59 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/pivot.h>
|
||||
#include <sstream>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_pivot::t_pivot(const t_pivot_recipe& r) {
|
||||
m_colname = r.m_colname;
|
||||
m_name = r.m_name;
|
||||
m_mode = r.m_mode;
|
||||
}
|
||||
|
||||
t_pivot::t_pivot(const std::string& colname) :
|
||||
m_colname(colname),
|
||||
m_name(colname),
|
||||
m_mode(PIVOT_MODE_NORMAL) {}
|
||||
|
||||
t_pivot::t_pivot(const std::string& colname, t_pivot_mode mode) :
|
||||
m_colname(colname),
|
||||
m_name(colname),
|
||||
m_mode(mode) {}
|
||||
|
||||
const std::string&
|
||||
t_pivot::name() const {
|
||||
return m_name;
|
||||
}
|
||||
|
||||
const std::string&
|
||||
t_pivot::colname() const {
|
||||
return m_colname;
|
||||
}
|
||||
|
||||
t_pivot_mode
|
||||
t_pivot::mode() const {
|
||||
return m_mode;
|
||||
}
|
||||
|
||||
t_pivot_recipe
|
||||
t_pivot::get_recipe() const {
|
||||
t_pivot_recipe rv;
|
||||
rv.m_colname = m_colname;
|
||||
rv.m_name = m_name;
|
||||
rv.m_mode = m_mode;
|
||||
return rv;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,360 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <cstdint>
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/pool.h>
|
||||
#include <perspective/update_task.h>
|
||||
#include <perspective/compat.h>
|
||||
#include <perspective/env_vars.h>
|
||||
#include <perspective/pyutils.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_updctx::t_updctx() = default;
|
||||
|
||||
t_updctx::t_updctx(t_uindex gnode_id, std::string ctx) :
|
||||
m_gnode_id(gnode_id),
|
||||
m_ctx(std::move(ctx)) {}
|
||||
|
||||
#if defined PSP_ENABLE_WASM && !defined PSP_ENABLE_PYTHON
|
||||
|
||||
t_pool::t_pool() : m_sleep(0) { m_run.clear(); }
|
||||
|
||||
#elif defined PSP_ENABLE_PYTHON
|
||||
|
||||
// t_val
|
||||
// empty_callback() {
|
||||
// return py::none();
|
||||
// }
|
||||
|
||||
t_pool::t_pool() :
|
||||
// : m_update_delegate(empty_callback())
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
m_lock(new std::shared_mutex()),
|
||||
#endif
|
||||
m_sleep(0) {
|
||||
m_run.clear();
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
t_pool::t_pool() : m_sleep(0) { m_run.clear(); }
|
||||
|
||||
#endif
|
||||
|
||||
t_pool::~t_pool() {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
delete m_lock;
|
||||
#endif
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::init() {
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.init " << std::endl;
|
||||
}
|
||||
m_run.test_and_set(std::memory_order_acquire);
|
||||
m_data_remaining.store(false);
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_pool::register_gnode(t_gnode* node) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_WRITE_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
m_gnodes.push_back(node);
|
||||
t_uindex id = m_gnodes.size() - 1;
|
||||
node->set_id(id);
|
||||
node->set_pool_cleanup([this, id]() { this->m_gnodes[id] = nullptr; });
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
node->set_lock(m_lock);
|
||||
#endif
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.register_gnode node => " << node << " rv => " << id
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
return id;
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::unregister_gnode(t_uindex idx) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_WRITE_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.unregister_gnode idx => " << idx << std::endl;
|
||||
}
|
||||
|
||||
m_gnodes[idx] = nullptr;
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::send(t_uindex gnode_id, t_uindex port_id, const t_data_table& table) {
|
||||
{
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_WRITE_LOCK(*m_lock);
|
||||
#endif
|
||||
// std::lock_guard<std::mutex> lg(m_mtx);
|
||||
m_data_remaining.store(true);
|
||||
|
||||
if (m_gnodes[gnode_id] != nullptr) {
|
||||
m_gnodes[gnode_id]->send(port_id, table);
|
||||
}
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.send gnode_id => " << gnode_id
|
||||
<< " port_id => " << port_id << " tbl_size => "
|
||||
<< table.size() << std::endl;
|
||||
}
|
||||
|
||||
if (t_env::log_data_pool_send()) {
|
||||
std::cout << "t_pool.send" << std::endl;
|
||||
table.pprint();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
std::shared_mutex*
|
||||
t_pool::get_lock() const {
|
||||
return m_lock;
|
||||
}
|
||||
#endif
|
||||
|
||||
void
|
||||
t_pool::_process(std::optional<std::function<void(std::uint32_t)>> callback) {
|
||||
auto work_to_do = m_data_remaining.load();
|
||||
if (work_to_do) {
|
||||
t_update_task task(*this);
|
||||
task.run(callback);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::stop() {
|
||||
m_run.clear(std::memory_order_release);
|
||||
_process();
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.stop" << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::set_sleep(t_uindex ms) {
|
||||
m_sleep.store(ms);
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.set_sleep ms => " << ms << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<t_stree*>
|
||||
t_pool::get_trees() {
|
||||
std::vector<t_stree*> rval;
|
||||
for (auto& g : m_gnodes) {
|
||||
if (g == nullptr) {
|
||||
continue;
|
||||
}
|
||||
auto trees = g->get_trees();
|
||||
rval.insert(std::end(rval), std::begin(trees), std::end(trees));
|
||||
}
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.get_trees: "
|
||||
<< " rv => " << rval << std::endl;
|
||||
}
|
||||
|
||||
return rval;
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::register_context(
|
||||
t_uindex gnode_id,
|
||||
const std::string& name,
|
||||
t_ctx_type type,
|
||||
std::uintptr_t ptr
|
||||
) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_WRITE_LOCK(*m_lock)
|
||||
#endif
|
||||
if (!validate_gnode_id(gnode_id)) {
|
||||
return;
|
||||
}
|
||||
m_gnodes[gnode_id]->_register_context(name, type, ptr);
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::notify_userspace(t_uindex port_id) {
|
||||
// #if defined PSP_ENABLE_WASM && !defined PSP_ENABLE_PYTHON
|
||||
// m_update_delegate.call<void>("_update_callback", port_id);
|
||||
// #elif PSP_ENABLE_PYTHON
|
||||
// if (!m_update_delegate.is_none()) {
|
||||
// m_update_delegate.attr("_update_callback")(port_id);
|
||||
// }
|
||||
// #endif
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::unregister_context(t_uindex gnode_id, const std::string& name) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_WRITE_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << repr() << " << t_pool.unregister_context: "
|
||||
<< " gnode_id => " << gnode_id << " name => " << name
|
||||
<< std::endl;
|
||||
}
|
||||
|
||||
if (!validate_gnode_id(gnode_id)) {
|
||||
return;
|
||||
}
|
||||
m_gnodes[gnode_id]->_unregister_context(name);
|
||||
}
|
||||
|
||||
bool
|
||||
t_pool::get_data_remaining() const {
|
||||
auto data = m_data_remaining.load();
|
||||
return data;
|
||||
}
|
||||
|
||||
std::vector<t_tscalar>
|
||||
t_pool::get_row_data_pkeys(
|
||||
t_uindex gnode_id, const std::vector<t_tscalar>& pkeys
|
||||
) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_READ_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
if (!validate_gnode_id(gnode_id)) {
|
||||
return {};
|
||||
}
|
||||
|
||||
auto rv = m_gnodes[gnode_id]->get_row_data_pkeys(pkeys);
|
||||
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.get_row_data_pkeys: "
|
||||
<< " gnode_id => " << gnode_id << " pkeys => " << pkeys
|
||||
<< " rv => " << rv << std::endl;
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
std::vector<t_updctx>
|
||||
t_pool::get_contexts_last_updated() {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_READ_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
std::vector<t_updctx> rval;
|
||||
|
||||
for (auto& m_gnode : m_gnodes) {
|
||||
if (m_gnode == nullptr) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto updated_contexts = m_gnode->get_contexts_last_updated();
|
||||
auto gnode_id = m_gnode->get_id();
|
||||
|
||||
for (const auto& ctx_name : updated_contexts) {
|
||||
if (t_env::log_progress()) {
|
||||
std::cout << "t_pool.get_contexts_last_updated: "
|
||||
<< " gnode_id => " << gnode_id << " ctx_name => "
|
||||
<< ctx_name << std::endl;
|
||||
}
|
||||
rval.emplace_back(gnode_id, ctx_name);
|
||||
}
|
||||
}
|
||||
return rval;
|
||||
}
|
||||
|
||||
bool
|
||||
t_pool::validate_gnode_id(t_uindex gnode_id) const {
|
||||
return (m_gnodes[gnode_id] != nullptr) && gnode_id < m_gnodes.size();
|
||||
}
|
||||
|
||||
std::string
|
||||
t_pool::repr() const {
|
||||
std::stringstream ss;
|
||||
ss << "t_pool<" << this << ">";
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::pprint_registered() const {
|
||||
auto self = repr();
|
||||
|
||||
for (auto* m_gnode : m_gnodes) {
|
||||
if (m_gnode == nullptr) {
|
||||
continue;
|
||||
}
|
||||
auto gnode_id = m_gnode->get_id();
|
||||
auto ctxnames = m_gnode->get_registered_contexts();
|
||||
|
||||
for (const auto& cname : ctxnames) {
|
||||
std::cout << self << " gnode_id => " << gnode_id << " ctxname => "
|
||||
<< cname << std::endl;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_pool::epoch() const {
|
||||
return m_epoch.load();
|
||||
}
|
||||
|
||||
void
|
||||
t_pool::inc_epoch() {
|
||||
++m_epoch;
|
||||
}
|
||||
|
||||
std::vector<t_uindex>
|
||||
t_pool::get_gnodes_last_updated() {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_READ_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
std::vector<t_uindex> rv;
|
||||
|
||||
for (t_uindex idx = 0, loop_end = m_gnodes.size(); idx < loop_end; ++idx) {
|
||||
if ((m_gnodes[idx] == nullptr) || !m_gnodes[idx]->was_updated()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
rv.push_back(idx);
|
||||
m_gnodes[idx]->clear_updated();
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
|
||||
t_gnode*
|
||||
t_pool::get_gnode(t_uindex idx) {
|
||||
#ifdef PSP_PARALLEL_FOR
|
||||
PSP_READ_LOCK(*m_lock)
|
||||
#endif
|
||||
|
||||
PSP_VERBOSE_ASSERT(
|
||||
idx < m_gnodes.size() && m_gnodes[idx], "Bad gnode encountered"
|
||||
);
|
||||
|
||||
return m_gnodes[idx];
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,113 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/port.h>
|
||||
|
||||
#include <utility>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_port::t_port(t_port_mode mode, t_schema schema) :
|
||||
m_schema(std::move(schema)),
|
||||
m_init(false),
|
||||
m_table(nullptr),
|
||||
m_prevsize(0) {
|
||||
LOG_CONSTRUCTOR("t_port");
|
||||
}
|
||||
|
||||
t_port::~t_port() { LOG_DESTRUCTOR("t_port"); }
|
||||
|
||||
void
|
||||
t_port::init() {
|
||||
m_table = nullptr;
|
||||
m_table = std::make_shared<t_data_table>(
|
||||
"", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_table->init();
|
||||
m_init = true;
|
||||
}
|
||||
|
||||
std::shared_ptr<t_data_table>
|
||||
t_port::get_table() {
|
||||
return m_table;
|
||||
}
|
||||
|
||||
void
|
||||
t_port::set_table(std::shared_ptr<t_data_table> table) {
|
||||
m_table = nullptr;
|
||||
m_table = std::move(table);
|
||||
}
|
||||
|
||||
void
|
||||
t_port::send(const std::shared_ptr<const t_data_table>& table) {
|
||||
m_table->append(*table);
|
||||
}
|
||||
|
||||
void
|
||||
t_port::send(const t_data_table& table) {
|
||||
m_table->append(table);
|
||||
}
|
||||
|
||||
t_schema
|
||||
t_port::get_schema() const {
|
||||
return m_schema;
|
||||
}
|
||||
|
||||
void
|
||||
t_port::release()
|
||||
|
||||
{
|
||||
if (m_table == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
t_uindex size = m_table->size();
|
||||
|
||||
m_table = nullptr;
|
||||
m_table = std::make_shared<t_data_table>(
|
||||
"", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
|
||||
);
|
||||
m_table->init();
|
||||
|
||||
m_prevsize = size;
|
||||
}
|
||||
|
||||
void
|
||||
t_port::release_or_clear()
|
||||
|
||||
{
|
||||
if (m_table == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
t_uindex size = m_table->size();
|
||||
|
||||
if (static_cast<double>(size) < 0.4 * double(m_prevsize)) {
|
||||
m_table->clear();
|
||||
} else {
|
||||
release();
|
||||
}
|
||||
|
||||
m_prevsize = size;
|
||||
}
|
||||
|
||||
void
|
||||
t_port::clear() {
|
||||
if (m_table == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
m_table->clear();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,46 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/process_state.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_process_state::t_process_state() = default;
|
||||
|
||||
void
|
||||
t_process_state::clear_transitional_data_tables() const {
|
||||
m_delta_data_table->clear();
|
||||
m_prev_data_table->clear();
|
||||
m_current_data_table->clear();
|
||||
m_transitions_data_table->clear();
|
||||
m_existed_data_table->clear();
|
||||
};
|
||||
|
||||
void
|
||||
t_process_state::reserve_transitional_data_tables(t_uindex size) const {
|
||||
m_delta_data_table->reserve(size);
|
||||
m_prev_data_table->reserve(size);
|
||||
m_current_data_table->reserve(size);
|
||||
m_transitions_data_table->reserve(size);
|
||||
m_existed_data_table->reserve(size);
|
||||
};
|
||||
|
||||
void
|
||||
t_process_state::set_size_transitional_data_tables(t_uindex size) const {
|
||||
m_delta_data_table->set_size(size);
|
||||
m_prev_data_table->set_size(size);
|
||||
m_current_data_table->set_size(size);
|
||||
m_transitions_data_table->set_size(size);
|
||||
m_existed_data_table->set_size(size);
|
||||
};
|
||||
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,28 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/pyutils.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
#ifdef PSP_ENABLE_PYTHON
|
||||
PerspectiveGILUnlock::PerspectiveGILUnlock() {}
|
||||
// m_thread_state(PyEval_SaveThread()) {}
|
||||
|
||||
PerspectiveGILUnlock::~PerspectiveGILUnlock() {
|
||||
// PyEval_RestoreThread(m_thread_state);
|
||||
}
|
||||
#endif
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,47 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/raii.h>
|
||||
#ifndef WIN32
|
||||
#ifdef PSP_DEBUG
|
||||
#include <sys/mman.h>
|
||||
#endif
|
||||
#endif
|
||||
|
||||
namespace perspective {
|
||||
|
||||
#ifndef PSP_ENABLE_WASM
|
||||
t_file_handle::t_file_handle(t_handle value) : m_value(value) {}
|
||||
|
||||
t_handle
|
||||
t_file_handle::value() const {
|
||||
return m_value;
|
||||
}
|
||||
#endif
|
||||
|
||||
t_mmap_handle::t_mmap_handle(void* value, t_uindex len) :
|
||||
m_value(value),
|
||||
m_len(len) {}
|
||||
|
||||
void*
|
||||
t_mmap_handle::value() {
|
||||
return m_value;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_mmap_handle::len() const {
|
||||
return m_len;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,52 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef __linux__
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <sys/mman.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_file_handle::~t_file_handle() {
|
||||
if (valid()) {
|
||||
t_index rcode = close(m_value);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_file_handle::valid() const {
|
||||
return m_value >= 0;
|
||||
}
|
||||
|
||||
void
|
||||
t_file_handle::release() {
|
||||
m_value = -1;
|
||||
}
|
||||
|
||||
t_mmap_handle::~t_mmap_handle() {
|
||||
if (valid()) {
|
||||
t_index rcode = munmap(m_value, m_len);
|
||||
PSP_VERBOSE_ASSERT(rcode == 0, "munmap failed.");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_mmap_handle::valid() {
|
||||
return m_value != MAP_FAILED;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,52 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef __APPLE__
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/raii.h>
|
||||
#include <sys/mman.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_file_handle::~t_file_handle() {
|
||||
if (valid()) {
|
||||
t_index rcode = close(m_value);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_file_handle::valid() const {
|
||||
return m_value >= 0;
|
||||
}
|
||||
|
||||
void
|
||||
t_file_handle::release() {
|
||||
m_value = -1;
|
||||
}
|
||||
|
||||
t_mmap_handle::~t_mmap_handle() {
|
||||
if (valid()) {
|
||||
t_index rcode = munmap(m_value, m_len);
|
||||
PSP_VERBOSE_ASSERT(rcode, == 0, "munmap failed.");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_mmap_handle::valid() {
|
||||
return m_value != MAP_FAILED;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,51 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#ifdef WIN32
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/raii.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_file_handle::~t_file_handle() {
|
||||
if (valid()) {
|
||||
auto rb = CloseHandle(m_value);
|
||||
PSP_VERBOSE_ASSERT(rb, "Error closing file");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_file_handle::valid() const {
|
||||
return m_value != INVALID_HANDLE_VALUE;
|
||||
}
|
||||
|
||||
void
|
||||
t_file_handle::release() {
|
||||
m_value = INVALID_HANDLE_VALUE;
|
||||
}
|
||||
|
||||
t_mmap_handle::~t_mmap_handle() {
|
||||
if (valid()) {
|
||||
auto rc = UnmapViewOfFile(m_value);
|
||||
PSP_VERBOSE_ASSERT(rc, "Error unmapping view");
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
t_mmap_handle::valid() {
|
||||
return m_value != 0;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,75 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/range.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
t_range::t_range(t_uindex bridx, t_uindex eridx) :
|
||||
m_bridx(bridx),
|
||||
m_eridx(eridx),
|
||||
m_mode(RANGE_ROW) {}
|
||||
|
||||
t_range::t_range(
|
||||
t_uindex bridx, t_uindex eridx, t_uindex bcidx, t_uindex ecidx
|
||||
) :
|
||||
m_bridx(bridx),
|
||||
m_eridx(eridx),
|
||||
m_bcidx(bcidx),
|
||||
m_ecidx(ecidx),
|
||||
m_mode(RANGE_ROW_COLUMN) {}
|
||||
|
||||
t_range::t_range() : m_mode(RANGE_ALL) {}
|
||||
|
||||
t_range::t_range(
|
||||
const std::vector<t_tscalar>& brpath, const std::vector<t_tscalar>& erpath
|
||||
) :
|
||||
m_brpath(brpath),
|
||||
m_erpath(erpath),
|
||||
m_mode(RANGE_ROW_PATH) {}
|
||||
|
||||
t_range::t_range(
|
||||
const std::vector<t_tscalar>& brpath,
|
||||
const std::vector<t_tscalar>& erpath,
|
||||
const std::vector<t_tscalar>& bcpath,
|
||||
const std::vector<t_tscalar>& ecpath
|
||||
) {}
|
||||
|
||||
t_range::t_range(const std::string& expr_name) {}
|
||||
|
||||
t_uindex
|
||||
t_range::bridx() const {
|
||||
return m_bridx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_range::eridx() const {
|
||||
return m_eridx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_range::bcidx() const {
|
||||
return m_bcidx;
|
||||
}
|
||||
|
||||
t_uindex
|
||||
t_range::ecidx() const {
|
||||
return m_ecidx;
|
||||
}
|
||||
|
||||
t_range_mode
|
||||
t_range::get_mode() const {
|
||||
return m_mode;
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,40 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/regex.h>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
RE2*
|
||||
t_regex_mapping::intern(const std::string& pattern) {
|
||||
if (m_regex_map.count(pattern) == 1) {
|
||||
return m_regex_map[pattern].get();
|
||||
}
|
||||
|
||||
std::shared_ptr<RE2> compiled_pattern =
|
||||
std::make_shared<RE2>(pattern, RE2::Quiet);
|
||||
|
||||
if (!compiled_pattern->ok()) {
|
||||
// TODO: provide a better error message when the regex can't compile.
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
m_regex_map[pattern] = compiled_pattern;
|
||||
return m_regex_map[pattern].get();
|
||||
}
|
||||
|
||||
void
|
||||
t_regex_mapping::clear() {
|
||||
m_regex_map.clear();
|
||||
}
|
||||
|
||||
} // end namespace perspective
|
||||
@@ -0,0 +1,176 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/residency.h>
|
||||
#include <perspective/storage.h>
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <vector>
|
||||
|
||||
namespace perspective {
|
||||
|
||||
bool g_residency_active = false;
|
||||
std::uint64_t g_residency_tick = 0;
|
||||
|
||||
t_residency_manager&
|
||||
t_residency_manager::inst() {
|
||||
static t_residency_manager s_inst;
|
||||
return s_inst;
|
||||
}
|
||||
|
||||
void
|
||||
t_residency_manager::register_store(t_lstore* store) {
|
||||
std::lock_guard<std::mutex> lk(m_mutex);
|
||||
m_stores.insert(store);
|
||||
}
|
||||
|
||||
void
|
||||
t_residency_manager::unregister_store(t_lstore* store) {
|
||||
std::lock_guard<std::mutex> lk(m_mutex);
|
||||
m_stores.erase(store);
|
||||
}
|
||||
|
||||
// Hard-coded residency budget (bytes) for WASM. A browser has no environment,
|
||||
// so `PSP_MEMORY_BUDGET` is unreachable there — without a budget, residency is
|
||||
// inert, on-disk columns never evict to OPFS, and the heap can grow past the
|
||||
// 2GB signed-pointer ceiling. This caps resident disk-backed column buffers so
|
||||
// the cold set is flushed to OPFS. Tunable via `-DPSP_WASM_MEMORY_BUDGET=...`.
|
||||
#ifndef PSP_WASM_MEMORY_BUDGET
|
||||
#define PSP_WASM_MEMORY_BUDGET (1024ull * 1024ull * 1024ull) // 1 GiB
|
||||
#endif
|
||||
|
||||
void
|
||||
t_residency_manager::refresh_config() {
|
||||
std::size_t budget = 0;
|
||||
#ifdef PSP_ENABLE_WASM
|
||||
budget = static_cast<std::size_t>(PSP_WASM_MEMORY_BUDGET);
|
||||
#else
|
||||
const char* budget_env = std::getenv("PSP_MEMORY_BUDGET");
|
||||
if (budget_env != nullptr) {
|
||||
budget = static_cast<std::size_t>(std::strtoull(budget_env, nullptr, 10));
|
||||
}
|
||||
#endif
|
||||
|
||||
bool was_active = g_residency_active;
|
||||
m_budget = budget;
|
||||
g_residency_active = budget > 0;
|
||||
|
||||
// If residency was just disabled, restore every evicted store so the lazy
|
||||
// `ensure_resident()` hook (
|
||||
// now a no-op) never sees a null `m_base`.
|
||||
if (was_active && !g_residency_active) {
|
||||
for (auto* store : m_stores) {
|
||||
store->restore();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
std::size_t
|
||||
t_residency_manager::resident_bytes() {
|
||||
std::lock_guard<std::mutex> lk(m_mutex);
|
||||
std::size_t total = 0;
|
||||
for (auto* store : m_stores) {
|
||||
if (store->is_resident()) {
|
||||
total += store->capacity();
|
||||
}
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
std::size_t
|
||||
t_residency_manager::prepare() {
|
||||
refresh_config();
|
||||
std::lock_guard<std::mutex> lk(m_mutex);
|
||||
m_pending.clear();
|
||||
m_pending_fnames.clear();
|
||||
|
||||
if (!g_residency_active) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
g_residency_tick = ++m_tick;
|
||||
std::size_t resident = 0;
|
||||
std::vector<t_lstore*> candidates;
|
||||
candidates.reserve(m_stores.size());
|
||||
for (auto* store : m_stores) {
|
||||
if (store->is_resident()) {
|
||||
resident += store->capacity();
|
||||
candidates.push_back(store);
|
||||
}
|
||||
}
|
||||
|
||||
if (resident <= m_budget) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::sort(
|
||||
candidates.begin(),
|
||||
candidates.end(),
|
||||
[](const t_lstore* a, const t_lstore* b) {
|
||||
return a->residency_tick() < b->residency_tick();
|
||||
}
|
||||
);
|
||||
|
||||
// Select victims to bring under budget, but do NOT evict yet — the JS driver
|
||||
// must open each victim's OPFS handle before `commit()` can flush it.
|
||||
for (auto* store : candidates) {
|
||||
if (resident <= m_budget) {
|
||||
break;
|
||||
}
|
||||
|
||||
resident -= store->capacity();
|
||||
m_pending.push_back(store);
|
||||
m_pending_fnames.push_back(store->get_fname());
|
||||
}
|
||||
|
||||
return m_pending.size();
|
||||
}
|
||||
|
||||
const char*
|
||||
t_residency_manager::victim_fname(std::size_t i) const {
|
||||
if (i >= m_pending_fnames.size()) {
|
||||
return "";
|
||||
}
|
||||
return m_pending_fnames[i].c_str();
|
||||
}
|
||||
|
||||
void
|
||||
t_residency_manager::commit() {
|
||||
std::size_t n = 0;
|
||||
{
|
||||
std::lock_guard<std::mutex> lk(m_mutex);
|
||||
for (auto* store : m_pending) {
|
||||
store->evict();
|
||||
++m_evictions;
|
||||
}
|
||||
|
||||
n = m_pending.size();
|
||||
m_pending.clear();
|
||||
m_pending_fnames.clear();
|
||||
}
|
||||
|
||||
if (n == 0) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
t_residency_manager::safepoint() {
|
||||
std::lock_guard<std::mutex> lk(m_safepoint_mutex);
|
||||
prepare();
|
||||
commit();
|
||||
}
|
||||
|
||||
} // namespace perspective
|
||||
@@ -0,0 +1,29 @@
|
||||
// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
|
||||
// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃
|
||||
// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃
|
||||
// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃
|
||||
// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃
|
||||
// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
|
||||
// ┃ Copyright (c) 2017, the Perspective Authors. ┃
|
||||
// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
|
||||
// ┃ This file is part of the Perspective library, distributed under the terms ┃
|
||||
// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
|
||||
// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
|
||||
|
||||
#include <perspective/first.h>
|
||||
#include <perspective/base.h>
|
||||
#include <perspective/rlookup.h>
|
||||
|
||||
namespace perspective {
|
||||
t_rlookup::t_rlookup(t_uindex idx, bool exists) : m_idx(idx), m_exists(exists) {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_CONSTRUCTOR("t_rlookup");
|
||||
}
|
||||
|
||||
t_rlookup::t_rlookup() = default;
|
||||
|
||||
t_rlookup::~t_rlookup() {
|
||||
PSP_TRACE_SENTINEL();
|
||||
LOG_DESTRUCTOR("t_rlookup");
|
||||
}
|
||||
} // namespace perspective
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user