chore: import upstream snapshot with attribution

rust/perspective-server/cpp/perspective/.clangd.in

@@ -0,0 +1,3 @@
+CompileFlags:
+    @CLANGD_ADD_FLAGS@
+    CompilationDatabase: @CMAKE_BINARY_DIR@

rust/perspective-server/cpp/perspective/CMakeLists.txt

@@ -0,0 +1,736 @@
+cmake_minimum_required(VERSION 3.18.2)
+project(psp)
+include(CheckCCompilerFlag)
+
+set(CMAKE_BUILD_TYPE "Release")
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+set(CMAKE_IGNORE_PATH "/opt/homebrew")
+
+# CMAKE POLICIES
+# option() should use new cmake behavior wrt variable clobbering
+cmake_policy(SET CMP0077 NEW)
+
+# BOOST_ROOT has been removed on Windows VMs in Azure:
+#
+# - https://github.com/actions/virtual-environments/issues/687
+# - https://github.com/actions/virtual-environments/issues/319
+#
+# `BOOST_ROOT` must be set in the environment, and policy CMP0074
+# must be set to `NEW` to allow BOOST_ROOT to be defined by env var
+cmake_policy(SET CMP0074 NEW)
+
+# Set CMP0094 to NEW - find the first version that matches constraints,
+# instead of the latest version installed.
+cmake_policy(SET CMP0094 NEW)
+
+if(NOT DEFINED PSP_CMAKE_MODULE_PATH)
+    set(PSP_CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake")
+endif()
+
+set(CMAKE_MODULE_PATH "${PSP_CMAKE_MODULE_PATH}/modules" ${CMAKE_MODULE_PATH})
+set(MSVC_RUNTIME_LIBRARY MultiThreaded)
+
+option(PSP_WASM64 "Enable WASM Memory64 support" OFF)
+option(PSP_WASM_OPFS "Enable on-disk (OPFS) tables in the browser via the custom `psp_opfs_*` bridge in `perspective-server.poly.ts` (NO_FILESYSTEM + JSPI; no WasmFS, no SharedArrayBuffer). Backwards compatible: the module loads and runs in-memory tables on non-JSPI browsers; on-disk activates only in a JSPI-capable Worker." ON)
+
+if(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    set(WIN32 ON)
+    set(MACOS OFF)
+    set(LINUX OFF)
+elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    set(WIN32 OFF)
+    set(MACOS ON)
+    set(LINUX OFF)
+else()
+    set(WIN32 OFF)
+    set(MACOS OFF)
+    set(LINUX ON)
+endif()
+
+if (WIN32)
+    add_compile_options("/EHsc")
+    set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>")
+endif()
+
+
+if(WIN32)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /MT /c /bigobj")
+else()
+    # set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS}")
+endif()
+
+# # Helper function
+function(string_starts_with str search)
+    string(FIND "${str}" "${search}" out)
+
+    if("${out}" EQUAL 0)
+        return(true)
+    endif()
+    return(false)
+endfunction()
+
+set(BUILD_MESSAGE "")
+
+function(psp_build_message message)
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${message}")
+endfunction()
+
+# ######################
+# BUILD CONFIGURATION #
+# ######################
+find_package(Color)
+find_package(InstallDependency)
+
+# OPTIONS
+option(CMAKE_BUILD_TYPE "Release/Debug build" RELEASE)
+option(PSP_WASM_BUILD "Build the WebAssembly Project" ON)
+option(PSP_CPP_BUILD "Build the C++ Project" OFF)
+option(PSP_PYTHON_BUILD "Build the Python Bindings" OFF)
+option(PSP_CPP_BUILD_STRICT "Build the C++ with strict warnings" OFF)
+option(PSP_SANITIZE "Build with sanitizers" OFF)
+option(PSP_HEAP_INSTRUMENTS "Build with heap inspection tooling" OFF)
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
+    set(PSP_WASM_BUILD ON)
+    set(PSP_CPP_BUILD OFF)
+else()
+    set(PSP_WASM_BUILD OFF)
+    set(PSP_CPP_BUILD ON)
+endif()
+
+if(PSP_WASM_BUILD AND PSP_CPP_BUILD)
+    message(FATAL_ERROR "${Red}CPP and Emscripten builds must be done separately${ColorReset}")
+endif()
+
+if(DEFINED ENV{PSP_DEBUG})
+    set(CMAKE_BUILD_TYPE DEBUG)
+else()
+    if(NOT DEFINED CMAKE_BUILD_TYPE)
+        set(CMAKE_BUILD_TYPE RELEASE)
+    endif()
+endif()
+
+if(DEFINED ENV{PSP_HEAP_INSTRUMENTS})
+    set(PSP_HEAP_INSTRUMENTS ON)
+else()
+    set(PSP_HEAP_INSTRUMENTS OFF)
+endif()
+
+if(DEFINED ENV{PSP_MANYLINUX})
+    set(MANYLINUX ON)
+else()
+    set(MANYLINUX OFF)
+endif()
+
+if(DEFINED ENV{PSP_USE_CCACHE})
+    set(CMAKE_C_COMPILE_LAUNCHER ccache)
+    set(CMAKE_CXX_COMPILER_LAUNCHER ccache)
+endif()
+
+if(NOT DEFINED PSP_CPP_BUILD)
+    set(PSP_CPP_BUILD ON)
+endif()
+
+if(NOT DEFINED PSP_PYTHON_BUILD)
+    set(PSP_PYTHON_BUILD OFF)
+elseif(PSP_PYTHON_BUILD)
+    if(NOT DEFINED PSP_PYTHON_VERSION)
+        set(PSP_PYTHON_VERSION 3.10)
+    endif()
+    if(PSP_WASM_BUILD)
+        set(PSP_PYODIDE 1)
+    else()
+        set(PSP_PYODIDE 0)
+    endif()
+endif()
+
+if(NOT DEFINED PSP_CPP_BUILD_STRICT)
+    set(PSP_CPP_BUILD_STRICT OFF)
+endif()
+
+if(PSP_WASM_BUILD)
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building WASM binding${ColorReset}")
+else()
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping WASM binding${ColorReset}")
+endif()
+
+if(NOT DEFINED PSP_CPP_SRC)
+    set(PSP_CPP_SRC "${CMAKE_CURRENT_SOURCE_DIR}")
+endif()
+
+if(PSP_CPP_BUILD)
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ binding${ColorReset}")
+else()
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping C++ binding${ColorReset}")
+endif()
+
+if(PSP_PYTHON_BUILD)
+    if(NOT DEFINED PSP_PYTHON_SRC)
+        set(PSP_PYTHON_SRC "${CMAKE_CURRENT_SOURCE_DIR}/../../python/perspective/perspective")
+    endif()
+
+    # set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building Python ${Red}${PSP_PYTHON_VERSION}${Cyan} binding${ColorReset}")
+else()
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Skipping Python binding${ColorReset}")
+endif()
+
+if(PSP_CPP_BUILD AND NOT PSP_CPP_BUILD_STRICT)
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Yellow}Building C++ without strict warnings${ColorReset}")
+else()
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building C++ with strict warnings${ColorReset}")
+endif()
+
+string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LOWER)
+
+if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Red}Building DEBUG${ColorReset}")
+    add_definitions(-DPSP_DEBUG)
+    add_definitions(-DPSP_STORAGE_VERIFY)
+    add_definitions(-D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_FAST)
+    add_definitions(-D_GLIBCXX_ASSERTIONS)
+else()
+    set(BUILD_MESSAGE "${BUILD_MESSAGE}\n${Cyan}Building RELEASE${ColorReset}")
+endif()
+
+if(PSP_PYTHON_BUILD AND MACOS)
+    # fix for threads on osx
+    # assume built-in pthreads on MacOS
+    set(CMAKE_THREAD_LIBS_INIT "-lpthread")
+    set(CMAKE_HAVE_THREADS_LIBRARY 1)
+    set(CMAKE_USE_WIN32_THREADS_INIT 0)
+    set(CMAKE_USE_PTHREADS_INIT 1)
+    set(THREADS_PREFER_PTHREAD_FLAG ON)
+
+    # don't link against build python
+    # https://blog.tim-smith.us/2015/09/python-extension-modules-os-x/
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup")
+endif()
+
+if(EMSCRIPTEN)
+    set(CMAKE_HAVE_LIBC_PTHREAD TRUE CACHE BOOL "psp: satisfy FindThreads on WASM without enabling pthreads" FORCE)
+endif()
+
+# ######################
+set(psp_INCLUDE_DIRS "${CMAKE_CURRENT_SOURCE_DIR}/src/include")
+
+if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD)
+    set(PSP_WASM_EXCEPTIONS ON)
+endif()
+
+if(PSP_PYODIDE)
+    set(PSP_WASM_EXCEPTIONS ON)
+endif()
+
+set(DEBUG_LEVEL "0")
+if(PSP_HEAP_INSTRUMENTS)
+    set(DEBUG_LEVEL "3")
+endif()
+
+if(PSP_WASM_BUILD)
+    ####################
+    # EMSCRIPTEN BUILD #
+    ####################
+    set(CMAKE_EXECUTABLE_SUFFIX ".js")
+    list(APPEND CMAKE_PREFIX_PATH /usr/local)
+
+    set(EXTENDED_FLAGS " \
+        -Wall \
+        -fcolor-diagnostics \
+    ")
+
+    if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
+        # Pyodide DEBUG block
+        set(OPT_FLAGS " \
+            -O0 \
+            -g3 \
+            -Wcast-align \
+            -Wover-aligned \
+            --emit-tsd=perspective-server.d.ts \
+            ")
+        if (PSP_WASM_EXCEPTIONS)
+            set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
+        endif()
+        if(PSP_SANITIZE)
+            set(OPT_FLAGS "${OPT_FLAGS} \
+                -fsanitize=undefined \
+                -fsanitize=address \
+            ")
+        endif()
+    else()
+        set(OPT_FLAGS " \
+            -O3 -g${DEBUG_LEVEL} \
+            -mbulk-memory \
+            -msimd128 \
+            -mrelaxed-simd \
+            -flto \
+            --emit-tsd=perspective-server.d.ts \
+        ")
+        if (PSP_WASM_EXCEPTIONS)
+            set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
+        endif()
+    endif()
+elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
+    if(WIN32)
+        if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
+            set(OPT_FLAGS " \
+                /DEBUG \
+                /Z7 \
+                /Zi
+                ")
+        else()
+            set(OPT_FLAGS " \
+                /NDEBUG \
+                /O2 \
+                ")
+        endif()
+    else()
+        if(CMAKE_BUILD_TYPE_LOWER STREQUAL debug)
+            if (PSP_PYODIDE)
+                set(OPT_FLAGS " \
+                    -O0 \
+                    -g3 \
+                    -gsource-map \
+                    --profiling \
+                    -Wcast-align \
+                    -Wover-aligned \
+                    ")
+            else ()
+                set(OPT_FLAGS " \
+                    -O0 \
+                    -fexceptions \
+                    -g3 \
+                    ")
+                if (PSP_WASM_EXCEPTIONS)
+                    set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
+                endif()
+            endif ()
+        else()
+            set(OPT_FLAGS " \
+                -O3 \
+                -fexceptions \
+                -g1 \
+                -fopenmp-simd \
+                ")
+            if (PSP_WASM_EXCEPTIONS)
+                set(OPT_FLAGS "${OPT_FLAGS} -fwasm-exceptions ")
+            endif()
+        endif()
+    endif()
+
+    set(ASYNC_MODE_FLAGS "")
+
+    if(WIN32)
+        foreach(warning 4244 4251 4267 4275 4290 4786 4305 4996)
+            SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd${warning}")
+        endforeach(warning)
+    else()
+    endif()
+
+    if(PSP_PYTHON_BUILD)
+        set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+    endif()
+endif()
+
+
+if (PSP_WASM_EXCEPTIONS)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
+        -O3 \
+        -g${DEBUG_LEVEL} \
+    ")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+        -fwasm-exceptions \
+        -O3 \
+        -g${DEBUG_LEVEL} \
+    ")
+else()
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+    -O3 \
+    ")
+endif()
+
+if (PSP_WASM64)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} \
+        -s MEMORY64=1 \
+    ")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+        -s MEMORY64=1 \
+    ")
+endif()
+
+set(RAPIDJSON_BUILD_TESTS OFF CACHE BOOL "Disable rapidjson tests")
+
+if(PSP_PYODIDE)
+    set(RELOCATABLE_FLAGS " -sRELOCATABLE=1 -sSIDE_MODULE=2 -fwasm-exceptions -sSUPPORT_LONGJMP=wasm \ ")
+    string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS} -sWASM_BIGINT=1 ")
+    string(APPEND CMAKE_C_FLAGS "${RELOCATABLE_FLAGS}")
+    string(APPEND CMAKE_CXX_FLAGS "${RELOCATABLE_FLAGS}")
+    string(APPEND CMAKE_CXX_FLAGS " -fvisibility=hidden")
+
+    # Emscripten exceptions
+    string(APPEND CMAKE_CXX_FLAGS " -fexceptions")
+endif()
+
+# Build (read: download and extract) header-only dependencies from external sources
+set(all_deps_INCLUDE_DIRS "")
+psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in")
+psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in")
+psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in")
+psp_build_dep("rapidjson" "${PSP_CMAKE_MODULE_PATH}/rapidjson.txt.in")
+list(APPEND all_deps_INCLUDE_DIRS
+    ${date_INCLUDE_DIRS}
+    ${hopscotch_INCLUDE_DIRS}
+    ${ordered-map_INCLUDE_DIRS}
+    ${rapidjson_INCLUDE_DIRS})
+
+psp_build_dep("Boost" "${PSP_CMAKE_MODULE_PATH}/Boost.txt.in")
+list(APPEND all_deps_INCLUDE_DIRS ${Boost_INCLUDE_DIRS})
+
+# Build dependencies as static libraries with add_subdirectory()
+# Note that values set above for CMAKE_C_FLAGS, CMAKE_CXX_FLAGS, etc. will
+# apply to dependency cmake builds
+
+# Arrow builds its own dependencies
+psp_build_message("${Cyan}Building Apache Arrow${ColorReset}")
+psp_build_dep("arrow" "${PSP_CMAKE_MODULE_PATH}/arrow.txt.in")
+list(APPEND all_deps_INCLUDE_DIRS
+    ${arrow_INCLUDE_DIRS})
+
+# Build re2 as our regex library
+# this is a workaround for some re2-specific weirdness
+add_definitions(-DTARGET_OS_OSX=1)
+psp_build_dep("re2" "${PSP_CMAKE_MODULE_PATH}/re2.txt.in")
+list(APPEND all_deps_INCLUDE_DIRS
+    ${re2_INCLUDE_DIRS})
+
+# Build exprtk for expression parsing
+psp_build_dep("exprtk" "${PSP_CMAKE_MODULE_PATH}/exprtk.txt.in")
+list(APPEND all_deps_INCLUDE_DIRS
+    ${exprtk_INCLUDE_DIRS})
+
+# Protobuf setup
+set(protobuf_BUILD_TESTS OFF FORCE)
+add_subdirectory(${PSP_CMAKE_MODULE_PATH}/../cpp/protos "${CMAKE_BINARY_DIR}/protos-build")
+
+# ####################
+set(CMAKE_C_FLAGS " \
+    ${CMAKE_C_FLAGS} \
+    ${CMAKE_C_FLAGS_RELEASE} \
+    ${EXTENDED_FLAGS} \
+    ${OPT_FLAGS} \
+    ")
+
+set(CMAKE_CXX_FLAGS " \
+    ${CMAKE_CXX_FLAGS} \
+    ${CMAKE_CXX_FLAGS_RELEASE} \
+    ${EXTENDED_FLAGS} \
+    ${OPT_FLAGS} \
+    ")
+
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c++1y")
+endif()
+
+set(SOURCE_FILES
+    ${PSP_CPP_SRC}/src/cpp/aggregate.cpp
+    ${PSP_CPP_SRC}/src/cpp/aggspec.cpp
+    ${PSP_CPP_SRC}/src/cpp/arg_sort.cpp
+    ${PSP_CPP_SRC}/src/cpp/arrow_loader.cpp
+    ${PSP_CPP_SRC}/src/cpp/arrow_writer.cpp
+    ${PSP_CPP_SRC}/src/cpp/base.cpp
+    ${PSP_CPP_SRC}/src/cpp/base_impl_linux.cpp
+    ${PSP_CPP_SRC}/src/cpp/base_impl_osx.cpp
+    ${PSP_CPP_SRC}/src/cpp/base_impl_wasm.cpp
+    ${PSP_CPP_SRC}/src/cpp/base_impl_win.cpp
+    # ${PSP_CPP_SRC}/src/cpp/binding.cpp
+
+    # ${PSP_CPP_SRC}/src/cpp/build_filter.cpp
+    # ${PSP_CPP_SRC}/src/cpp/calc_agg_dtype.cpp
+    ${PSP_CPP_SRC}/src/cpp/column.cpp
+    ${PSP_CPP_SRC}/src/cpp/comparators.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat_impl_linux.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat_impl_osx.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat_impl_wasm.cpp
+    ${PSP_CPP_SRC}/src/cpp/compat_impl_win.cpp
+    ${PSP_CPP_SRC}/src/cpp/computed_expression.cpp
+    ${PSP_CPP_SRC}/src/cpp/computed_function.cpp
+    ${PSP_CPP_SRC}/src/cpp/config.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_base.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_grouped_pkey.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_handle.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_one.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_two.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_zero.cpp
+    ${PSP_CPP_SRC}/src/cpp/context_unit.cpp
+    ${PSP_CPP_SRC}/src/cpp/data.cpp
+    ${PSP_CPP_SRC}/src/cpp/data_slice.cpp
+    ${PSP_CPP_SRC}/src/cpp/data_table.cpp
+    ${PSP_CPP_SRC}/src/cpp/date.cpp
+    ${PSP_CPP_SRC}/src/cpp/dense_nodes.cpp
+    ${PSP_CPP_SRC}/src/cpp/dense_tree_context.cpp
+    ${PSP_CPP_SRC}/src/cpp/dense_tree.cpp
+    ${PSP_CPP_SRC}/src/cpp/dependency.cpp
+    ${PSP_CPP_SRC}/src/cpp/expression_tables.cpp
+    ${PSP_CPP_SRC}/src/cpp/expression_vocab.cpp
+    ${PSP_CPP_SRC}/src/cpp/extract_aggregate.cpp
+    ${PSP_CPP_SRC}/src/cpp/filter.cpp
+    ${PSP_CPP_SRC}/src/cpp/flat_traversal.cpp
+    ${PSP_CPP_SRC}/src/cpp/get_data_extents.cpp
+    ${PSP_CPP_SRC}/src/cpp/gnode.cpp
+    ${PSP_CPP_SRC}/src/cpp/gnode_state.cpp
+    ${PSP_CPP_SRC}/src/cpp/mask.cpp
+    ${PSP_CPP_SRC}/src/cpp/multi_sort.cpp
+    ${PSP_CPP_SRC}/src/cpp/none.cpp
+    ${PSP_CPP_SRC}/src/cpp/path.cpp
+    ${PSP_CPP_SRC}/src/cpp/pivot.cpp
+    ${PSP_CPP_SRC}/src/cpp/pool.cpp
+    ${PSP_CPP_SRC}/src/cpp/port.cpp
+    ${PSP_CPP_SRC}/src/cpp/process_state.cpp
+    ${PSP_CPP_SRC}/src/cpp/pyutils.cpp
+    ${PSP_CPP_SRC}/src/cpp/raii.cpp
+    ${PSP_CPP_SRC}/src/cpp/raii_impl_linux.cpp
+    ${PSP_CPP_SRC}/src/cpp/raii_impl_osx.cpp
+    ${PSP_CPP_SRC}/src/cpp/raii_impl_win.cpp
+    ${PSP_CPP_SRC}/src/cpp/range.cpp
+    ${PSP_CPP_SRC}/src/cpp/regex.cpp
+    ${PSP_CPP_SRC}/src/cpp/rlookup.cpp
+    ${PSP_CPP_SRC}/src/cpp/scalar.cpp
+    ${PSP_CPP_SRC}/src/cpp/schema_column.cpp
+    ${PSP_CPP_SRC}/src/cpp/schema.cpp
+    ${PSP_CPP_SRC}/src/cpp/slice.cpp
+    ${PSP_CPP_SRC}/src/cpp/sort_specification.cpp
+    ${PSP_CPP_SRC}/src/cpp/sparse_tree.cpp
+    ${PSP_CPP_SRC}/src/cpp/sparse_tree_node.cpp
+    ${PSP_CPP_SRC}/src/cpp/residency.cpp
+    ${PSP_CPP_SRC}/src/cpp/step_delta.cpp
+    ${PSP_CPP_SRC}/src/cpp/storage.cpp
+    ${PSP_CPP_SRC}/src/cpp/storage_impl_linux.cpp
+    ${PSP_CPP_SRC}/src/cpp/storage_impl_osx.cpp
+    ${PSP_CPP_SRC}/src/cpp/storage_impl_wasm.cpp
+    ${PSP_CPP_SRC}/src/cpp/storage_impl_win.cpp
+    ${PSP_CPP_SRC}/src/cpp/sym_table.cpp
+    ${PSP_CPP_SRC}/src/cpp/table.cpp
+    ${PSP_CPP_SRC}/src/cpp/time.cpp
+    ${PSP_CPP_SRC}/src/cpp/traversal.cpp
+    ${PSP_CPP_SRC}/src/cpp/traversal_nodes.cpp
+    ${PSP_CPP_SRC}/src/cpp/tree_context_common.cpp
+    ${PSP_CPP_SRC}/src/cpp/utils.cpp
+    ${PSP_CPP_SRC}/src/cpp/update_task.cpp
+    ${PSP_CPP_SRC}/src/cpp/view.cpp
+    ${PSP_CPP_SRC}/src/cpp/view_config.cpp
+    ${PSP_CPP_SRC}/src/cpp/vocab.cpp
+    ${PSP_CPP_SRC}/src/cpp/arrow_csv.cpp
+    ${PSP_CPP_SRC}/src/cpp/join_engine.cpp
+    ${PSP_CPP_SRC}/src/cpp/server.cpp
+    ${PSP_CPP_SRC}/src/cpp/binding_api.cpp
+)
+
+if(PSP_HEAP_INSTRUMENTS)
+    list(APPEND SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/heap_instruments.cpp)
+    add_compile_definitions(HEAP_INSTRUMENTS=1)
+endif()
+
+set(PYTHON_SOURCE_FILES ${SOURCE_FILES})
+set(WASM_SOURCE_FILES ${SOURCE_FILES})
+
+message("${BUILD_MESSAGE}\n")
+
+if(WIN32)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc /MP /MT /c /bigobj")
+else()
+    # set(CMAKE_CXX_FLAGS " ${CMAKE_CXX_FLAGS}")
+endif()
+
+set(PSP_EXPORTED_FUNCTIONS 
+    _psp_poll
+    _psp_new_server
+    _psp_free
+    _psp_alloc
+    _psp_handle_request
+    _psp_new_session
+    _psp_close_session
+    _psp_delete_server
+    _psp_is_memory64
+    _psp_num_cpus
+    _psp_set_num_cpus
+    _psp_residency_prepare
+    _psp_residency_victim_fname
+    _psp_residency_commit
+)
+
+if(PSP_HEAP_INSTRUMENTS)
+    list(APPEND PSP_EXPORTED_FUNCTIONS
+        _psp_print_used_memory
+        _psp_dump_stack_traces
+        _psp_clear_stack_traces
+    )
+endif()
+
+string(JOIN "," PSP_EXPORTED_FUNCTIONS_JOINED ${PSP_EXPORTED_FUNCTIONS})
+
+# Common flags for WASM/JS build and Pyodide
+if(PSP_PYODIDE)
+    set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
+        --no-entry \
+        -s EXPORTED_FUNCTIONS=${PSP_EXPORTED_FUNCTIONS_JOINED} \
+        -s SIDE_MODULE=2 \
+    ")
+else()
+# -s MEMORY_GROWTH_GEOMETRIC_STEP=1.0 \
+# -s MEMORY_GROWTH_GEOMETRIC_CAP=536870912 \
+
+    set(PSP_FILESYSTEM_FLAGS "-s NO_FILESYSTEM=1 ")
+    set(PSP_CLOSURE_FLAGS "--closure=1 ")
+
+    # On-disk tables in the browser are backed by OPFS via emscripten's WasmFS.
+    if(PSP_WASM_OPFS)
+        add_compile_definitions(PSP_WASM_OPFS_PERSIST=1)
+    endif()
+
+    set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \
+        --no-entry \
+        ${PSP_CLOSURE_FLAGS} \
+        ${PSP_FILESYSTEM_FLAGS} \
+        -s ALLOW_MEMORY_GROWTH=1 \
+        -s MODULARIZE=1 \
+        -s WASM_BIGINT=1 \
+        -s INCOMING_MODULE_JS_API=locateFile,psp_heap_size,psp_stack_trace,HEAPU8,HEAPU32,instantiateWasm \
+        -s TEXTDECODER=2 \
+        -s STANDALONE_WASM=1 \
+        -s DYNAMIC_EXECUTION=0 \
+        -s POLYFILL=0 \
+        -s EXPORT_NAME=\"load_perspective\" \
+        -s ERROR_ON_UNDEFINED_SYMBOLS=0 \
+        -s NODEJS_CATCH_EXIT=0 \
+        -s NODEJS_CATCH_REJECTION=0 \
+        -s USE_ES6_IMPORT_META=1 \
+        -s EXPORT_ES6=1 \
+        -s EXPORTED_RUNTIME_METHODS=HEAPU8 \
+        -s EXPORTED_FUNCTIONS=${PSP_EXPORTED_FUNCTIONS_JOINED} \
+    ")
+
+    if(PSP_WASM64)
+        set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
+            -s MAXIMUM_MEMORY=16gb \
+        ")
+    else()
+        set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} \
+            -s MAXIMUM_MEMORY=4gb \
+        ")
+    endif()
+endif()
+
+if (PSP_WASM_EXCEPTIONS)
+    set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} -s EXCEPTION_STACK_TRACES=1 ")
+endif()
+
+if(PSP_SANITIZE)
+    set(PSP_SANITIZE_FLAGS
+        -sINITIAL_MEMORY=640mb
+        -sTOTAL_MEMORY=640mb
+        -sALLOW_MEMORY_GROWTH=1
+    )
+else()
+    set(PSP_SANITIZE_FLAGS)
+endif()
+
+if(PSP_WASM_BUILD AND NOT PSP_PYTHON_BUILD)
+    set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
+
+    add_library(psp ${WASM_SOURCE_FILES})
+    target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
+    target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
+    target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1)
+    set_target_properties(psp PROPERTIES COMPILE_FLAGS "")
+    target_link_libraries(psp PRIVATE arrow_static re2 protos)
+
+    add_executable(perspective_esm src/cpp/binding_api.cpp)
+    message(STATUS "all_deps_INCLUDE_DIRS ${all_deps_INCLUDE_DIRS}")
+    target_include_directories(perspective_esm PRIVATE ${psp_INCLUDE_DIRS})
+    target_include_directories(perspective_esm SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
+    target_link_libraries(perspective_esm psp protos)
+    target_compile_definitions(perspective_esm PRIVATE PSP_ENABLE_WASM=1)
+    target_link_options(perspective_esm PUBLIC -sENVIRONMENT=web  ${PSP_SANITIZE_FLAGS})
+    set_target_properties(perspective_esm PROPERTIES RUNTIME_OUTPUT_DIRECTORY "./web/")
+    set_target_properties(perspective_esm PROPERTIES OUTPUT_NAME "perspective-server")
+elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
+    if(NOT WIN32)
+        set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
+
+        # Look for the binary using @loader_path (relative to binary location)
+        set(CMAKE_MACOSX_RPATH TRUE)
+        set(CMAKE_SKIP_BUILD_RPATH FALSE)
+        set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+        set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+        set(CMAKE_INSTALL_NAME_DIR "@rpath/")
+
+        # module_origin_path is the location of the binary
+        if(MACOS)
+            set(module_origin_path "@loader_path/")
+        else()
+            set(module_origin_path "\$ORIGIN")
+        endif()
+    else()
+        set(CMAKE_SHARED_LIBRARY_PREFIX lib)
+    endif()
+
+    if(PSP_PYTHON_BUILD)
+        # #######################
+        # Python extra targets #
+        # #######################
+        if(PSP_WASM_BUILD)
+            # Pyodide
+            set(CMAKE_EXECUTABLE_SUFFIX ".wasm")
+            set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ")
+            add_library(psp STATIC ${PYTHON_SOURCE_FILES})
+            target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
+            target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
+            target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1)
+        else()
+            # Cpython
+            add_library(psp STATIC ${PYTHON_SOURCE_FILES})
+            target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
+            target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
+            target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1)
+        endif()
+
+        if(MACOS OR NOT MANYLINUX)
+            set_property(TARGET psp PROPERTY INSTALL_RPATH ${CMAKE_INSTALL_RPATH} ${module_origin_path})
+
+            target_compile_options(psp PRIVATE -fvisibility=hidden)
+        elseif(MANYLINUX)
+            # intentionally blank
+        else()
+            target_compile_options(psp PRIVATE -fvisibility=hidden)
+        endif()
+
+        # Linking against arrow_static also links against its bundled dependencies
+        target_link_libraries(psp PRIVATE arrow_static re2 protos)
+    else()
+        add_library(psp STATIC ${WASM_SOURCE_FILES})
+        target_include_directories(psp PRIVATE ${psp_INCLUDE_DIRS})
+        target_include_directories(psp SYSTEM PRIVATE ${all_deps_INCLUDE_DIRS})
+        target_compile_options(psp PRIVATE -fvisibility=hidden)
+        target_link_libraries(psp PRIVATE arrow_static re2 protos)
+    endif()
+
+    if(PSP_CPP_BUILD_STRICT AND NOT WIN32)
+        target_compile_options(psp PRIVATE -Wall -Werror)
+        target_compile_options(psp PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
+        if(PSP_PYTHON_BUILD)
+            target_compile_options(psppy PRIVATE $<$<CONFIG:DEBUG>:-fPIC -O0>)
+        endif()
+    elseif(WIN32)
+        target_compile_definitions(psp PRIVATE PERSPECTIVE_EXPORTS=1)
+        target_compile_definitions(psp PRIVATE WIN32=1)
+        target_compile_definitions(psp PRIVATE _WIN32=1)
+    endif()
+endif()
+
+# TODO this needs to be omitted when built by `cmake-rs`, because
+#`cargo publish --dry-run` complains when this generate `.clangd` in the Rust
+# root dir instead of teh `target` dir.
+if(NOT DEFINED ENV{PSP_DISABLE_CLANGD})
+    include(SetupClangd)
+endif()

rust/perspective-server/cpp/perspective/env.js

@@ -0,0 +1,15 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+// this line/hack allows us to bypass the emscripten environment
+// so we can run the same JS-file on both web and node.
+globalThis.window = {};

rust/perspective-server/cpp/perspective/patches/arrow_strptime.patch

@@ -0,0 +1,53 @@
+diff --git a/cpp/src/arrow/util/decimal.cc b/cpp/src/arrow/util/decimal.cc
+index c8457eae8..aa2b41cc9 100644
+--- a/cpp/src/arrow/util/decimal.cc
++++ b/cpp/src/arrow/util/decimal.cc
+@@ -111,7 +111,11 @@ struct DecimalRealConversion : public BaseDecimalRealConversion {
+ 
+     // 2. Losslessly convert `real` to `mant * 2**k`
+     int binary_exp = 0;
++#ifdef __EMSCRIPTEN__
++    const Real real_mant = std::frexpf(real, &binary_exp);
++#else
+     const Real real_mant = std::frexp(real, &binary_exp);
++#endif
+     // `real_mant` is within 0.5 and 1 and has M bits of precision.
+     // Multiply it by 2^M to get an exact integer.
+     const uint64_t mant = static_cast<uint64_t>(std::ldexp(real_mant, kMantissaBits));
+diff --git a/cpp/src/arrow/util/value_parsing.h b/cpp/src/arrow/util/value_parsing.h
+index 609906052..1e3dfae7c 100644
+--- a/cpp/src/arrow/util/value_parsing.h
++++ b/cpp/src/arrow/util/value_parsing.h
+@@ -804,7 +804,7 @@ static inline bool ParseTimestampStrptime(const char* buf, size_t length,
+   std::string clean_copy(buf, length);
+   struct tm result;
+   memset(&result, 0, sizeof(struct tm));
+-#ifdef _WIN32
++#if defined(_WIN32) || defined(__EMSCRIPTEN__)
+   char* ret = arrow_strptime(clean_copy.c_str(), format, &result);
+ #else
+   char* ret = strptime(clean_copy.c_str(), format, &result);
+diff --git a/cpp/src/arrow/vendored/xxhash/xxhash.h b/cpp/src/arrow/vendored/xxhash/xxhash.h
+index a18e8c762..235590b19 100644
+--- a/cpp/src/arrow/vendored/xxhash/xxhash.h
++++ b/cpp/src/arrow/vendored/xxhash/xxhash.h
+@@ -169,6 +169,11 @@
+  * xxHash prototypes and implementation
+  */
+ 
++
++#ifdef __EMSCRIPTEN__
++#include <emscripten.h>
++#endif
++
+ #if defined (__cplusplus)
+ extern "C" {
+ #endif
+@@ -3422,6 +3427,7 @@ XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_can
+ #  endif
+ #endif
+ 
++
+ #if defined(__GNUC__) || defined(__clang__)
+ #  if defined(__ARM_FEATURE_SVE)
+ #    include <arm_sve.h>

rust/perspective-server/cpp/perspective/patches/fix_arrow_libtool.patch

@@ -0,0 +1,59 @@
+From f2b12a311d1a14ab5d9f67428201e1389748a9c1 Mon Sep 17 00:00:00 2001
+From: Tom Jakubowski <tom@prospective.dev>
+Date: Fri, 11 Oct 2024 12:23:29 -0700
+Subject: [PATCH] GH-44384: [C++] Use CMAKE_LIBTOOL on macOS
+
+When a builder sets `CMAKE_LIBTOOL`, use that as the program to bundle
+dependencies.  This matches the behavior of the Windows build.
+
+Also make a nitpicky minor update to the error message when a non-Apple
+libtool is detected.
+---
+ cpp/cmake_modules/BuildUtils.cmake | 30 +++++++++++++++++-------------
+ 1 file changed, 17 insertions(+), 13 deletions(-)
+
+diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
+index 692efa78376f4..eb94563afcf77 100644
+--- a/cpp/cmake_modules/BuildUtils.cmake
++++ b/cpp/cmake_modules/BuildUtils.cmake
+@@ -97,23 +97,27 @@ function(arrow_create_merged_static_lib output_target)
+   endforeach()
+ 
+   if(APPLE)
+-    # The apple-distributed libtool is what we want for bundling, but there is
+-    # a GNU libtool that has a namecollision (and happens to be bundled with R, too).
+-    # We are not compatible with GNU libtool, so we need to avoid it.
+-
+-    # check in the obvious places first to find Apple's libtool
+-    # HINTS is used before system paths and before PATHS, so we use that
+-    # even though hard coded paths should go in PATHS
+-    # TODO: use a VALIDATOR when we require cmake >= 3.25
+-    find_program(LIBTOOL_MACOS libtool HINTS /usr/bin
+-                                             /Library/Developer/CommandLineTools/usr/bin)
+-
+-    # confirm that the libtool we found is not GNU libtool
++    if(CMAKE_LIBTOOL)
++      set(LIBTOOL_MACOS ${CMAKE_LIBTOOL})
++    else()
++      # The apple-distributed libtool is what we want for bundling, but there is
++      # a GNU libtool that has a namecollision (and happens to be bundled with R, too).
++      # We are not compatible with GNU libtool, so we need to avoid it.
++
++      # check in the obvious places first to find Apple's libtool
++      # HINTS is used before system paths and before PATHS, so we use that
++      # even though hard coded paths should go in PATHS
++      # TODO: use a VALIDATOR when we require cmake >= 3.25
++      find_program(LIBTOOL_MACOS libtool
++                   HINTS /usr/bin /Library/Developer/CommandLineTools/usr/bin)
++    endif()
++
++    # confirm that the libtool we found is Apple's libtool
+     execute_process(COMMAND ${LIBTOOL_MACOS} -V
+                     OUTPUT_VARIABLE LIBTOOL_V_OUTPUT
+                     OUTPUT_STRIP_TRAILING_WHITESPACE)
+     if(NOT "${LIBTOOL_V_OUTPUT}" MATCHES ".*cctools-([0-9.]+).*")
+-      message(FATAL_ERROR "libtool found appears to be the incompatible GNU libtool: ${LIBTOOL_MACOS}"
++      message(FATAL_ERROR "libtool found appears not to be Apple's libtool: ${LIBTOOL_MACOS}"
+       )
+     endif()
+ 

rust/perspective-server/cpp/perspective/src/cpp/aggregate.cpp

@@ -0,0 +1,465 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/aggregate.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_aggregate::t_aggregate(
+    const t_dtree& tree,
+    t_aggtype aggtype,
+    std::vector<std::shared_ptr<const t_column>> icolumns,
+    std::shared_ptr<t_column> ocolumn
+) :
+    m_tree(tree),
+    m_aggtype(aggtype),
+    m_icolumns(std::move(icolumns)),
+    m_ocolumn(std::move(ocolumn)) {}
+
+void
+t_aggregate::init() {
+    switch (m_aggtype) {
+        case AGGTYPE_SUM:
+        case AGGTYPE_PCT_SUM_PARENT:
+        case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::int64_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::int32_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::int16_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<
+                        t_aggimpl_sum<std::int8_t, std::int64_t, std::int64_t>>(
+                    );
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::uint64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::uint32_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::uint16_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<t_aggimpl_sum<double, double, double>>();
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<t_aggimpl_sum<float, double, double>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_sum<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        case AGGTYPE_MUL: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::int64_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::int32_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::int16_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<
+                        t_aggimpl_mul<std::int8_t, std::int64_t, std::int64_t>>(
+                    );
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::uint64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::uint32_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::uint16_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<t_aggimpl_mul<double, double, double>>();
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<t_aggimpl_mul<float, double, double>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_mul<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        case AGGTYPE_COUNT: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_STR: {
+                    build_aggregate<
+                        t_aggimpl_count<t_uindex, std::uint64_t, std::uint64_t>>(
+                    );
+                } break;
+                case DTYPE_TIME:
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_count<
+                        std::int64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_count<
+                        std::int32_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_count<
+                        std::int16_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<t_aggimpl_count<
+                        std::int8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_count<
+                        std::uint64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_DATE:
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_count<
+                        std::uint32_t,
+                        std::uint32_t,
+                        std::uint32_t>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_count<
+                        std::uint16_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_count<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<
+                        t_aggimpl_count<double, std::uint64_t, std::uint64_t>>(
+                    );
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<
+                        t_aggimpl_count<float, std::uint64_t, std::uint64_t>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_count<
+                        std::uint8_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        case AGGTYPE_MEAN: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::int64_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::int32_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::int16_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::int8_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::uint64_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::uint32_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::uint16_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::uint8_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<t_aggimpl_mean<
+                        double,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<t_aggimpl_mean<
+                        float,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_mean<
+                        std::uint8_t,
+                        std::pair<double, double>,
+                        double>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        case AGGTYPE_HIGH_WATER_MARK: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_TIME:
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::int64_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::int32_t,
+                        std::int32_t,
+                        std::int32_t>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::int16_t,
+                        std::int16_t,
+                        std::int16_t>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<
+                        t_aggimpl_hwm<std::int8_t, std::int8_t, std::int8_t>>();
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::uint64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_DATE:
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::uint32_t,
+                        std::uint32_t,
+                        std::uint32_t>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::uint16_t,
+                        std::uint16_t,
+                        std::uint16_t>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::uint8_t,
+                        std::uint8_t,
+                        std::uint8_t>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<t_aggimpl_hwm<double, double, double>>();
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<t_aggimpl_hwm<float, float, float>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_hwm<
+                        std::uint8_t,
+                        std::uint8_t,
+                        std::uint8_t>>();
+                } break;
+                case DTYPE_STR: {
+                    build_aggregate<
+                        t_aggimpl_hwm<const char*, const char*, const char*>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        case AGGTYPE_LOW_WATER_MARK: {
+            switch (m_icolumns[0]->get_dtype()) {
+                case DTYPE_TIME:
+                case DTYPE_INT64: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::int64_t,
+                        std::int64_t,
+                        std::int64_t>>();
+                } break;
+                case DTYPE_INT32: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::int32_t,
+                        std::int32_t,
+                        std::int32_t>>();
+                } break;
+                case DTYPE_INT16: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::int16_t,
+                        std::int16_t,
+                        std::int16_t>>();
+                } break;
+                case DTYPE_INT8: {
+                    build_aggregate<
+                        t_aggimpl_lwm<std::int8_t, std::int8_t, std::int8_t>>();
+                } break;
+                case DTYPE_UINT64: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::uint64_t,
+                        std::uint64_t,
+                        std::uint64_t>>();
+                } break;
+                case DTYPE_DATE:
+                case DTYPE_UINT32: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::uint32_t,
+                        std::uint32_t,
+                        std::uint32_t>>();
+                } break;
+                case DTYPE_UINT16: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::uint16_t,
+                        std::uint16_t,
+                        std::uint16_t>>();
+                } break;
+                case DTYPE_UINT8: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::uint8_t,
+                        std::uint8_t,
+                        std::uint8_t>>();
+                } break;
+                case DTYPE_FLOAT64: {
+                    build_aggregate<t_aggimpl_lwm<double, double, double>>();
+                } break;
+                case DTYPE_FLOAT32: {
+                    build_aggregate<t_aggimpl_lwm<float, float, float>>();
+                } break;
+                case DTYPE_BOOL: {
+                    build_aggregate<t_aggimpl_lwm<
+                        std::uint8_t,
+                        std::uint8_t,
+                        std::uint8_t>>();
+                } break;
+                case DTYPE_STR: {
+                    build_aggregate<
+                        t_aggimpl_lwm<const char*, const char*, const char*>>();
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Unexpected dtype");
+                }
+            }
+        } break;
+        default: {
+            // Other aggregates will be filled in later
+        }
+    }
+}
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/aggspec.cpp

@@ -0,0 +1,468 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/aggspec.h>
+#include <perspective/base.h>
+#include <sstream>
+#include <utility>
+
+namespace perspective {
+
+t_col_name_type::t_col_name_type() : m_type(DTYPE_NONE) {}
+
+t_col_name_type::t_col_name_type(std::string name, t_dtype type) :
+    m_name(std::move(name)),
+    m_type(type) {}
+
+t_aggspec::t_aggspec() = default;
+
+t_aggspec::t_aggspec(
+    const std::string& name,
+    t_aggtype agg,
+    const std::vector<t_dep>& dependencies
+) :
+    m_name(name),
+    m_disp_name(name),
+    m_agg(agg),
+    m_dependencies(dependencies) {}
+
+t_aggspec::t_aggspec(
+    const std::string& aggname, t_aggtype agg, const std::string& dep
+) :
+    m_name(aggname),
+    m_disp_name(aggname),
+    m_agg(agg),
+    m_dependencies(std::vector<t_dep>{t_dep(dep, DEPTYPE_COLUMN)}) {}
+
+t_aggspec::t_aggspec(t_aggtype agg, const std::string& dep) :
+    m_agg(agg),
+    m_dependencies(std::vector<t_dep>{t_dep(dep, DEPTYPE_COLUMN)}) {}
+
+t_aggspec::t_aggspec(
+    std::string name,
+    std::string disp_name,
+    t_aggtype agg,
+    const std::vector<t_dep>& dependencies
+) :
+    m_name(std::move(name)),
+    m_disp_name(std::move(disp_name)),
+    m_agg(agg),
+    m_dependencies(dependencies) {}
+
+t_aggspec::t_aggspec(
+    std::string name,
+    std::string disp_name,
+    t_aggtype agg,
+    const std::vector<t_dep>& dependencies,
+    t_sorttype sort_type
+) :
+    m_name(std::move(name)),
+    m_disp_name(std::move(disp_name)),
+    m_agg(agg),
+    m_dependencies(dependencies),
+    m_sort_type(sort_type) {}
+
+t_aggspec::t_aggspec(
+    std::string aggname,
+    std::string disp_aggname,
+    t_aggtype agg,
+    t_uindex agg_one_idx,
+    t_uindex agg_two_idx,
+    double agg_one_weight,
+    double agg_two_weight
+) :
+    m_name(std::move(aggname)),
+    m_disp_name(std::move(disp_aggname)),
+    m_agg(agg),
+    m_agg_one_idx(agg_one_idx),
+    m_agg_two_idx(agg_two_idx),
+    m_agg_one_weight(agg_one_weight),
+    m_agg_two_weight(agg_two_weight) {}
+
+t_aggspec::~t_aggspec() = default;
+
+std::string
+t_aggspec::name() const {
+    return m_name;
+}
+
+t_tscalar
+t_aggspec::name_scalar() const {
+    t_tscalar s;
+    s.set(m_name.c_str());
+    return s;
+}
+
+std::string
+t_aggspec::disp_name() const {
+    return m_disp_name;
+}
+
+t_aggtype
+t_aggspec::agg() const {
+    return m_agg;
+}
+
+std::string
+t_aggspec::agg_str() const {
+    switch (m_agg) {
+        case AGGTYPE_SUM: {
+            return "sum";
+        } break;
+        case AGGTYPE_SUM_ABS: {
+            return "sum abs";
+        } break;
+        case AGGTYPE_ABS_SUM: {
+            return "abs sum";
+        } break;
+        case AGGTYPE_MUL: {
+            return "mul";
+        } break;
+        case AGGTYPE_COUNT: {
+            return "count";
+        } break;
+        case AGGTYPE_MEAN: {
+            return "mean";
+        } break;
+        case AGGTYPE_WEIGHTED_MEAN: {
+            return "weighted mean";
+        } break;
+        case AGGTYPE_UNIQUE: {
+            return "unique";
+        } break;
+        case AGGTYPE_ANY: {
+            return "any";
+        } break;
+        case AGGTYPE_Q1: {
+            return "q1";
+        } break;
+        case AGGTYPE_Q3: {
+            return "q3";
+        } break;
+        case AGGTYPE_MEDIAN: {
+            return "median";
+        } break;
+        case AGGTYPE_JOIN: {
+            return "join";
+        } break;
+        case AGGTYPE_SCALED_DIV: {
+            return "scaled div";
+        } break;
+        case AGGTYPE_SCALED_ADD: {
+            return "scaled add";
+        } break;
+        case AGGTYPE_SCALED_MUL: {
+            return "scaled mul";
+        } break;
+        case AGGTYPE_DOMINANT: {
+            return "dominant";
+        } break;
+        case AGGTYPE_FIRST: {
+            return "first";
+        } break;
+        case AGGTYPE_LAST_BY_INDEX: {
+            return "last by index";
+        } break;
+        case AGGTYPE_LAST_MINUS_FIRST: {
+            return "last minus first";
+        } break;
+        case AGGTYPE_PY_AGG: {
+            return "py agg";
+        } break;
+        case AGGTYPE_AND: {
+            return "and";
+        } break;
+        case AGGTYPE_OR: {
+            return "or";
+        } break;
+        case AGGTYPE_LAST_VALUE: {
+            return "last value";
+        }
+        case AGGTYPE_MAX_BY: {
+            return "max by";
+        }
+        case AGGTYPE_MIN_BY: {
+            return "min by";
+        }
+        case AGGTYPE_MAX: {
+            return "max";
+        }
+        case AGGTYPE_MIN: {
+            return "min";
+        }
+        case AGGTYPE_HIGH_WATER_MARK: {
+            return "high water mark";
+        }
+        case AGGTYPE_LOW_WATER_MARK: {
+            return "low water mark";
+        }
+        case AGGTYPE_HIGH_MINUS_LOW: {
+            return "high minuslow";
+        } break;
+        case AGGTYPE_UDF_COMBINER: {
+            std::stringstream ss;
+            ss << "udf_combiner_" << disp_name();
+            return ss.str();
+        }
+        case AGGTYPE_UDF_REDUCER: {
+
+            std::stringstream ss;
+            ss << "udf_reducer_" << disp_name();
+            return ss.str();
+        }
+        case AGGTYPE_SUM_NOT_NULL: {
+            return "sum not null";
+        }
+        case AGGTYPE_MEAN_BY_COUNT: {
+            return "mean by count";
+        }
+        case AGGTYPE_IDENTITY: {
+            return "identity";
+        }
+        case AGGTYPE_DISTINCT_COUNT: {
+            return "distinct count";
+        }
+        case AGGTYPE_DISTINCT_LEAF: {
+            return "distinct leaf";
+        }
+        case AGGTYPE_PCT_SUM_PARENT: {
+            return "pct sum parent";
+        }
+        case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
+            return "pct sum grand total";
+        }
+        case AGGTYPE_VARIANCE: {
+            return "variance";
+        }
+        case AGGTYPE_STANDARD_DEVIATION: {
+            return "stddev";
+        }
+        case AGGTYPE_GMV: {
+            return "gmv";
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unknown agg type");
+            return "unknown";
+        } break;
+    }
+}
+
+const std::vector<t_dep>&
+t_aggspec::get_dependencies() const {
+    return m_dependencies;
+}
+
+t_dtype
+get_simple_accumulator_type(t_dtype coltype) {
+    switch (coltype) {
+        case DTYPE_BOOL:
+        case DTYPE_INT64:
+        case DTYPE_INT32:
+        case DTYPE_INT16:
+        case DTYPE_INT8: {
+            return DTYPE_INT64;
+        } break;
+        case DTYPE_UINT64:
+        case DTYPE_UINT32:
+        case DTYPE_UINT16:
+        case DTYPE_UINT8: {
+            return DTYPE_UINT64;
+        }
+        case DTYPE_FLOAT64:
+        case DTYPE_FLOAT32: {
+            return DTYPE_FLOAT64;
+        }
+
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unexpected coltype");
+        }
+    }
+    return DTYPE_NONE;
+}
+
+t_sorttype
+t_aggspec::get_sort_type() const {
+    return m_sort_type;
+}
+
+t_uindex
+t_aggspec::get_agg_one_idx() const {
+    return m_agg_one_idx;
+}
+
+t_uindex
+t_aggspec::get_agg_two_idx() const {
+    return m_agg_two_idx;
+}
+
+double
+t_aggspec::get_agg_one_weight() const {
+    return m_agg_one_weight;
+}
+
+double
+t_aggspec::get_agg_two_weight() const {
+    return m_agg_two_weight;
+}
+
+t_invmode
+t_aggspec::get_inv_mode() const {
+    return m_invmode;
+}
+
+std::vector<std::string>
+t_aggspec::get_input_depnames() const {
+    std::vector<std::string> rval;
+    rval.reserve(m_dependencies.size());
+    for (const auto& d : m_dependencies) {
+        rval.push_back(d.name());
+    }
+    return rval;
+}
+
+std::vector<std::string>
+t_aggspec::get_output_depnames() const {
+    std::vector<std::string> rval;
+    rval.reserve(m_dependencies.size());
+    for (const auto& d : m_dependencies) {
+        rval.push_back(d.name());
+    }
+    return rval;
+}
+
+std::vector<t_col_name_type>
+t_aggspec::get_output_specs(const t_schema& schema) const {
+    switch (agg()) {
+        case AGGTYPE_SUM:
+        case AGGTYPE_SUM_ABS:
+        case AGGTYPE_ABS_SUM:
+        case AGGTYPE_GMV:
+        case AGGTYPE_PCT_SUM_PARENT:
+        case AGGTYPE_PCT_SUM_GRAND_TOTAL:
+        case AGGTYPE_MUL:
+        case AGGTYPE_SUM_NOT_NULL: {
+            t_dtype coltype = schema.get_dtype(m_dependencies[0].name());
+            return mk_col_name_type_vec(
+                name(), get_simple_accumulator_type(coltype)
+            );
+        }
+        case AGGTYPE_ANY:
+        case AGGTYPE_UNIQUE:
+        case AGGTYPE_DOMINANT:
+        case AGGTYPE_Q1:
+        case AGGTYPE_Q3:
+        case AGGTYPE_MEDIAN:
+        case AGGTYPE_FIRST:
+        case AGGTYPE_LAST_BY_INDEX:
+        case AGGTYPE_LAST_MINUS_FIRST:
+        case AGGTYPE_OR:
+        case AGGTYPE_LAST_VALUE:
+        case AGGTYPE_MAX:
+        case AGGTYPE_MAX_BY:
+        case AGGTYPE_MIN_BY:
+        case AGGTYPE_MIN:
+        case AGGTYPE_HIGH_WATER_MARK:
+        case AGGTYPE_LOW_WATER_MARK:
+        case AGGTYPE_HIGH_MINUS_LOW:
+        case AGGTYPE_IDENTITY:
+        case AGGTYPE_DISTINCT_LEAF: {
+            t_dtype coltype = schema.get_dtype(m_dependencies[0].name());
+            std::vector<t_col_name_type> rval(1);
+            rval[0].m_name = name();
+            rval[0].m_type = coltype;
+            return rval;
+        }
+        case AGGTYPE_COUNT: {
+            return mk_col_name_type_vec(name(), DTYPE_INT64);
+        }
+        case AGGTYPE_MEAN:
+        case AGGTYPE_MEAN_BY_COUNT:
+        case AGGTYPE_WEIGHTED_MEAN: {
+            return mk_col_name_type_vec(name(), DTYPE_F64PAIR);
+        }
+        case AGGTYPE_JOIN: {
+            return mk_col_name_type_vec(name(), DTYPE_STR);
+        }
+        case AGGTYPE_SCALED_DIV:
+        case AGGTYPE_SCALED_ADD:
+        case AGGTYPE_SCALED_MUL:
+        case AGGTYPE_VARIANCE:
+        case AGGTYPE_STANDARD_DEVIATION: {
+            return mk_col_name_type_vec(name(), DTYPE_FLOAT64);
+        }
+        case AGGTYPE_UDF_COMBINER:
+        case AGGTYPE_UDF_REDUCER: {
+            std::vector<t_col_name_type> rval;
+            rval.reserve(m_odependencies.size());
+            for (const auto& d : m_odependencies) {
+                t_col_name_type tp(d.name(), d.dtype());
+                rval.push_back(tp);
+            }
+            return rval;
+        }
+        case AGGTYPE_AND: {
+            return mk_col_name_type_vec(name(), DTYPE_BOOL);
+        }
+        case AGGTYPE_DISTINCT_COUNT: {
+            return mk_col_name_type_vec(name(), DTYPE_UINT32);
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unknown agg type");
+        }
+    }
+
+    return {};
+}
+
+std::vector<t_col_name_type>
+t_aggspec::mk_col_name_type_vec(const std::string& name, t_dtype dtype) const {
+    std::vector<t_col_name_type> rval(1);
+    rval[0].m_name = name;
+    rval[0].m_type = dtype;
+    return rval;
+}
+
+bool
+t_aggspec::is_combiner_agg() const {
+    return m_agg == AGGTYPE_UDF_COMBINER;
+}
+
+bool
+t_aggspec::is_reducer_agg() const {
+    return m_agg == AGGTYPE_UDF_REDUCER;
+}
+
+bool
+t_aggspec::is_non_delta() const {
+    switch (m_agg) {
+        case AGGTYPE_LAST_VALUE:
+        case AGGTYPE_LOW_WATER_MARK:
+        case AGGTYPE_HIGH_WATER_MARK: {
+            return true;
+        }
+        default:
+            return false;
+    }
+    return false;
+}
+
+std::string
+t_aggspec::get_first_depname() const {
+    if (m_dependencies.empty()) {
+        return "";
+    }
+
+    return m_dependencies[0].name();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/arg_sort.cpp

@@ -0,0 +1,113 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <functional>
+#include <perspective/arg_sort.h>
+#include <perspective/multi_sort.h>
+#include <perspective/scalar.h>
+
+namespace perspective {
+
+void
+argsort(std::vector<t_index>& output, const t_multisorter& sorter) {
+    if (output.empty()) {
+        return;
+    }
+    // Output should be the same size is v
+    for (t_index i = 0, loop_end = output.size(); i != loop_end; ++i) {
+        output[i] = i;
+    }
+    std::sort(output.begin(), output.end(), sorter);
+}
+
+t_argsort_comparator::t_argsort_comparator(
+    const std::vector<t_tscalar>& v, const t_sorttype& sort_type
+) :
+    m_v(v),
+    m_sort_type(sort_type) {}
+
+bool
+t_argsort_comparator::operator()(t_index a, t_index b) const {
+    switch (m_sort_type) {
+        case SORTTYPE_ASCENDING:
+            return t_argsort_comparator_impl<SORTTYPE_ASCENDING>(m_v)(a, b);
+        case SORTTYPE_DESCENDING:
+            return t_argsort_comparator_impl<SORTTYPE_DESCENDING>(m_v)(a, b);
+        case SORTTYPE_ASCENDING_ABS:
+            return t_argsort_comparator_impl<SORTTYPE_ASCENDING_ABS>(m_v)(a, b);
+        case SORTTYPE_DESCENDING_ABS:
+            return t_argsort_comparator_impl<SORTTYPE_DESCENDING_ABS>(m_v)(
+                a, b
+            );
+        case SORTTYPE_NONE:
+            return t_argsort_comparator_impl<SORTTYPE_NONE>(m_v)(a, b);
+    }
+
+    return a < b;
+}
+
+namespace {
+
+void
+init_output(std::vector<t_index>& output) {
+    for (t_index i = 0, loop_end = output.size(); i != loop_end; ++i) {
+        output[i] = i;
+    }
+}
+
+} // anonymous namespace
+
+void
+simple_argsort(
+    std::vector<t_tscalar>& v,
+    std::vector<t_index>& output,
+    const t_sorttype& sort_type
+) {
+    init_output(output);
+
+    switch (sort_type) {
+        case SORTTYPE_ASCENDING: {
+            std::sort(
+                output.begin(),
+                output.end(),
+                t_argsort_comparator_impl<SORTTYPE_ASCENDING>(v)
+            );
+        } break;
+        case SORTTYPE_DESCENDING: {
+            std::sort(
+                output.begin(),
+                output.end(),
+                t_argsort_comparator_impl<SORTTYPE_DESCENDING>(v)
+            );
+        } break;
+        case SORTTYPE_ASCENDING_ABS: {
+            std::sort(
+                output.begin(),
+                output.end(),
+                t_argsort_comparator_impl<SORTTYPE_ASCENDING_ABS>(v)
+            );
+        } break;
+        case SORTTYPE_DESCENDING_ABS: {
+            std::sort(
+                output.begin(),
+                output.end(),
+                t_argsort_comparator_impl<SORTTYPE_DESCENDING_ABS>(v)
+            );
+        } break;
+        case SORTTYPE_NONE: {
+            // output is already identity-initialized
+        } break;
+    }
+}
+
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/arrow_csv.cpp

@@ -0,0 +1,639 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <chrono>
+#include <perspective/base.h>
+#include <perspective/arrow_csv.h>
+#include <arrow/util/value_parsing.h>
+#include <arrow/io/memory.h>
+#include <arrow/csv/reader.h>
+
+template <class TimePoint>
+static inline arrow::TimestampType::c_type
+ConvertTimePoint(TimePoint tp, arrow::TimeUnit::type unit) {
+    auto duration = tp.time_since_epoch();
+    switch (unit) {
+        case arrow::TimeUnit::SECOND:
+            return std::chrono::duration_cast<std::chrono::seconds>(duration)
+                .count();
+        case arrow::TimeUnit::MILLI:
+            return std::chrono::duration_cast<std::chrono::milliseconds>(
+                       duration
+            )
+                .count();
+        case arrow::TimeUnit::MICRO:
+            return std::chrono::duration_cast<std::chrono::microseconds>(
+                       duration
+            )
+                .count();
+        case arrow::TimeUnit::NANO:
+            return std::chrono::duration_cast<std::chrono::nanoseconds>(duration
+            )
+                .count();
+        default:
+            // Compiler errors without default case even though all enum cases
+            // are handled
+            assert(0);
+            return 0;
+    }
+}
+
+static inline bool
+ParseYYYY_MM_DD(const char* s, arrow_vendored::date::year_month_day* out) {
+    uint16_t year = 0;
+    uint8_t month = 0;
+    uint8_t day = 0;
+    if (ARROW_PREDICT_FALSE(s[4] != '-') || ARROW_PREDICT_FALSE(s[7] != '-')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 4, &year))) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 5, 2, &month)
+        )) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 8, 2, &day))) {
+        return false;
+    }
+    *out = {
+        arrow_vendored::date::year{year},
+        arrow_vendored::date::month{month},
+        arrow_vendored::date::day{day}
+    };
+    return out->ok();
+}
+
+static inline bool
+ParseYYYY_DD_MM(const char* s, arrow_vendored::date::year_month_day* out) {
+    uint16_t year = 0;
+    uint8_t month = 0;
+    uint8_t day = 0;
+    if (ARROW_PREDICT_FALSE(s[2] != '/') || ARROW_PREDICT_FALSE(s[5] != '/')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 6, 4, &year))) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 2, &month)
+        )) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 3, 2, &day))) {
+        return false;
+    }
+    *out = {
+        arrow_vendored::date::year{year},
+        arrow_vendored::date::month{month},
+        arrow_vendored::date::day{day}
+    };
+    return out->ok();
+}
+
+static inline bool
+ParseYYYY_D_M(const char* s, arrow_vendored::date::year_month_day* out) {
+    uint16_t year = 0;
+    uint8_t month = 0;
+    uint8_t day = 0;
+
+    if (ARROW_PREDICT_FALSE(s[1] != '/') || ARROW_PREDICT_FALSE(s[3] != '/')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 4, 4, &year))) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 0, 1, &month)
+        )) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 2, 1, &day))) {
+        return false;
+    }
+    *out = {
+        arrow_vendored::date::year{year},
+        arrow_vendored::date::month{month},
+        arrow_vendored::date::day{day}
+    };
+    return out->ok();
+}
+
+static inline std::string
+extract_string(const char* ch, int start_idx, int number_of_chars) {
+    std::string out;
+    for (int i = start_idx; i < start_idx + number_of_chars; i++) {
+        out += ch[i];
+    }
+    return out;
+}
+
+static inline bool
+ParseAM_PM(const char* s, std::chrono::seconds& seconds, int length) {
+    int hour = 0;
+    int twelve_hours = 12;
+    std::string am_pm;
+    std::string hour_string;
+
+    if (length == 21) {
+        am_pm = extract_string(s, 19, 2);
+        hour_string = extract_string(s, 10, 2);
+        hour = atoi(hour_string.c_str());
+        if (hour == 0) {
+            return false;
+        }
+    } else if (length == 23) {
+        am_pm = extract_string(s, 21, 2);
+        hour_string = extract_string(s, 12, 2);
+        hour = atoi(hour_string.c_str());
+        if (hour == 0) {
+            return false;
+        }
+    }
+
+    if ((am_pm == "PM" || am_pm == "pm") && (hour < twelve_hours)) {
+        std::chrono::hours hours_obj(twelve_hours);
+        seconds = std::chrono::duration_cast<std::chrono::seconds>(hours_obj);
+    } else if ((am_pm == "AM" || am_pm == "am") && (hour == twelve_hours)) {
+        std::chrono::hours hours_obj(twelve_hours);
+        seconds =
+            std::chrono::duration_cast<std::chrono::seconds>(hours_obj) * -1;
+    }
+    return true;
+}
+
+namespace perspective::apachearrow {
+
+class UnixTimestampParser : public arrow::TimestampParser {
+public:
+    bool
+    operator()(
+        const char* s,
+        size_t length,
+        arrow::TimeUnit::type out_unit,
+        int64_t* out,
+        bool* out_zone_offset_present = NULLPTR
+    ) const override {
+        size_t endptr;
+        std::string val(s, s + length);
+        int64_t value = std::stoll(static_cast<std::string>(val), &endptr, 10);
+        if (endptr != length) {
+            return false;
+        }
+        (*out) = value;
+        return true;
+    }
+
+    [[nodiscard]]
+    const char*
+    kind() const override {
+        return "unixtimestamp";
+    }
+};
+
+static inline bool
+ParseSSS(const char* s, std::chrono::milliseconds* out) {
+    uint16_t millis = 0;
+    if (ARROW_PREDICT_FALSE(s[0] != '.')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 3, &millis)
+        )) {
+        return false;
+    }
+
+    if (ARROW_PREDICT_FALSE(millis >= 999)) {
+        return false;
+    }
+    *out = std::chrono::milliseconds(millis);
+    return true;
+}
+
+static inline bool
+ParseSSSSSS(const char* s, std::chrono::microseconds* out) {
+    uint32_t nanos = 0;
+    if (ARROW_PREDICT_FALSE(s[0] != '.')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 6, &nanos)
+        )) {
+        return false;
+    }
+
+    if (ARROW_PREDICT_FALSE(nanos >= 999999)) {
+        return false;
+    }
+    *out = std::chrono::microseconds(nanos);
+    return true;
+}
+
+static inline bool
+ParseSSSSSSSSS(const char* s, std::chrono::nanoseconds* out) {
+    uint32_t nanos = 0;
+    if (ARROW_PREDICT_FALSE(s[0] != '.')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 9, &nanos)
+        )) {
+        return false;
+    }
+
+    if (ARROW_PREDICT_FALSE(nanos >= 999999999)) {
+        return false;
+    }
+    *out = std::chrono::nanoseconds(nanos);
+    return true;
+}
+
+static inline bool
+ParseTZ(const char* s, std::chrono::minutes* out) {
+    uint8_t hours = 0;
+    uint8_t minutes = 0;
+    if ((ARROW_PREDICT_FALSE(s[0] != '+') && ARROW_PREDICT_FALSE(s[0] != '-'))
+        || ARROW_PREDICT_FALSE(s[3] != ':')) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 1, 2, &hours)
+        )) {
+        return false;
+    }
+    if (ARROW_PREDICT_FALSE(!arrow::internal::ParseUnsigned(s + 4, 2, &minutes)
+        )) {
+        return false;
+    }
+
+    if (ARROW_PREDICT_FALSE(hours >= 12)
+        || ARROW_PREDICT_FALSE(minutes >= 59)) {
+        return false;
+    }
+    int32_t total = hours * 60 + minutes;
+    if (s[0] == '+') {
+        total = -total;
+    }
+    *out = std::chrono::minutes(total);
+    return true;
+}
+
+class CustomISO8601Parser : public arrow::TimestampParser {
+public:
+    bool
+    operator()(
+        const char* s,
+        size_t length,
+        arrow::TimeUnit::type unit,
+        int64_t* out,
+        bool* out_zone_offset_present = NULLPTR
+    ) const override {
+        // if we are trying to parse this with seconds, fail
+        // and it will try to parse this again but as
+        // nanoseconds :) then it wont truncate the fractional bits.
+        if (unit == arrow::TimeUnit::SECOND) {
+            return false;
+        }
+
+        if (!arrow::internal::ParseTimestampISO8601(s, length, unit, out)) {
+            if (s[length - 1] == 'Z') {
+                --length;
+            }
+            if (length == 23) {
+                // "YYYY-MM-DD[ T]hh:mm:ss.sss" -- millis
+
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                std::chrono::milliseconds millis;
+                if (ARROW_PREDICT_FALSE(!ParseSSS(s + 19, &millis))) {
+                    return false;
+                }
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
+                );
+                return true;
+            }
+
+            if (length == 25) {
+                // "2008-09-15[ T]15:53:00+05:00" -- seconds with TZ
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                std::chrono::minutes tz;
+                if (ARROW_PREDICT_FALSE(!ParseTZ(s + 19, &tz))) {
+                    return false;
+                }
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + tz + seconds, unit
+                );
+                return true;
+            }
+            if (length == 26) {
+                // YYYY-MM-DD[ T]hh:mm:ss.ssssss  -- micros
+
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                std::chrono::microseconds micros;
+                if (ARROW_PREDICT_FALSE(!ParseSSSSSS(s + 19, &micros))) {
+                    return false;
+                }
+                // round the micros into millis as Perspective does not support
+                // nano precision.
+                auto millis =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(micros
+                    );
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
+                );
+
+                return true;
+            }
+
+            if (length == 29) {
+                // YYYY-MM-DD[ T]hh:mm:ss.sssssssss  -- nanos
+                // arrow handles YYYY-MM-DD[ T]hh:mm:ss.sss[+-]HH:MM
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                // we can now be at sss[+-]HH:MM  -- millis and TZ
+                // or sssssssss -- nanos
+                std::chrono::nanoseconds nanos;
+                if (ARROW_PREDICT_FALSE(!ParseSSSSSSSSS(s + 19, &nanos))) {
+                    return false;
+                }
+                // Truncate the nanos into millis as Perspective does not
+                // support nano precision.
+                auto millis =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(nanos
+                    );
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + millis, unit
+                );
+
+                return true;
+            }
+            if (length == 32) {
+                // YYYY-MM-DD[ T]hh:mm:ss.ssssss[+-]HH:MM  -- micros with TZ
+
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                std::chrono::microseconds micros;
+                if (ARROW_PREDICT_FALSE(!ParseSSSSSS(s + 19, &micros))) {
+                    return false;
+                }
+                // round the micros into millis as Perspective does not support
+                // nano precision.
+                auto millis =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(micros
+                    );
+
+                std::chrono::minutes tz;
+                if (ARROW_PREDICT_FALSE(!ParseTZ(s + 26, &tz))) {
+                    return false;
+                }
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + millis + tz,
+                    unit
+                );
+
+                return true;
+            }
+
+            if (length == 35) {
+                // YYYY-MM-DD[ T]hh:mm:ss.sssssssss[+-]HH:MM -- nanos with TZ
+
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_MM_DD(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 11, &seconds
+                    ))) {
+                    return false;
+                }
+                std::chrono::nanoseconds nanos;
+                if (ARROW_PREDICT_FALSE(!ParseSSSSSSSSS(s + 19, &nanos))) {
+                    return false;
+                }
+                // round the nanos into millis as Perspective does not support
+                // nano precision.
+                auto millis =
+                    std::chrono::duration_cast<std::chrono::milliseconds>(nanos
+                    );
+
+                std::chrono::minutes tz;
+                if (ARROW_PREDICT_FALSE(!ParseTZ(s + 29, &tz))) {
+                    return false;
+                }
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + millis + tz,
+                    unit
+                );
+
+                return true;
+            }
+            return false;
+        }
+        return true;
+    }
+
+    [[nodiscard]]
+    const char*
+    kind() const override {
+        return "custom_ISO8601";
+    }
+};
+
+class USTimestampParser : public arrow::TimestampParser {
+public:
+    bool
+    operator()(
+        const char* s,
+        size_t length,
+        arrow::TimeUnit::type unit,
+        int64_t* out,
+        bool* out_zone_offset_present = NULLPTR
+    ) const override {
+
+        if (!arrow::internal::ParseTimestampISO8601(s, length, unit, out)) {
+            if (length == 23) {
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_DD_MM(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 12, &seconds
+                    ))) {
+                    return false;
+                }
+
+                std::chrono::seconds am_pm(0);
+                if (ARROW_PREDICT_FALSE(!ParseAM_PM(s, am_pm, 23))) {
+                    return false;
+                }
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + am_pm, unit
+                );
+                return true;
+            }
+            if (length == 21) {
+                arrow_vendored::date::year_month_day ymd;
+                if (ARROW_PREDICT_FALSE(!ParseYYYY_D_M(s, &ymd))) {
+                    return false;
+                }
+                std::chrono::seconds seconds;
+                if (ARROW_PREDICT_FALSE(!arrow::internal::detail::ParseHH_MM_SS(
+                        s + 10, &seconds
+                    ))) {
+                    return false;
+                }
+
+                std::chrono::seconds am_pm(0);
+                if (ARROW_PREDICT_FALSE(!ParseAM_PM(s, am_pm, 21))) {
+                    return false;
+                }
+
+                *out = ConvertTimePoint(
+                    arrow_vendored::date::sys_days(ymd) + seconds + am_pm, unit
+                );
+                return true;
+            }
+            return false;
+        }
+        return true;
+    }
+
+    [[nodiscard]]
+    const char*
+    kind() const override {
+        return "USTimestampParser";
+    }
+};
+
+std::vector<std::shared_ptr<arrow::TimestampParser>> DATE_PARSERS{
+    std::make_shared<CustomISO8601Parser>(),
+    std::make_shared<USTimestampParser>(),
+    arrow::TimestampParser::MakeStrptime("%Y-%m-%d\\D%H:%M:%S.%f"),
+    arrow::TimestampParser::MakeStrptime("%m/%d/%Y, %I:%M:%S %p"),
+    // US locale string
+    arrow::TimestampParser::MakeStrptime("%m-%d-%Y"),
+    arrow::TimestampParser::MakeStrptime("%m/%d/%Y"),
+    arrow::TimestampParser::MakeStrptime("%d %m %Y"),
+    // TODO: time type column
+    arrow::TimestampParser::MakeStrptime("%H:%M:%S.%f")
+};
+
+std::vector<std::shared_ptr<arrow::TimestampParser>> DATE_READERS{
+    std::make_shared<UnixTimestampParser>(),
+    std::make_shared<CustomISO8601Parser>(),
+    std::make_shared<USTimestampParser>(),
+    arrow::TimestampParser::MakeStrptime("%Y-%m-%d\\D%H:%M:%S.%f"),
+    arrow::TimestampParser::MakeStrptime("%m/%d/%Y, %I:%M:%S %p"),
+    // US locale
+    arrow::TimestampParser::MakeStrptime("%m-%d-%Y"),
+    arrow::TimestampParser::MakeStrptime("%m/%d/%Y"),
+    arrow::TimestampParser::MakeStrptime("%d %m %Y"),
+    arrow::TimestampParser::MakeStrptime("%H:%M:%S.%f")
+};
+
+std::optional<int64_t>
+parseAsArrowTimestamp(const std::string& input) {
+    for (const auto& candidate : DATE_PARSERS) {
+        int64_t datetime;
+        if (candidate->operator()(
+                input.c_str(), input.size(), arrow::TimeUnit::MILLI, &datetime
+            )) {
+            return datetime;
+        }
+    }
+
+    return std::nullopt;
+}
+
+std::shared_ptr<::arrow::Table>
+csvToTable(
+    const std::string_view& csv,
+    bool is_update,
+    std::unordered_map<std::string, std::shared_ptr<arrow::DataType>>& schema
+) {
+    const arrow::io::IOContext& io_context = arrow::io::default_io_context();
+    auto input = std::make_shared<arrow::io::BufferReader>(csv);
+    auto read_options = arrow::csv::ReadOptions::Defaults();
+    auto parse_options = arrow::csv::ParseOptions::Defaults();
+    auto convert_options = arrow::csv::ConvertOptions::Defaults();
+#ifdef PSP_PARALLEL_FOR
+    read_options.use_threads = true;
+#else
+    read_options.use_threads = false;
+#endif
+    parse_options.newlines_in_values = true;
+
+    if (is_update) {
+        convert_options.column_types = std::move(schema);
+        convert_options.timestamp_parsers = DATE_READERS;
+    } else {
+        convert_options.timestamp_parsers = DATE_PARSERS;
+    }
+
+    auto maybe_reader = arrow::csv::TableReader::Make(
+        io_context, input, read_options, parse_options, convert_options
+    );
+
+    std::shared_ptr<arrow::csv::TableReader> reader = *maybe_reader;
+
+    auto maybe_table = reader->Read();
+    if (!maybe_table.ok()) {
+        PSP_COMPLAIN_AND_ABORT(maybe_table.status().ToString());
+    }
+    return *maybe_table;
+}
+
+} // namespace perspective::apachearrow

rust/perspective-server/cpp/perspective/src/cpp/arrow_writer.cpp

@@ -0,0 +1,81 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/arrow_writer.h>
+
+namespace perspective::apachearrow {
+using namespace perspective;
+
+// TODO: unsure about efficacy of these functions when get<T> exists
+template <>
+double
+get_scalar<double>(t_tscalar& t) {
+    return t.to_double();
+}
+template <>
+float
+get_scalar<float>(t_tscalar& t) {
+    return static_cast<float>(t.to_double());
+}
+template <>
+std::uint8_t
+get_scalar<std::uint8_t>(t_tscalar& t) {
+    return static_cast<std::uint8_t>(t.to_int64());
+}
+template <>
+std::int8_t
+get_scalar<std::int8_t>(t_tscalar& t) {
+    return static_cast<std::int8_t>(t.to_int64());
+}
+template <>
+std::int16_t
+get_scalar<std::int16_t>(t_tscalar& t) {
+    return static_cast<std::int16_t>(t.to_int64());
+}
+template <>
+std::uint16_t
+get_scalar<std::uint16_t>(t_tscalar& t) {
+    return static_cast<std::uint16_t>(t.to_int64());
+}
+template <>
+std::int32_t
+get_scalar<std::int32_t>(t_tscalar& t) {
+    return static_cast<std::int32_t>(t.to_int64());
+}
+template <>
+std::uint32_t
+get_scalar<std::uint32_t>(t_tscalar& t) {
+    return static_cast<std::uint32_t>(t.to_int64());
+}
+template <>
+std::int64_t
+get_scalar<std::int64_t>(t_tscalar& t) {
+    return static_cast<std::int64_t>(t.to_int64());
+}
+template <>
+std::uint64_t
+get_scalar<std::uint64_t>(t_tscalar& t) {
+    return static_cast<std::uint64_t>(t.to_int64());
+}
+template <>
+bool
+get_scalar<bool>(t_tscalar& t) {
+    return t.get<bool>();
+}
+
+// std::int32_t
+// get_idx(std::int32_t cidx, std::int32_t ridx, std::int32_t stride,
+//     t_get_data_extents extents) {
+//     return (ridx - extents.m_srow) * stride + (cidx - extents.m_scol);
+// }
+
+} // namespace perspective::apachearrow

rust/perspective-server/cpp/perspective/src/cpp/base.cpp

@@ -0,0 +1,821 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <cstdint>
+#include <limits>
+#include <perspective/exception.h>
+
+namespace perspective {
+
+void
+psp_abort(const std::string& message) {
+    throw PerspectiveException(message.c_str());
+}
+
+bool
+is_numeric_type(t_dtype dtype) {
+    switch (dtype) {
+        case DTYPE_UINT8:
+        case DTYPE_UINT16:
+        case DTYPE_UINT32:
+        case DTYPE_UINT64:
+        case DTYPE_INT8:
+        case DTYPE_INT16:
+        case DTYPE_INT32:
+        case DTYPE_INT64:
+        case DTYPE_FLOAT32:
+        case DTYPE_FLOAT64: {
+            return true;
+        } break;
+        default: {
+            return false;
+        }
+    }
+}
+
+bool
+is_linear_order_type(t_dtype dtype) {
+    switch (dtype) {
+        case DTYPE_UINT8:
+        case DTYPE_UINT16:
+        case DTYPE_UINT32:
+        case DTYPE_UINT64:
+        case DTYPE_INT8:
+        case DTYPE_INT16:
+        case DTYPE_INT32:
+        case DTYPE_INT64:
+        case DTYPE_FLOAT32:
+        case DTYPE_FLOAT64:
+        case DTYPE_DATE:
+        case DTYPE_TIME:
+        case DTYPE_BOOL: {
+            return true;
+        } break;
+        default: {
+            return false;
+        }
+    }
+}
+
+bool
+is_floating_point(t_dtype dtype) {
+    return (dtype == DTYPE_FLOAT32 || dtype == DTYPE_FLOAT64);
+}
+
+bool
+is_deterministic_sized(t_dtype dtype) {
+    switch (dtype) {
+        case DTYPE_OBJECT:
+        case DTYPE_INT64:
+        case DTYPE_UINT64:
+        case DTYPE_INT32:
+        case DTYPE_UINT32:
+        case DTYPE_INT16:
+        case DTYPE_UINT16:
+        case DTYPE_BOOL:
+        case DTYPE_INT8:
+        case DTYPE_UINT8:
+        case DTYPE_FLOAT64:
+        case DTYPE_FLOAT32:
+        case DTYPE_STR:
+        case DTYPE_TIME:
+        case DTYPE_DATE:
+        case DTYPE_F64PAIR: {
+            return true;
+        }
+        default: {
+            return false;
+        }
+    }
+
+    PSP_COMPLAIN_AND_ABORT("Reached unreachable");
+    return false;
+}
+
+t_uindex
+get_dtype_size(t_dtype dtype) {
+    switch (dtype) {
+        case DTYPE_OBJECT: {
+            return sizeof(void*);
+        }
+        case DTYPE_INT64:
+        case DTYPE_UINT64: {
+            return sizeof(std::int64_t);
+        }
+        case DTYPE_INT32:
+        case DTYPE_UINT32: {
+            return sizeof(std::int32_t);
+        }
+        case DTYPE_INT16:
+        case DTYPE_UINT16: {
+            return 2;
+        }
+        case DTYPE_BOOL:
+        case DTYPE_INT8:
+        case DTYPE_UINT8:
+        case DTYPE_NONE: {
+            return 1;
+        }
+        case DTYPE_FLOAT64: {
+            return sizeof(double);
+        }
+        case DTYPE_FLOAT32: {
+            return sizeof(float);
+        }
+        case DTYPE_STR: {
+            return sizeof(t_uindex);
+        }
+        case DTYPE_TIME: {
+            return sizeof(std::int64_t);
+        }
+        case DTYPE_DATE: {
+            return sizeof(std::uint32_t);
+        }
+        case DTYPE_F64PAIR: {
+            return sizeof(std::pair<double, double>);
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unknown dtype");
+        }
+    }
+
+    PSP_COMPLAIN_AND_ABORT("Reached unreachable");
+    return sizeof(DTYPE_INT64);
+}
+
+bool
+is_vlen_dtype(t_dtype dtype) {
+    return dtype == DTYPE_STR || dtype == DTYPE_USER_VLEN;
+}
+
+std::string
+get_dtype_descr(t_dtype dtype) {
+    switch (dtype) {
+        case DTYPE_NONE: {
+            return "none";
+        } break;
+        case DTYPE_INT64: {
+            return "int64";
+        } break;
+        case DTYPE_INT32: {
+            return "int32";
+        } break;
+        case DTYPE_INT16: {
+            return "int16";
+        } break;
+        case DTYPE_INT8: {
+            return "int8";
+        } break;
+        case DTYPE_UINT64: {
+            return "uint64";
+        } break;
+        case DTYPE_UINT32: {
+            return "uint32";
+        } break;
+        case DTYPE_UINT16: {
+            return "uint16";
+        } break;
+        case DTYPE_UINT8: {
+            return "uint8";
+        } break;
+        case DTYPE_BOOL: {
+            return "bool";
+        } break;
+        case DTYPE_FLOAT64: {
+            return "float64";
+        } break;
+        case DTYPE_FLOAT32: {
+            return "float32";
+        } break;
+        case DTYPE_STR: {
+            return "str";
+        } break;
+        case DTYPE_TIME: {
+            return "datetime";
+        } break;
+        case DTYPE_DATE: {
+            return "date";
+        } break;
+        case DTYPE_ENUM: {
+            return "e";
+        } break;
+        case DTYPE_OID: {
+            return "oid";
+        } break;
+        case DTYPE_USER_FIXED: {
+            return "ufix";
+        } break;
+        case DTYPE_LAST: {
+            return "last";
+        } break;
+        case DTYPE_USER_VLEN: {
+            return "uvlen";
+        } break;
+        case DTYPE_F64PAIR: {
+            return "f64pair";
+        } break;
+        case DTYPE_OBJECT: {
+            return "object";
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Encountered unknown dtype");
+        }
+    }
+    return {"dummy"};
+}
+
+std::string
+dtype_to_str(t_dtype dtype) {
+    std::stringstream ss;
+    switch (dtype) {
+        case DTYPE_FLOAT32:
+        case DTYPE_FLOAT64: {
+            ss << "float";
+        } break;
+        case DTYPE_UINT8:
+        case DTYPE_UINT16:
+        case DTYPE_UINT32:
+        case DTYPE_UINT64:
+        case DTYPE_INT8:
+        case DTYPE_INT16:
+        case DTYPE_INT32:
+        case DTYPE_INT64: {
+            ss << "integer";
+        } break;
+        case DTYPE_BOOL: {
+            ss << "boolean";
+        } break;
+        case DTYPE_DATE: {
+            ss << "date";
+        } break;
+        case DTYPE_TIME: {
+            ss << "datetime";
+        } break;
+        case DTYPE_STR: {
+            ss << "string";
+        } break;
+        case DTYPE_OBJECT: {
+            ss << "object";
+        } break;
+        case DTYPE_NONE: {
+            ss << "none";
+        } break;
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Cannot convert unknown dtype to string!");
+        }
+    }
+
+    return ss.str();
+}
+
+t_dtype
+str_to_dtype(const std::string& typestring) {
+    // returns most commonly used types in the JS/python public APIs.
+    if (typestring == "integer") {
+        return DTYPE_INT32;
+    }
+    if (typestring == "float") {
+        return DTYPE_FLOAT64;
+    }
+    if (typestring == "boolean") {
+        return DTYPE_BOOL;
+    }
+    if (typestring == "date") {
+        return DTYPE_DATE;
+    }
+    if (typestring == "datetime") {
+        return DTYPE_TIME;
+    }
+    if (typestring == "string") {
+        return DTYPE_STR;
+    }
+
+    PSP_COMPLAIN_AND_ABORT(
+        "Could not convert unknown type string `" + typestring + "` to dtype."
+    );
+    return DTYPE_NONE;
+}
+
+std::string
+filter_op_to_str(t_filter_op op) {
+    switch (op) {
+        case FILTER_OP_LT: {
+            return "<";
+        } break;
+        case FILTER_OP_LTEQ: {
+            return "<=";
+        } break;
+        case FILTER_OP_GT: {
+            return ">";
+        } break;
+        case FILTER_OP_GTEQ: {
+            return ">=";
+        } break;
+        case FILTER_OP_EQ: {
+            return "==";
+        } break;
+        case FILTER_OP_NE: {
+            return "!=";
+        } break;
+        case FILTER_OP_BEGINS_WITH: {
+            return "startswith";
+        } break;
+        case FILTER_OP_ENDS_WITH: {
+            return "endswith";
+        } break;
+        case FILTER_OP_CONTAINS: {
+            return "contains";
+        } break;
+        case FILTER_OP_OR: {
+            return "or";
+        } break;
+        case FILTER_OP_IN: {
+            return "in";
+        } break;
+        case FILTER_OP_NOT_IN: {
+            return "not in";
+        } break;
+        case FILTER_OP_AND: {
+            return "and";
+        } break;
+        case FILTER_OP_IS_NULL: {
+            return "is null";
+        } break;
+        case FILTER_OP_IS_NOT_NULL: {
+            return "is not null";
+        } break;
+    }
+    PSP_COMPLAIN_AND_ABORT("Reached end of function");
+    return "";
+}
+
+t_filter_op
+str_to_filter_op(const std::string& str) {
+    if (str == "<") {
+        return t_filter_op::FILTER_OP_LT;
+    }
+    if (str == "<=") {
+        return t_filter_op::FILTER_OP_LTEQ;
+    }
+    if (str == ">") {
+        return t_filter_op::FILTER_OP_GT;
+    }
+    if (str == ">=") {
+        return t_filter_op::FILTER_OP_GTEQ;
+    }
+    if (str == "==") {
+        return t_filter_op::FILTER_OP_EQ;
+    }
+    if (str == "!=") {
+        return t_filter_op::FILTER_OP_NE;
+    }
+    if (str == "begins with" || str == "startswith") {
+        return t_filter_op::FILTER_OP_BEGINS_WITH;
+    }
+    if (str == "ends with" || str == "endswith") {
+        return t_filter_op::FILTER_OP_ENDS_WITH;
+    }
+    if (str == "in") {
+        return t_filter_op::FILTER_OP_IN;
+    }
+    if (str == "contains") {
+        return t_filter_op::FILTER_OP_CONTAINS;
+    }
+    if (str == "not in") {
+        return t_filter_op::FILTER_OP_NOT_IN;
+    }
+    if (str == "&" || str == "and") {
+        return t_filter_op::FILTER_OP_AND;
+    }
+    if (str == "|" || str == "or") {
+        return t_filter_op::FILTER_OP_OR;
+    }
+    if (str == "is null" || str == "is None") {
+        return t_filter_op::FILTER_OP_IS_NULL;
+    }
+    if (str == "is not null" || str == "is not None") {
+        return t_filter_op::FILTER_OP_IS_NOT_NULL;
+    }
+
+    std::stringstream ss;
+    ss << "Unknown filter operator string: `" << str << "`" << std::endl;
+    PSP_COMPLAIN_AND_ABORT(ss.str());
+    return t_filter_op::FILTER_OP_AND;
+}
+
+t_sorttype
+str_to_sorttype(const std::string& str) {
+    if (str == "none") {
+        return SORTTYPE_NONE;
+    }
+    if (str == "asc" || str == "col asc") {
+        return SORTTYPE_ASCENDING;
+    }
+    if (str == "desc" || str == "col desc") {
+        return SORTTYPE_DESCENDING;
+    }
+    if (str == "asc abs" || str == "col asc abs") {
+        return SORTTYPE_ASCENDING_ABS;
+    }
+    if (str == "desc abs" || str == "col desc abs") {
+        return SORTTYPE_DESCENDING_ABS;
+    }
+
+    std::stringstream ss;
+    ss << "Unknown sort type string: `" << str << std::endl;
+    PSP_COMPLAIN_AND_ABORT(ss.str());
+    return SORTTYPE_DESCENDING;
+}
+
+std::string
+sorttype_to_str(t_sorttype type) {
+    switch (type) {
+        case SORTTYPE_NONE: {
+            return "none";
+        } break;
+        case SORTTYPE_ASCENDING: {
+            return "asc";
+        } break;
+        case SORTTYPE_DESCENDING: {
+            return "desc";
+        } break;
+        case SORTTYPE_ASCENDING_ABS: {
+            return "asc abs";
+        } break;
+        case SORTTYPE_DESCENDING_ABS: {
+            return "desc abs";
+        } break;
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unknown sort type");
+        }
+    }
+}
+
+t_aggtype
+str_to_aggtype(const std::string& str) {
+    if (str == "distinct count" || str == "distinctcount" || str == "distinct"
+        || str == "distinct_count") {
+        return t_aggtype::AGGTYPE_DISTINCT_COUNT;
+    }
+    if (str == "sum") {
+        return t_aggtype::AGGTYPE_SUM;
+    }
+    if (str == "mul") {
+        return t_aggtype::AGGTYPE_MUL;
+    }
+    if (str == "avg" || str == "mean") {
+        return t_aggtype::AGGTYPE_MEAN;
+    }
+    if (str == "count") {
+        return t_aggtype::AGGTYPE_COUNT;
+    }
+    if (str == "weighted mean" || str == "weighted_mean") {
+        return t_aggtype::AGGTYPE_WEIGHTED_MEAN;
+    }
+    if (str == "unique") {
+        return t_aggtype::AGGTYPE_UNIQUE;
+    }
+    if (str == "any") {
+        return t_aggtype::AGGTYPE_ANY;
+    }
+    if (str == "q1") {
+        return t_aggtype::AGGTYPE_Q1;
+    }
+    if (str == "q3") {
+        return t_aggtype::AGGTYPE_Q3;
+    }
+    if (str == "median") {
+        return t_aggtype::AGGTYPE_MEDIAN;
+    }
+    if (str == "join") {
+        return t_aggtype::AGGTYPE_JOIN;
+    }
+    if (str == "div") {
+        return t_aggtype::AGGTYPE_SCALED_DIV;
+    }
+    if (str == "add") {
+        return t_aggtype::AGGTYPE_SCALED_ADD;
+    }
+    if (str == "dominant") {
+        return t_aggtype::AGGTYPE_DOMINANT;
+    }
+    if (str == "first by index" || str == "first") {
+        return t_aggtype::AGGTYPE_FIRST;
+    }
+    if (str == "last by index") {
+        return t_aggtype::AGGTYPE_LAST_BY_INDEX;
+    }
+    if (str == "last minus first") {
+        return t_aggtype::AGGTYPE_LAST_MINUS_FIRST;
+    }
+    if (str == "py_agg") {
+        return t_aggtype::AGGTYPE_PY_AGG;
+    }
+    if (str == "and") {
+        return t_aggtype::AGGTYPE_AND;
+    }
+    if (str == "or") {
+        return t_aggtype::AGGTYPE_OR;
+    }
+    if (str == "last" || str == "last_value") {
+        return t_aggtype::AGGTYPE_LAST_VALUE;
+    }
+    if (str == "max_by" || str == "max by") {
+        return t_aggtype::AGGTYPE_MAX_BY;
+    }
+    if (str == "min_by" || str == "min by") {
+        return t_aggtype::AGGTYPE_MIN_BY;
+    }
+    if (str == "max") {
+        return t_aggtype::AGGTYPE_MAX;
+    }
+    if (str == "min") {
+        return t_aggtype::AGGTYPE_MIN;
+    }
+    if (str == "high" || str == "high_water_mark") {
+        return t_aggtype::AGGTYPE_HIGH_WATER_MARK;
+    }
+    if (str == "low" || str == "low_water_mark") {
+        return t_aggtype::AGGTYPE_LOW_WATER_MARK;
+    }
+    if (str == "high minus low") {
+        return t_aggtype::AGGTYPE_HIGH_MINUS_LOW;
+    }
+    if (str == "sum abs" || str == "sum_abs") {
+        return t_aggtype::AGGTYPE_SUM_ABS;
+    }
+    if (str == "abs sum" || str == "abs_sum") {
+        return t_aggtype::AGGTYPE_ABS_SUM;
+    }
+    if (str == "sum not null" || str == "sum_not_null") {
+        return t_aggtype::AGGTYPE_SUM_NOT_NULL;
+    }
+    if (str == "mean by count" || str == "mean_by_count") {
+        return t_aggtype::AGGTYPE_MEAN_BY_COUNT;
+    }
+    if (str == "identity") {
+        return t_aggtype::AGGTYPE_IDENTITY;
+    }
+    if (str == "distinct leaf" || str == "distinct_leaf") {
+        return t_aggtype::AGGTYPE_DISTINCT_LEAF;
+    }
+    if (str == "pct sum parent" || str == "pct_sum_parent") {
+        return t_aggtype::AGGTYPE_PCT_SUM_PARENT;
+    }
+    if (str == "pct sum total" || str == "pct_sum_total"
+        || str == "pct sum grand total" || str == "pct_sum_grand_total") {
+        return t_aggtype::AGGTYPE_PCT_SUM_GRAND_TOTAL;
+    }
+    if (str.find("udf_combiner_") != std::string::npos) {
+        return t_aggtype::AGGTYPE_UDF_COMBINER;
+    }
+    if (str.find("udf_reducer_") != std::string::npos) {
+        return t_aggtype::AGGTYPE_UDF_REDUCER;
+    }
+    if (str == "var" || str == "variance") {
+        return t_aggtype::AGGTYPE_VARIANCE;
+    }
+    if (str == "stddev" || str == "standard deviation") {
+        return t_aggtype::AGGTYPE_STANDARD_DEVIATION;
+    }
+    if (str == "gmv") {
+        return t_aggtype::AGGTYPE_GMV;
+    }
+
+    std::stringstream ss;
+    ss << "Encountered unknown aggregate operation: '" << str << "'"
+       << "\n";
+    PSP_COMPLAIN_AND_ABORT(ss.str());
+    // use any as default
+    return t_aggtype::AGGTYPE_ANY;
+}
+
+t_aggtype
+_get_default_aggregate(t_dtype dtype) {
+    t_aggtype agg_op;
+    switch (dtype) {
+        case DTYPE_FLOAT64:
+        case DTYPE_FLOAT32:
+        case DTYPE_UINT8:
+        case DTYPE_UINT16:
+        case DTYPE_UINT32:
+        case DTYPE_UINT64:
+        case DTYPE_INT8:
+        case DTYPE_INT16:
+        case DTYPE_INT32:
+        case DTYPE_INT64: {
+            agg_op = t_aggtype::AGGTYPE_SUM;
+        } break;
+        default: {
+            agg_op = t_aggtype::AGGTYPE_COUNT;
+        }
+    }
+    return agg_op;
+}
+
+std::string
+_get_default_aggregate_string(t_dtype dtype) {
+    std::string agg_op_str;
+    switch (dtype) {
+        case DTYPE_FLOAT64:
+        case DTYPE_FLOAT32:
+        case DTYPE_UINT8:
+        case DTYPE_UINT16:
+        case DTYPE_UINT32:
+        case DTYPE_UINT64:
+        case DTYPE_INT8:
+        case DTYPE_INT16:
+        case DTYPE_INT32:
+        case DTYPE_INT64: {
+            agg_op_str = "sum";
+        } break;
+        default: {
+            agg_op_str = "count";
+        }
+    }
+    return agg_op_str;
+}
+
+std::string
+get_status_descr(t_status status) {
+    switch (status) {
+        case STATUS_INVALID: {
+            return "i";
+        }
+        case STATUS_VALID: {
+            return "v";
+        }
+        case STATUS_CLEAR: {
+            return "c";
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unexpected status found");
+        }
+    }
+    return "";
+}
+
+void
+check_init(bool init, const char* file, std::int32_t line) {
+    PSP_VERBOSE_ASSERT(init, "touching uninited object");
+}
+
+bool
+is_neq_transition(t_value_transition t) {
+    return t > VALUE_TRANSITION_EQ_TT;
+}
+
+std::string
+value_transition_to_str(t_value_transition t) {
+    switch (t) {
+        case VALUE_TRANSITION_EQ_FF:
+            return "VALUE_TRANSITION_EQ_FF";
+        case VALUE_TRANSITION_EQ_TT:
+            return "VALUE_TRANSITION_EQ_TT";
+        case VALUE_TRANSITION_NEQ_FT:
+            return "VALUE_TRANSITION_NEQ_FT";
+        case VALUE_TRANSITION_NEQ_TF:
+            return "VALUE_TRANSITION_NEQ_TF";
+        case VALUE_TRANSITION_NEQ_TT:
+            return "VALUE_TRANSITION_NEQ_TT";
+        case VALUE_TRANSITION_NEQ_TDF:
+            return "VALUE_TRANSITION_NEQ_TDF";
+        case VALUE_TRANSITION_NEQ_TDT:
+            return "VALUE_TRANSITION_NEQ_TDT";
+        case VALUE_TRANSITION_NVEQ_FT:
+            return "VALUE_TRANSITION_NVEQ_FT";
+        default:
+            break;
+    }
+
+    PSP_COMPLAIN_AND_ABORT("Unexpected value transition.");
+    return "";
+}
+
+t_uindex
+root_pidx() {
+    return std::numeric_limits<t_uindex>::max();
+}
+
+bool
+is_internal_colname(const std::string& c) {
+    return c == "psp_";
+}
+
+template <typename T>
+t_dtype
+type_to_dtype() {
+    return DTYPE_NONE;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::int64_t>() {
+    return DTYPE_INT64;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::int32_t>() {
+    return DTYPE_INT32;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::int16_t>() {
+    return DTYPE_INT16;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::int8_t>() {
+    return DTYPE_INT8;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::uint64_t>() {
+    return DTYPE_UINT64;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::uint32_t>() {
+    return DTYPE_UINT32;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::uint16_t>() {
+    return DTYPE_UINT16;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::uint8_t>() {
+    return DTYPE_UINT8;
+}
+
+template <>
+t_dtype
+type_to_dtype<double>() {
+    return DTYPE_FLOAT64;
+}
+
+template <>
+t_dtype
+type_to_dtype<float>() {
+    return DTYPE_FLOAT32;
+}
+
+template <>
+t_dtype
+type_to_dtype<bool>() {
+    return DTYPE_BOOL;
+}
+
+template <>
+t_dtype
+type_to_dtype<t_time>() {
+    return DTYPE_TIME;
+}
+
+template <>
+t_dtype
+type_to_dtype<t_date>() {
+    return DTYPE_DATE;
+}
+
+template <>
+t_dtype
+type_to_dtype<std::string>() {
+    return DTYPE_STR;
+}
+
+template <>
+t_dtype
+type_to_dtype<void*>() {
+    return DTYPE_OBJECT;
+}
+
+} // end namespace perspective
+
+namespace std {
+
+void
+string_to_lower(string& str) {
+    transform(str.begin(), str.end(), str.begin(), ::tolower);
+}
+
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/base_impl_linux.cpp

@@ -0,0 +1,26 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __linux__
+#include <perspective/first.h>
+#include <perspective/base.h>
+
+namespace perspective {
+
+std::string
+get_error_str() {
+    // handled by perror
+    return std::string();
+}
+
+} // end namespace perspective
+#endif

rust/perspective-server/cpp/perspective/src/cpp/base_impl_osx.cpp

@@ -0,0 +1,26 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __APPLE__
+#include <perspective/first.h>
+#include <perspective/base.h>
+
+namespace perspective {
+
+std::string
+get_error_str() {
+    // handled by perror
+    return {};
+}
+
+} // end namespace perspective
+#endif

rust/perspective-server/cpp/perspective/src/cpp/base_impl_wasm.cpp

@@ -0,0 +1,26 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef PSP_ENABLE_WASM
+#include <perspective/first.h>
+#include <perspective/base.h>
+
+namespace perspective {
+
+std::string
+get_error_str() {
+    // handled by perror
+    return std::string();
+}
+
+} // end namespace perspective
+#endif

rust/perspective-server/cpp/perspective/src/cpp/base_impl_win.cpp

@@ -0,0 +1,47 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef WIN32
+#include <perspective/first.h>
+#include <perspective/base.h>
+
+namespace perspective {
+
+std::string
+get_error_str() {
+    DWORD errid = ::GetLastError();
+
+    if (errid == 0) {
+        return std::string();
+    }
+
+    LPSTR buf = 0;
+
+    size_t size = FormatMessageA(
+        FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM
+            | FORMAT_MESSAGE_IGNORE_INSERTS,
+        NULL,
+        errid,
+        MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+        (LPSTR)&buf,
+        0,
+        NULL
+    );
+
+    std::string message(buf, size);
+    LocalFree(buf);
+
+    return message;
+}
+
+} // end namespace perspective
+#endif

rust/perspective-server/cpp/perspective/src/cpp/binding.cpp

@@ -0,0 +1,16 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/binding.h>
+using namespace perspective;

rust/perspective-server/cpp/perspective/src/cpp/binding_api.cpp

@@ -0,0 +1,180 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include "perspective/exports.h"
+#include "perspective/server.h"
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <tsl/hopscotch_map.h>
+#include <arrow/util/thread_pool.h>
+
+#if HEAP_INSTRUMENTS
+#include <emscripten/heap.h>
+
+#define UNINSTRUMENTED_MALLOC(x) emscripten_builtin_malloc(x)
+#define UNINSTRUMENTED_FREE(x) emscripten_builtin_free(x)
+#else
+#define UNINSTRUMENTED_MALLOC(x) malloc(x)
+#define UNINSTRUMENTED_FREE(x) free(x)
+#endif
+
+using namespace perspective::server;
+
+#pragma pack(push, 1)
+struct EncodedApiResp {
+    void* data;
+    std::uint32_t size;
+    std::uint32_t client_id;
+};
+
+struct EncodedApiEntries {
+    std::uint32_t size;
+    EncodedApiResp* entries;
+};
+#pragma pack(pop)
+
+void
+encode_api_response(
+    const ProtoServerResp<std::string>& msg, EncodedApiResp* encoded
+) {
+    auto* data = static_cast<char*>(UNINSTRUMENTED_MALLOC(msg.data.size()));
+    std::copy(msg.data.begin(), msg.data.end(), data);
+
+    encoded->data = data;
+    encoded->size = msg.data.size();
+    encoded->client_id = msg.client_id;
+}
+
+EncodedApiEntries*
+encode_api_responses(const std::vector<ProtoServerResp<std::string>>& msgs) {
+    auto* encoded = static_cast<EncodedApiEntries*>(
+        UNINSTRUMENTED_MALLOC(sizeof(EncodedApiEntries))
+    );
+    encoded->entries = static_cast<EncodedApiResp*>(
+        UNINSTRUMENTED_MALLOC(sizeof(EncodedApiResp) * msgs.size())
+    );
+
+    encoded->size = msgs.size();
+    auto* encoded_mem = encoded->entries;
+    for (int i = 0; i < msgs.size(); i++) {
+        encode_api_response(msgs[i], &encoded_mem[i]);
+    }
+
+    return encoded;
+}
+
+extern "C" {
+
+PERSPECTIVE_EXPORT
+ProtoServer*
+psp_new_server(bool realtime_mode) {
+    return new ProtoServer(realtime_mode);
+}
+
+PERSPECTIVE_EXPORT
+EncodedApiEntries*
+psp_handle_request(
+    ProtoServer* server,
+    std::uint32_t client_id,
+    char* msg_ptr,
+    std::size_t msg_len
+) {
+    std::string msg(msg_ptr, msg_len);
+    auto msgs = server->handle_request(client_id, msg);
+    return encode_api_responses(msgs);
+}
+
+PERSPECTIVE_EXPORT
+EncodedApiEntries*
+psp_poll(ProtoServer* server) {
+    auto responses = server->poll();
+    return encode_api_responses(responses);
+}
+
+PERSPECTIVE_EXPORT
+std::size_t
+psp_residency_prepare(ProtoServer* server) {
+    return server->residency_prepare();
+}
+
+PERSPECTIVE_EXPORT
+const char*
+psp_residency_victim_fname(ProtoServer* server, std::size_t i) {
+    return server->residency_victim_fname(i);
+}
+
+PERSPECTIVE_EXPORT
+void
+psp_residency_commit(ProtoServer* server) {
+    server->residency_commit();
+}
+
+PERSPECTIVE_EXPORT
+std::uint32_t
+psp_new_session(ProtoServer* server) {
+    return server->new_session();
+}
+
+PERSPECTIVE_EXPORT
+void
+psp_close_session(ProtoServer* server, std::uint32_t client_id) {
+    server->close_session(client_id);
+}
+
+PERSPECTIVE_EXPORT
+std::size_t
+psp_alloc(std::size_t size) {
+    // We use this to allocate stack traces for instrumentation with heap
+    // profiling builds.
+    auto* mem = (char*)UNINSTRUMENTED_MALLOC(size);
+    return (size_t)mem;
+}
+
+PERSPECTIVE_EXPORT
+void
+psp_free(void* ptr) {
+    UNINSTRUMENTED_FREE(ptr);
+}
+
+PERSPECTIVE_EXPORT
+void
+psp_delete_server(void* proto_server) {
+    auto* server = (ProtoServer*)proto_server;
+    delete server;
+}
+
+PERSPECTIVE_EXPORT
+bool
+psp_is_memory64() {
+#if UINTPTR_MAX == 0xffffffff
+    return false;
+#elif UINTPTR_MAX == 0xffffffffffffffff
+    return true;
+#endif
+}
+
+PERSPECTIVE_EXPORT
+std::int32_t
+psp_num_cpus() {
+    return arrow::GetCpuThreadPoolCapacity();
+}
+
+PERSPECTIVE_EXPORT
+void
+psp_set_num_cpus(std::int32_t num_cpus) {
+    PSP_CHECK_ARROW_STATUS(arrow::SetCpuThreadPoolCapacity(num_cpus));
+
+}
+
+} // end extern "C"

rust/perspective-server/cpp/perspective/src/cpp/comparators.cpp

@@ -0,0 +1,16 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/comparators.h>
+
+namespace perspective {} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/compat.cpp

@@ -0,0 +1,38 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/compat.h>
+#include <perspective/raii.h>
+#include <perspective/utils.h>
+
+#ifndef WIN32
+#include <sys/mman.h>
+#include <fcntl.h>
+#endif
+
+namespace perspective {
+
+std::pair<std::uint32_t, std::uint32_t>
+file_size_pair(t_handle h) {
+    t_uindex sz = file_size(h);
+    return std::pair<std::uint32_t, std::uint32_t>(upper32(sz), lower32(sz));
+}
+
+t_rfmapping::t_rfmapping() = default;
+
+t_rfmapping::t_rfmapping(t_handle fd, void* base, t_uindex size) :
+    m_fd(fd),
+    m_base(base),
+    m_size(size) {}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/compat_impl_linux.cpp

@@ -0,0 +1,231 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+
+#ifndef PSP_ENABLE_WASM
+#ifdef __linux__
+#include <perspective/compat.h>
+#include <perspective/raii.h>
+#include <perspective/raw_types.h>
+#include <perspective/utils.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdio.h>
+
+namespace perspective {
+static void map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+);
+
+t_uindex
+file_size(t_handle h) {
+    struct stat st;
+    t_index rcode = fstat(h, &st);
+    PSP_VERBOSE_ASSERT(rcode == 0, "Error in stat");
+    return st.st_size;
+}
+
+void
+close_file(t_handle h) {
+    t_index rcode = close(h);
+    PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
+}
+
+void
+flush_mapping(void* base, t_uindex len) {
+    t_index rcode = msync(base, len, MS_SYNC);
+    PSP_VERBOSE_ASSERT(rcode != -1, "Error in msync");
+}
+
+t_rfmapping::~t_rfmapping() {
+    t_index rcode = munmap(m_base, m_size);
+    PSP_VERBOSE_ASSERT(rcode == 0, "munmap failed.");
+
+    rcode = close(m_fd);
+    PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
+}
+
+static void
+map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+) {
+    t_file_handle fh(open(fname.c_str(), fflag, fmode));
+
+    PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
+
+    if (is_read) {
+        size = file_size(fh.value());
+    } else {
+        t_index rcode = ftruncate(fh.value(), size);
+        PSP_VERBOSE_ASSERT(rcode >= 0, "ftruncate failed.");
+    }
+
+    void* ptr = mmap(0, size, mprot, mflag, fh.value(), 0);
+
+    PSP_VERBOSE_ASSERT(ptr != MAP_FAILED, "error in mmap");
+
+    t_handle fd = fh.value();
+    fh.release();
+
+    out.m_fd = fd;
+    out.m_base = ptr;
+    out.m_size = size;
+}
+
+void
+map_file_read(const std::string& fname, t_rfmapping& out) {
+    map_file_internal_(
+        fname,
+        O_RDONLY,
+        S_IRUSR,
+        0, // no disposition
+        PROT_READ,
+        MAP_SHARED,
+        true,
+        0,
+        out
+    );
+}
+
+void
+map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
+    return map_file_internal_(
+        fname,
+        O_RDWR | O_TRUNC | O_CREAT,
+        S_IRUSR | S_IWUSR,
+        0, // no disposition
+        PROT_WRITE | PROT_READ,
+        MAP_SHARED,
+        false,
+        size,
+        out
+    );
+}
+
+std::int64_t
+psp_curtime() {
+    struct timespec t;
+    std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
+    PSP_VERBOSE_ASSERT(rcode == 0, "Failure in clock_gettime");
+    std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
+    return ns;
+}
+
+std::int64_t
+get_page_size() {
+    static std::int64_t pgsize = getpagesize();
+    return pgsize;
+}
+
+std::int64_t
+psp_curmem() {
+    static double multiplier = getpagesize() / 1024000.;
+
+    struct t_statm {
+        long int m_size, m_resident, m_share, m_text, m_lib, m_data, m_dt;
+    };
+
+    t_statm result;
+
+    const char* statm_path = "/proc/self/statm";
+
+    FILE* f = fopen(statm_path, "r");
+    if (!f) {
+        perror(statm_path);
+        abort();
+    }
+
+    PSP_VERBOSE_ASSERT(
+        fscanf(
+            f,
+            "%ld %ld %ld %ld %ld %ld %ld",
+            &result.m_size,
+            &result.m_resident,
+            &result.m_share,
+            &result.m_text,
+            &result.m_lib,
+            &result.m_data,
+            &result.m_dt
+        ) == 7,
+        "Failed to read memory size"
+    );
+    fclose(f);
+    return result.m_resident * multiplier;
+}
+
+void
+set_thread_name(std::thread& thr, const std::string& name) {
+#ifdef PSP_PARALLEL_FOR
+    auto handle = thr.native_handle();
+    pthread_setname_np(handle, name.c_str());
+#endif
+}
+
+void
+set_thread_name(const std::string& name) {
+    // prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+rmfile(const std::string& fname) {
+    unlink(fname.c_str());
+}
+
+void
+launch_proc(const std::string& cmdline) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+std::string
+cwd() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return "";
+}
+
+void*
+psp_dbg_malloc(size_t size) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return nullptr;
+}
+
+void
+psp_dbg_free(void* mem) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+#endif
+#endif

rust/perspective-server/cpp/perspective/src/cpp/compat_impl_osx.cpp

@@ -0,0 +1,234 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifndef PSP_ENABLE_WASM
+#ifdef __APPLE__
+
+#include <perspective/first.h>
+#include <perspective/compat.h>
+#include <perspective/raii.h>
+#include <perspective/raw_types.h>
+#include <perspective/utils.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <cstdio>
+
+namespace perspective {
+static void map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+);
+
+t_uindex
+file_size(t_handle h) {
+    struct stat st;
+    t_index rcode = fstat(h, &st);
+    PSP_VERBOSE_ASSERT(rcode, == 0, "Error in stat");
+    return st.st_size;
+}
+
+void
+close_file(t_handle h) {
+    t_index rcode = close(h);
+    PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
+}
+
+void
+flush_mapping(void* base, t_uindex len) {
+    t_index rcode = msync(base, len, MS_SYNC);
+    PSP_VERBOSE_ASSERT(rcode, != -1, "Error in msync");
+}
+
+t_rfmapping::~t_rfmapping() {
+    t_index rcode = munmap(m_base, m_size);
+    PSP_VERBOSE_ASSERT(rcode, == 0, "munmap failed.");
+
+    rcode = close(m_fd);
+    PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
+}
+
+static void
+map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+) {
+    t_file_handle fh(open(fname.c_str(), fflag, fmode));
+
+    PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
+
+    if (is_read) {
+        size = file_size(fh.value());
+    } else {
+        t_index rcode = ftruncate(fh.value(), size);
+        PSP_VERBOSE_ASSERT(rcode, >= 0, "ftruncate failed.");
+    }
+
+    void* ptr = mmap(nullptr, size, mprot, mflag, fh.value(), 0);
+
+    PSP_VERBOSE_ASSERT(ptr, != MAP_FAILED, "error in mmap");
+
+    t_handle fd = fh.value();
+    fh.release();
+
+    out.m_fd = fd;
+    out.m_base = ptr;
+    out.m_size = size;
+}
+
+void
+map_file_read(const std::string& fname, t_rfmapping& out) {
+    map_file_internal_(
+        fname,
+        O_RDONLY,
+        S_IRUSR,
+        0, // no disposition
+        PROT_READ,
+        MAP_SHARED,
+        true,
+        0,
+        out
+    );
+}
+
+void
+map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
+    return map_file_internal_(
+        fname,
+        O_RDWR | O_TRUNC | O_CREAT,
+        S_IRUSR | S_IWUSR,
+        0, // no disposition
+        PROT_WRITE | PROT_READ,
+        MAP_SHARED,
+        false,
+        size,
+        out
+    );
+}
+
+std::int64_t
+psp_curtime() {
+    // CLOCK_MONOTONIC and clock_gettime are not implemented in OSX < 10.12,
+    // and it breaks builds on conda-forge. Because this method is only called
+    // in logging and trace functions that are not called in the codebase,
+    // deprecate this method for OSX and return 0.
+
+    // struct timespec t;
+    // std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
+    // PSP_VERBOSE_ASSERT(rcode, == 0, "Failure in clock_gettime");
+    // std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
+    return 0;
+}
+
+std::int64_t
+get_page_size() {
+    static std::int64_t pgsize = getpagesize();
+    return pgsize;
+}
+
+std::int64_t
+psp_curmem() {
+    static double multiplier = getpagesize() / 1024000.;
+
+    struct t_statm {
+        long int m_size, m_resident, m_share, m_text, m_lib, m_data, m_dt;
+    };
+
+    t_statm result;
+
+    const char* statm_path = "/proc/self/statm";
+
+    FILE* f = fopen(statm_path, "r");
+    if (f == nullptr) {
+        perror(statm_path);
+        abort();
+    }
+
+    PSP_VERBOSE_ASSERT(
+        fscanf(
+            f,
+            "%ld %ld %ld %ld %ld %ld %ld",
+            &result.m_size,
+            &result.m_resident,
+            &result.m_share,
+            &result.m_text,
+            &result.m_lib,
+            &result.m_data,
+            &result.m_dt
+        ) == 7,
+        "Failed to read memory size"
+    );
+    fclose(f);
+    return result.m_resident * multiplier;
+}
+
+void
+set_thread_name(std::thread& thr, const std::string& name) {
+#ifdef PSP_PARALLEL_FOR
+    pthread_setname_np(name.c_str());
+#endif
+}
+
+void
+set_thread_name(const std::string& name) {
+    // prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+rmfile(const std::string& fname) {
+    unlink(fname.c_str());
+}
+
+void
+launch_proc(const std::string& cmdline) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+std::string
+cwd() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return "";
+}
+
+void*
+psp_dbg_malloc(size_t size) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return nullptr;
+}
+
+void
+psp_dbg_free(void* mem) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+#endif
+#endif

rust/perspective-server/cpp/perspective/src/cpp/compat_impl_wasm.cpp

@@ -0,0 +1,168 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef PSP_ENABLE_WASM
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/compat.h>
+#include <perspective/opfs.h>
+#include <perspective/raii.h>
+#include <perspective/raw_types.h>
+#include <perspective/utils.h>
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/mman.h>
+#include <sys/resource.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdio.h>
+
+namespace perspective {
+static void map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+);
+
+t_uindex
+file_size(t_handle h) {
+    struct stat st {};
+    if (fstat(h, &st) != 0) {
+        return 0;
+    }
+    return static_cast<t_uindex>(st.st_size);
+}
+
+void
+close_file(t_handle h) {
+    // On WASM (NO_FILESYSTEM) disk stores are backed by the OPFS/node-fs bridge
+    // keyed by filename, not real fds — `create_file()` returns a sentinel (1).
+    // Closing it issued `fd_close(1)` (stdout) on every disk-column free; never
+    // touch the std streams (0/1/2). Real fds (none under NO_FILESYSTEM) close.
+    if (h > 2) {
+        close(h);
+    }
+}
+
+void
+flush_mapping(void* base, t_uindex len) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+t_rfmapping::~t_rfmapping() {}
+
+static void
+map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+map_file_read(const std::string& fname, t_rfmapping& out) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+std::int64_t
+psp_curtime() {
+    // CLOCK_MONOTONIC and clock_gettime are not implemented in OSX < 10.12,
+    // and it breaks builds on conda-forge. Because this method is only called
+    // in logging and trace functions that are not called in the codebase,
+    // deprecate this method for OSX and return 0.
+
+    // struct timespec t;
+    // std::int32_t rcode = clock_gettime(CLOCK_MONOTONIC, &t);
+    // PSP_VERBOSE_ASSERT(rcode, == 0, "Failure in clock_gettime");
+    // std::int64_t ns = t.tv_nsec + t.tv_sec * 1000000000;
+    return 0;
+}
+
+std::int64_t
+get_page_size() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+std::int64_t
+psp_curmem() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+set_thread_name(std::thread& thr, const std::string& name) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+set_thread_name(const std::string& name) {
+    // prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+rmfile(const std::string& fname) {
+#ifdef PSP_HAS_OPFS_BRIDGE
+    // Close the kept-open OPFS sync access handle and delete the backing file.
+    // (`unlink` is a no-op under `NO_FILESYSTEM`, and would leak the open handle
+    // the residency bridge holds for this file.)
+    psp_opfs_remove(fname.c_str());
+#else
+    unlink(fname.c_str());
+#endif
+}
+
+void
+launch_proc(const std::string& cmdline) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+std::string
+cwd() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return "";
+}
+
+void*
+psp_dbg_malloc(size_t size) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return nullptr;
+}
+
+void
+psp_dbg_free(void* mem) {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/compat_impl_win.cpp

@@ -0,0 +1,312 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifndef PSP_ENABLE_WASM
+#ifdef WIN32
+#include <perspective/portable.h>
+#include <perspective/first.h>
+#include <perspective/raw_types.h>
+#include <perspective/compat.h>
+#include <perspective/raii.h>
+#include <perspective/utils.h>
+#include <cstdio>
+#include <psapi.h>
+#include <cstdint>
+
+namespace perspective {
+
+static void map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+);
+
+t_uindex
+file_size(t_handle h) {
+    LARGE_INTEGER sz;
+    BOOL rb = GetFileSizeEx(h, &sz);
+    PSP_VERBOSE_ASSERT(rb != 0, "Error getting filesize.");
+    return sz.QuadPart;
+}
+
+void
+close_file(t_handle h) {
+    BOOL rb = CloseHandle(h);
+    PSP_VERBOSE_ASSERT(rb != 0, "Error closing file");
+}
+
+void
+flush_mapping(void* base, t_uindex len) {
+    BOOL rb = FlushViewOfFile(base, size_t(len));
+    PSP_VERBOSE_ASSERT(rb != 0, "Error flushing view");
+}
+
+t_rfmapping::~t_rfmapping() {
+    BOOL rb = UnmapViewOfFile(m_base);
+    PSP_VERBOSE_ASSERT(rb != 0, "Error unmapping view");
+
+    close_file(m_fd);
+}
+
+static void
+map_file_internal_(
+    const std::string& fname,
+    t_fflag fflag,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflag,
+    bool is_read,
+    t_uindex size,
+    t_rfmapping& out
+) {
+    t_file_handle fh(CreateFile(
+        fname.c_str(),
+        fflag,
+        FILE_SHARE_READ,
+        0, // security
+        creation_disposition,
+        FILE_ATTRIBUTE_NORMAL,
+        0 // template file
+    ));
+
+    PSP_VERBOSE_ASSERT(fh.valid(), "Error opening file");
+
+    if (is_read) {
+        size = file_size(fh.value());
+    } else {
+        LARGE_INTEGER sz;
+        sz.QuadPart = size;
+
+        auto rb = SetFilePointerEx(fh.value(), sz, 0, FILE_BEGIN);
+        PSP_VERBOSE_ASSERT(rb, "Error setting fpointer");
+
+        rb = SetEndOfFile(fh.value());
+        PSP_VERBOSE_ASSERT(rb, "Error setting eof");
+    }
+
+    t_handle m = CreateFileMapping(
+        fh.value(),
+        0, // default security
+        mprot,
+        upper32(size),
+        lower32(size),
+        0 // anonymous mapping
+    );
+
+    PSP_VERBOSE_ASSERT(m != 0, "Error creating filemapping");
+
+    void* ptr = MapViewOfFile(
+        m,
+        mflag,
+        0, // 0 offset
+        0, // 0 offset
+        0  // entire file
+    );
+
+    PSP_VERBOSE_ASSERT(ptr != 0, "Error mapping file");
+
+    // Handle is safe to close once view is
+    // created for it
+    CloseHandle(m);
+
+    t_handle fd = fh.value();
+    fh.release();
+
+    out.m_fd = fd;
+    out.m_base = ptr;
+    out.m_size = size;
+}
+
+void
+map_file_read(const std::string& fname, t_rfmapping& out) {
+    map_file_internal_(
+        fname,
+        GENERIC_READ,
+        0, // unused
+        OPEN_EXISTING,
+        PAGE_READONLY,
+        FILE_MAP_READ,
+        true,
+        0,
+        out
+    );
+}
+
+void
+map_file_write(const std::string& fname, t_uindex size, t_rfmapping& out) {
+    map_file_internal_(
+        fname,
+        GENERIC_READ | GENERIC_WRITE,
+        0, // unused
+        CREATE_ALWAYS,
+        PAGE_READWRITE,
+        FILE_MAP_WRITE,
+        false,
+        static_cast<size_t>(size),
+        out
+    );
+}
+
+int64_t
+psp_curtime() {
+    return GetTickCount() * static_cast<int64_t>(1000000);
+}
+
+int64_t
+psp_curmem() {
+    PROCESS_MEMORY_COUNTERS mem;
+    GetProcessMemoryInfo(GetCurrentProcess(), &mem, sizeof(mem));
+    return mem.WorkingSetSize / 1024;
+}
+
+#pragma pack(push, 8)
+typedef struct t_win_thrstruct {
+    DWORD dwType;     // Must be 0x1000.
+    LPCSTR szName;    // Pointer to name (in user addr space).
+    DWORD dwThreadID; // Thread ID (-1=caller thread).
+    DWORD dwFlags;    // Reserved for future use, must be zero.
+} THREADNAME_INFO;
+#pragma pack(pop)
+
+static void
+set_thread_name_win(uint32_t thrid, const std::string& name) {
+    const DWORD MS_VC_EXCEPTION = 0x406D1388;
+    t_win_thrstruct thrstruct;
+    thrstruct.dwType = 0x1000;
+    thrstruct.szName = name.c_str();
+    thrstruct.dwThreadID = thrid;
+    thrstruct.dwFlags = 0;
+
+    SUPPRESS_WARNINGS_VC(6320 6322)
+    __try {
+        RaiseException(
+            MS_VC_EXCEPTION,
+            0,
+            sizeof(thrstruct) / sizeof(ULONG_PTR),
+            (ULONG_PTR*)&thrstruct
+        );
+    } __except (EXCEPTION_EXECUTE_HANDLER) {}
+    RESTORE_WARNINGS_VC()
+}
+
+void
+set_thread_name(std::thread& thr, const std::string& name) {
+    auto thrid = ::GetThreadId(static_cast<HANDLE>(thr.native_handle()));
+    set_thread_name_win(thrid, name);
+}
+
+void
+set_thread_name(const std::string& name) {
+    set_thread_name_win(GetCurrentThreadId(), name);
+}
+
+void
+rmfile(const std::string& fname) {
+    DeleteFile(fname.c_str());
+}
+
+void
+launch_proc(const std::string& cmdline) {
+    STARTUPINFO si;
+    PROCESS_INFORMATION pi;
+
+    ZeroMemory(&si, sizeof(si));
+    si.cb = sizeof(si);
+    ZeroMemory(&pi, sizeof(pi));
+
+    if (!CreateProcess(
+            NULL, // No module name (use command line)
+            const_cast<char*>(cmdline.c_str()), // Command line
+            NULL,  // Process handle not inheritable
+            NULL,  // Thread handle not inheritable
+            FALSE, // Set handle inheritance to FALSE
+            0,     // No creation flags
+            NULL,  // Use parent's environment block
+            NULL,  // Use parent's starting directory
+            &si,   // Pointer to STARTUPINFO structure
+            &pi
+        ) // Pointer to PROCESS_INFORMATION structure
+    ) {
+        std::cout << "CreateProcess failed => " << GetLastError() << "\n";
+        return;
+    }
+
+    // Wait until child process exits.
+    WaitForSingleObject(pi.hProcess, INFINITE);
+
+    // Close process and thread handles.
+    CloseHandle(pi.hProcess);
+    CloseHandle(pi.hThread);
+}
+
+std::string
+cwd() {
+    char path[FILENAME_MAX];
+    auto rc = GetCurrentDirectory(FILENAME_MAX, path);
+    PSP_VERBOSE_ASSERT(rc, "Error in cwd");
+    return std::string(path);
+}
+
+int64_t
+get_page_size() {
+    SYSTEM_INFO info;
+    GetSystemInfo(&info);
+    return info.dwPageSize;
+}
+
+void*
+psp_dbg_malloc(size_t size) {
+    SYSTEM_INFO sys_info;
+    GetSystemInfo(&sys_info);
+    auto page = 2 * sys_info.dwPageSize;
+    assert((page & (static_cast<size_t>(page) - 1)) == 0);
+    auto rounded_size =
+        (size + static_cast<size_t>(page) - 1) & (-static_cast<size_t>(page));
+    BYTE* start = (BYTE*)VirtualAlloc(
+        NULL, rounded_size + page, MEM_COMMIT, PAGE_READWRITE
+    );
+    DWORD old_protect;
+    BOOL res =
+        VirtualProtect(start + rounded_size, page, PAGE_NOACCESS, &old_protect);
+    assert(res);
+    UNREFERENCED_PARAMETER(res);
+    return start + (rounded_size - size);
+}
+
+void
+psp_dbg_free(void* mem) {
+    VirtualFree(mem, 0, MEM_RELEASE);
+}
+
+void*
+psp_page_aligned_malloc(int64_t size) {
+    return _aligned_malloc(
+        static_cast<size_t>(size), static_cast<size_t>(get_page_size())
+    );
+}
+
+void
+psp_page_aligned_free(void* mem) {
+    _aligned_free(mem);
+}
+
+} // end namespace perspective
+
+#endif
+#endif

rust/perspective-server/cpp/perspective/src/cpp/computed_expression.cpp

@@ -0,0 +1,687 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/computed_expression.h>
+
+#include <utility>
+
+namespace perspective {
+
+computed_function::bucket t_computed_expression_parser::BUCKET_FN =
+    computed_function::bucket();
+
+computed_function::hour_of_day t_computed_expression_parser::HOUR_OF_DAY_FN =
+    computed_function::hour_of_day();
+
+computed_function::percent_of t_computed_expression_parser::PERCENT_OF_FN =
+    computed_function::percent_of();
+
+computed_function::inrange_fn t_computed_expression_parser::INRANGE_FN =
+    computed_function::inrange_fn();
+
+computed_function::min_fn t_computed_expression_parser::MIN_FN =
+    computed_function::min_fn();
+
+computed_function::max_fn t_computed_expression_parser::MAX_FN =
+    computed_function::max_fn();
+
+computed_function::coalesce t_computed_expression_parser::COALESCE_FN =
+    computed_function::coalesce();
+
+computed_function::contains t_computed_expression_parser::CONTAINS_FN =
+    computed_function::contains();
+
+computed_function::diff3 t_computed_expression_parser::diff3 =
+    computed_function::diff3();
+
+computed_function::norm3 t_computed_expression_parser::norm3 =
+    computed_function::norm3();
+
+computed_function::cross_product3 t_computed_expression_parser::cross_product3 =
+    computed_function::cross_product3();
+
+computed_function::dot_product3 t_computed_expression_parser::dot_product3 =
+    computed_function::dot_product3();
+
+computed_function::length t_computed_expression_parser::LENGTH_FN =
+    computed_function::length();
+
+computed_function::is_null t_computed_expression_parser::IS_NULL_FN =
+    computed_function::is_null();
+
+computed_function::is_not_null t_computed_expression_parser::IS_NOT_NULL_FN =
+    computed_function::is_not_null();
+
+computed_function::to_integer t_computed_expression_parser::TO_INTEGER_FN =
+    computed_function::to_integer();
+
+computed_function::to_float t_computed_expression_parser::TO_FLOAT_FN =
+    computed_function::to_float();
+
+computed_function::to_boolean t_computed_expression_parser::TO_BOOLEAN_FN =
+    computed_function::to_boolean();
+
+computed_function::make_date t_computed_expression_parser::MAKE_DATE_FN =
+    computed_function::make_date();
+
+computed_function::make_datetime
+    t_computed_expression_parser::MAKE_DATETIME_FN =
+        computed_function::make_datetime();
+
+computed_function::random t_computed_expression_parser::RANDOM_FN =
+    computed_function::random();
+
+t_tscalar t_computed_expression_parser::TRUE_SCALAR = mktscalar(true);
+
+t_tscalar t_computed_expression_parser::FALSE_SCALAR = mktscalar(false);
+
+/******************************************************************************
+ *
+ * t_computed_expression
+ */
+
+t_computed_expression::t_computed_expression(
+    std::string expression_alias,
+    std::string expression_string,
+    std::string parsed_expression_string,
+    const std::vector<std::pair<std::string, std::string>>& column_ids,
+    t_dtype dtype
+) :
+    m_expression_alias(std::move(expression_alias)),
+    m_expression_string(std::move(expression_string)),
+    m_parsed_expression_string(std::move(parsed_expression_string)),
+    m_column_ids(column_ids),
+    m_dtype(dtype) {}
+
+struct t_computed_expression_cache {
+    t_computed_expression_cache() = default;
+    t_computed_expression_cache(const t_computed_expression_cache&) = delete;
+    t_computed_expression_cache& operator=(const t_computed_expression_cache&) =
+        delete;
+    t_computed_expression_cache(t_computed_expression_cache&&) = delete;
+    t_computed_expression_cache& operator=(t_computed_expression_cache&&) =
+        delete;
+
+    std::shared_ptr<t_data_table> m_current_source_table;
+    t_uindex m_row_idx{0};
+    std::vector<std::pair<std::string, t_tscalar>> m_values;
+    std::vector<std::shared_ptr<t_column>> m_columns;
+    std::vector<t_dtype> m_input_dtypes;
+    exprtk::symbol_table<t_tscalar> m_sym_table;
+    exprtk::expression<t_tscalar> m_expr;
+    std::unique_ptr<t_computed_function_store> m_function_store;
+};
+
+t_computed_expression::~t_computed_expression() = default;
+
+void
+t_computed_expression::compute(
+    const std::shared_ptr<t_data_table>& source_table,
+    const t_gstate::t_mapping& pkey_map,
+    const std::shared_ptr<t_data_table>& destination_table,
+    t_expression_vocab& vocab,
+    t_regex_mapping& regex_mapping
+) const {
+    auto num_input_columns = m_column_ids.size();
+
+    bool needs_build = !m_cache;
+    if (!needs_build) {
+        for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+            const std::string& column_name = m_column_ids[cidx].second;
+            if (source_table->get_column(column_name)->get_dtype()
+                != m_cache->m_input_dtypes[cidx]) {
+                needs_build = true;
+                break;
+            }
+        }
+    }
+
+    if (needs_build) {
+        m_cache = std::make_unique<t_computed_expression_cache>();
+        auto& cache = *m_cache;
+
+        cache.m_current_source_table = source_table;
+        cache.m_sym_table.add_constants(); // pi, infinity, etc.
+
+        // is_type_validator = false: we are computing values, not type-checking.
+        cache.m_function_store = std::make_unique<t_computed_function_store>(
+            vocab,
+            regex_mapping,
+            false,
+            cache.m_current_source_table,
+            pkey_map,
+            cache.m_row_idx
+        );
+        cache.m_function_store->register_computed_functions(cache.m_sym_table);
+
+        // Size exactly once; the symbol table binds a T* into each
+        // m_values[cidx].second, so this storage must never be reallocated.
+        cache.m_values.resize(num_input_columns);
+        cache.m_columns.resize(num_input_columns);
+        cache.m_input_dtypes.resize(num_input_columns);
+
+        for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+            const std::string& column_id = m_column_ids[cidx].first;
+            const std::string& column_name = m_column_ids[cidx].second;
+            t_dtype dtype = source_table->get_column(column_name)->get_dtype();
+
+            cache.m_input_dtypes[cidx] = dtype;
+
+            t_tscalar rval;
+            rval.clear();
+            rval.m_type = dtype;
+
+            cache.m_values[cidx] =
+                std::pair<std::string, t_tscalar>(column_id, rval);
+            cache.m_sym_table.add_variable(
+                column_id, cache.m_values[cidx].second
+            );
+        }
+
+        cache.m_expr.register_symbol_table(cache.m_sym_table);
+
+        // Load-bearing for compile-once: exprtk constant-folds an all-literal
+        // function call (e.g. intern('x'), concat('a','b')) to a literal node at
+        // compile time ONLY when the function reports no side effects. Our
+        // functions inherit the igeneric_function/ifunction default
+        // has_side_effects() == true and never call disable_has_side_effects(),
+        // so such calls are not folded and re-evaluate on every value(). If a
+        // future exprtk bump flips that default, the first call's result would
+        // be frozen into the cached AST -- revisit this cache if so.
+        if (!m_computed_expression_parser.m_parser->compile(
+                m_parsed_expression_string, cache.m_expr
+            )) {
+            std::stringstream ss;
+            ss << "[t_computed_expression::compute] Failed to parse "
+                  "expression: `"
+               << m_parsed_expression_string << "`, failed with error: "
+               << m_computed_expression_parser.m_parser->error() << '\n';
+
+            PSP_COMPLAIN_AND_ABORT(ss.str());
+        }
+    }
+
+    auto& cache = *m_cache;
+
+    // Re-point the per-call inputs the compiled function objects read through
+    // (the source table differs across the master/flattened/delta/prev/current
+    // tables within a single update), and re-fetch this call's source columns.
+    cache.m_current_source_table = source_table;
+    for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+        cache.m_columns[cidx] =
+            source_table->get_column(m_column_ids[cidx].second);
+    }
+
+    // create or get output column using m_expression_alias
+    auto output_column =
+        destination_table->add_column_sptr(m_expression_alias, m_dtype, true);
+    auto num_rows = source_table->size();
+    output_column->reserve(num_rows);
+
+    for (t_uindex ridx = 0; ridx < num_rows; ++ridx) {
+        for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+            cache.m_values[cidx].second.set(
+                cache.m_columns[cidx]->get_scalar(ridx)
+            );
+        }
+        cache.m_row_idx = ridx;
+
+        t_tscalar value = cache.m_expr.value();
+
+        if (!value.is_valid() || value.is_none()) {
+            output_column->clear(ridx);
+            continue;
+        }
+
+        output_column->set_scalar(ridx, value);
+    }
+
+    // order()'s accumulator must still be reset at the end of every call.
+    cache.m_function_store->clear_computed_function_state();
+};
+
+const std::string&
+t_computed_expression::get_expression_alias() const {
+    return m_expression_alias;
+}
+
+const std::string&
+t_computed_expression::get_expression_string() const {
+    return m_expression_string;
+}
+
+const std::string&
+t_computed_expression::get_parsed_expression_string() const {
+    return m_parsed_expression_string;
+}
+
+const std::vector<std::pair<std::string, std::string>>&
+t_computed_expression::get_column_ids() const {
+    return m_column_ids;
+}
+
+t_dtype
+t_computed_expression::get_dtype() const {
+    return m_dtype;
+}
+
+/******************************************************************************
+ *
+ * t_computed_expression_parser
+ */
+
+t_computed_expression_parser::t_computed_expression_parser() {
+    // Change ExprTk's default compilation options to only check for correctness
+    // of brackets and sequences. ExprTk defaults will replace "true" and
+    // "false" with 1 and 0, which we don't want. Using the tokens "true" and
+    // "false" will raise a syntax error, which is the correct behavior.
+
+    m_parser = std::make_shared<exprtk::parser<t_tscalar>>(
+        exprtk::parser<t_tscalar>::settings_t::e_joiner
+        + exprtk::parser<t_tscalar>::settings_t::e_numeric_check
+        + exprtk::parser<t_tscalar>::settings_t::e_bracket_check
+        + exprtk::parser<t_tscalar>::settings_t::e_sequence_check
+        // exprtk::parser<t_tscalar>::settings_t::e_commutative_check;
+        // exprtk::parser<t_tscalar>::settings_t::e_strength_reduction;
+    );
+
+    m_parser->settings()
+        .disable_control_structure(
+            exprtk::parser<t_tscalar>::settings_store::e_ctrl_repeat_loop
+        )
+        .disable_base_function(
+            exprtk::parser<t_tscalar>::settings_store::e_bf_inrange
+        )
+        .disable_base_function(
+            exprtk::parser<t_tscalar>::settings_store::e_bf_min
+        )
+        .disable_base_function(
+            exprtk::parser<t_tscalar>::settings_store::e_bf_max
+        );
+}
+
+std::shared_ptr<t_computed_expression>
+t_computed_expression_parser::precompute(
+    const std::string& expression_alias,
+    const std::string& expression_string,
+    const std::string& parsed_expression_string,
+    const std::vector<std::pair<std::string, std::string>>& column_ids,
+    const std::shared_ptr<t_data_table>& source_table,
+    const t_gstate::t_mapping& pkey_map,
+    const std::shared_ptr<t_schema>& schema,
+    t_expression_vocab& vocab,
+    t_regex_mapping& regex_mapping
+) const {
+    exprtk::symbol_table<t_tscalar> sym_table;
+    sym_table.add_constants();
+
+    t_uindex row_idx = 0;
+
+    // Create a function store, with is_type_validator set to true as we are
+    // just getting the output types.
+    t_computed_function_store function_store(
+        vocab, regex_mapping, true, source_table, pkey_map, row_idx
+    );
+    function_store.register_computed_functions(sym_table);
+
+    std::vector<t_tscalar> values;
+
+    auto num_input_columns = column_ids.size();
+    values.resize(num_input_columns);
+
+    // Add a new scalar of the input column type to the symtable, which
+    // will use the scalar to validate the output type of the expression.
+    for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+        const std::string& column_id = column_ids[cidx].first;
+        const std::string& column_name = column_ids[cidx].second;
+
+        t_tscalar rval;
+        rval.clear();
+        rval.m_type = schema->get_dtype(column_name);
+
+        // If we are accessing string columns, the scalar CANNOT have a
+        // nullptr as a string - use the empty string that we interned
+        // in the gnode's expression vocab in t_gnode::init() at idx 0
+        if (rval.m_type == DTYPE_STR) {
+            rval.set(vocab.get_empty_string());
+            rval.m_status = STATUS_INVALID;
+        }
+
+        values[cidx] = rval;
+
+        sym_table.add_variable(column_id, values[cidx]);
+    }
+
+    exprtk::expression<t_tscalar> expr_definition;
+    expr_definition.register_symbol_table(sym_table);
+
+    if (!m_parser->compile(parsed_expression_string, expr_definition)) {
+        std::stringstream ss;
+        ss << "[t_computed_expression_parser::precompute] Failed to parse "
+              "expression: `"
+           << parsed_expression_string
+           << "`, failed with error: " << m_parser->error() << '\n';
+        PSP_COMPLAIN_AND_ABORT(ss.str());
+    }
+
+    t_tscalar v = expr_definition.value();
+    function_store.clear_computed_function_state();
+
+    return std::make_shared<t_computed_expression>(
+        expression_alias,
+        expression_string,
+        parsed_expression_string,
+        column_ids,
+        v.get_dtype()
+    );
+}
+
+t_dtype
+t_computed_expression_parser::get_dtype(
+    const std::string& expression_alias,
+    const std::string& expression_string,
+    const std::string& parsed_expression_string,
+    const std::vector<std::pair<std::string, std::string>>& column_ids,
+    const std::shared_ptr<t_data_table>& source_table,
+    const t_gstate::t_mapping& pkey_map,
+    const t_schema& schema,
+    t_expression_error& error,
+    t_expression_vocab& vocab,
+    t_regex_mapping& regex_mapping
+) const {
+    exprtk::symbol_table<t_tscalar> sym_table;
+    sym_table.add_constants();
+
+    std::vector<t_tscalar> values;
+
+    t_uindex row_idx = 0;
+
+    // Create a function store, with is_type_validator set to true as we are
+    // just validating the output types.
+    t_computed_function_store function_store(
+        vocab, regex_mapping, true, source_table, pkey_map, row_idx
+    );
+    function_store.register_computed_functions(sym_table);
+
+    auto num_input_columns = column_ids.size();
+    values.resize(num_input_columns);
+
+    // Add a new scalar of the input column type to the symtable, which
+    // will use the scalar to validate the output type of the expression.
+    for (t_uindex cidx = 0; cidx < num_input_columns; ++cidx) {
+        const std::string& column_id = column_ids[cidx].first;
+        const std::string& column_name = column_ids[cidx].second;
+
+        if (!schema.has_column(column_name)) {
+            error.m_error_message =
+                ("Value Error - Input column \"" + column_name
+                 + "\" does not exist.");
+            error.m_line = 0;
+            error.m_column = 0;
+            return DTYPE_NONE;
+        }
+
+        t_tscalar rval;
+
+        // clear() is important as it sets the status to valid and the
+        // underlying union value to 0, which prevents any reading of
+        // uninitialized values out of the union.
+        rval.clear();
+        rval.m_type = schema.get_dtype(column_name);
+
+        // If we are accessing string columns, the scalar CANNOT have a
+        // nullptr as a string - use the empty string that we interned
+        // in the gnode's expression vocab in t_gnode::init() at idx 0
+        if (rval.m_type == DTYPE_STR) {
+            rval.set(vocab.get_empty_string());
+            rval.m_status = STATUS_INVALID;
+        }
+
+        values[cidx] = rval;
+
+        sym_table.add_variable(column_id, values[cidx]);
+    }
+
+    exprtk::expression<t_tscalar> expr_definition;
+    expr_definition.register_symbol_table(sym_table);
+
+    if (!m_parser->compile(parsed_expression_string, expr_definition)) {
+        // Error count should always be above 0 if there is a compile error -
+        // We simply take the first error and return it.
+        if (m_parser->error_count() > 0) {
+            auto parser_error = m_parser->get_error(0);
+
+            // Given an error object and an expression, `update_error` maps the
+            // error to a line and column number inside the expression.
+            exprtk::parser_error::update_error(
+                parser_error, parsed_expression_string
+            );
+
+            // Take the error message and strip the ExprTk error code
+            std::string error_message(parser_error.diagnostic);
+
+            // strip the Exprtk error codes such as "ERR001 -"
+            error.m_error_message =
+                error_message.substr(error_message.find("- ") + 2);
+
+            error.m_line = parser_error.line_no;
+            error.m_column = parser_error.column_no;
+        } else {
+            // If for whatever reason the error count is at 0, output a generic
+            // parser error so that we report back a valid error object.
+            error.m_error_message = "Parser Error";
+            error.m_line = 0;
+            error.m_column = 0;
+        }
+
+        return DTYPE_NONE;
+    }
+
+    t_tscalar v = expr_definition.value();
+    t_dtype dtype = v.get_dtype();
+
+    function_store.clear_computed_function_state();
+
+    if (v.m_status == STATUS_CLEAR || dtype == DTYPE_NONE) {
+        error.m_error_message =
+            "Type Error - inputs do not resolve to a valid expression.";
+        error.m_line = 0;
+        error.m_column = 0;
+        return DTYPE_NONE;
+    }
+
+    return dtype;
+}
+
+t_validated_expression_map::t_validated_expression_map() = default;
+
+void
+t_validated_expression_map::add_expression(
+    const std::string& expression_alias, const std::string& type_string
+) {
+    // If the expression is in the error map, it should be removed as it is now
+    // valid. This can happen when validating multiple expressions with the same
+    // alias where the first instance is invalid, and the next occurence is
+    // valid, and so on. This maintains the property that an expression will
+    // exist in the schema OR the error map.
+    auto error_iter = m_expression_errors.find(expression_alias);
+
+    if (error_iter != m_expression_errors.end()) {
+        m_expression_errors.erase(error_iter);
+    }
+
+    m_expression_schema[expression_alias] = type_string;
+}
+
+void
+t_validated_expression_map::add_error(
+    const std::string& expression_alias, t_expression_error& error
+) {
+    // If the expression is in the schema, it should be removed as it is now
+    // invalid. This can happen when validating multiple expressions with the
+    // same alias where the first instance is valid, and the next occurence is
+    // invalid, and so on. This maintains the property that an expression will
+    // exist in the schema OR the error map.
+    auto schema_iter = m_expression_schema.find(expression_alias);
+
+    if (schema_iter != m_expression_schema.end()) {
+        m_expression_schema.erase(schema_iter);
+    }
+
+    m_expression_errors[expression_alias] = error;
+}
+
+std::map<std::string, std::string>
+t_validated_expression_map::get_expression_schema() const {
+    return m_expression_schema;
+}
+
+std::map<std::string, t_expression_error>
+t_validated_expression_map::get_expression_errors() const {
+    return m_expression_errors;
+}
+
+t_computed_function_store::t_computed_function_store(
+    t_expression_vocab& vocab,
+    t_regex_mapping& regex_mapping,
+    bool is_type_validator,
+    const std::shared_ptr<t_data_table>& source_table,
+    const t_gstate::t_mapping& pkey_map,
+    t_uindex& row_idx
+) :
+    m_day_of_week_fn(computed_function::day_of_week(vocab, is_type_validator)),
+    m_month_of_year_fn(
+        computed_function::month_of_year(vocab, is_type_validator)
+    ),
+    m_intern_fn(computed_function::intern(vocab, is_type_validator)),
+    m_concat_fn(computed_function::concat(vocab, is_type_validator)),
+    m_order_fn(computed_function::order(is_type_validator)),
+    m_upper_fn(computed_function::upper(vocab, is_type_validator)),
+    m_lower_fn(computed_function::lower(vocab, is_type_validator)),
+    m_to_string_fn(computed_function::to_string(vocab, is_type_validator)),
+    m_match_fn(computed_function::match(regex_mapping)),
+    m_match_all_fn(computed_function::match_all(regex_mapping)),
+    m_search_fn(
+        computed_function::search(vocab, regex_mapping, is_type_validator)
+    ),
+    m_indexof_fn(computed_function::indexof(regex_mapping)),
+    m_substring_fn(computed_function::substring(vocab, is_type_validator)),
+    m_replace_fn(
+        computed_function::replace(vocab, regex_mapping, is_type_validator)
+    ),
+    m_replace_all_fn(
+        computed_function::replace_all(vocab, regex_mapping, is_type_validator)
+    ),
+    m_index_fn(computed_function::index(pkey_map, source_table, row_idx)),
+    m_col_fn(
+        computed_function::col(vocab, is_type_validator, source_table, row_idx)
+    ),
+    m_vlookup_fn(computed_function::vlookup(
+        vocab, is_type_validator, source_table, row_idx
+    )) {}
+
+void
+t_computed_function_store::register_computed_functions(
+    exprtk::symbol_table<t_tscalar>& sym_table
+) {
+    // General/numeric functions
+    sym_table.add_function("bucket", t_computed_expression_parser::BUCKET_FN);
+    sym_table.add_reserved_function(
+        "inrange", t_computed_expression_parser::INRANGE_FN
+    );
+    sym_table.add_reserved_function(
+        "min", t_computed_expression_parser::MIN_FN
+    );
+    sym_table.add_reserved_function(
+        "max", t_computed_expression_parser::MAX_FN
+    );
+    sym_table.add_function(
+        "coalesce", t_computed_expression_parser::COALESCE_FN
+    );
+    sym_table.add_function(
+        "contains", t_computed_expression_parser::CONTAINS_FN
+    );
+    sym_table.add_reserved_function(
+        "diff3", t_computed_expression_parser::diff3
+    );
+    sym_table.add_reserved_function(
+        "norm3", t_computed_expression_parser::norm3
+    );
+    sym_table.add_reserved_function(
+        "cross_product3", t_computed_expression_parser::cross_product3
+    );
+    sym_table.add_reserved_function(
+        "dot_product3", t_computed_expression_parser::dot_product3
+    );
+    sym_table.add_function(
+        "percent_of", t_computed_expression_parser::PERCENT_OF_FN
+    );
+    sym_table.add_function("is_null", t_computed_expression_parser::IS_NULL_FN);
+    sym_table.add_function(
+        "is_not_null", t_computed_expression_parser::IS_NOT_NULL_FN
+    );
+    sym_table.add_function("random", t_computed_expression_parser::RANDOM_FN);
+
+    // Date/datetime functions
+    sym_table.add_function(
+        "hour_of_day", t_computed_expression_parser::HOUR_OF_DAY_FN
+    );
+    sym_table.add_function("day_of_week", m_day_of_week_fn);
+    sym_table.add_function("month_of_year", m_month_of_year_fn);
+    sym_table.add_function("today", computed_function::today);
+    sym_table.add_function("now", computed_function::now);
+
+    // String functions
+    sym_table.add_function("intern", m_intern_fn);
+    sym_table.add_function("concat", m_concat_fn);
+    sym_table.add_function("order", m_order_fn);
+    sym_table.add_function("upper", m_upper_fn);
+    sym_table.add_function("lower", m_lower_fn);
+    sym_table.add_function("length", t_computed_expression_parser::LENGTH_FN);
+
+    // Type conversion functions
+    sym_table.add_function(
+        "integer", t_computed_expression_parser::TO_INTEGER_FN
+    );
+    sym_table.add_function("float", t_computed_expression_parser::TO_FLOAT_FN);
+    sym_table.add_function(
+        "boolean", t_computed_expression_parser::TO_BOOLEAN_FN
+    );
+    sym_table.add_function("date", t_computed_expression_parser::MAKE_DATE_FN);
+    sym_table.add_function(
+        "datetime", t_computed_expression_parser::MAKE_DATETIME_FN
+    );
+    sym_table.add_function("string", m_to_string_fn);
+
+    // Regex functions
+    sym_table.add_function("match", m_match_fn);
+    sym_table.add_function("match_all", m_match_all_fn);
+    sym_table.add_function("search", m_search_fn);
+    sym_table.add_function("indexof", m_indexof_fn);
+    sym_table.add_function("substring", m_substring_fn);
+    sym_table.add_function("replace", m_replace_fn);
+    sym_table.add_function("replace_all", m_replace_all_fn);
+    sym_table.add_function("index", m_index_fn);
+    sym_table.add_function("col", m_col_fn);
+    sym_table.add_function("vlookup", m_vlookup_fn);
+
+    // And scalar constants
+    sym_table.add_constant("True", t_computed_expression_parser::TRUE_SCALAR);
+    sym_table.add_constant("False", t_computed_expression_parser::FALSE_SCALAR);
+}
+
+void
+t_computed_function_store::clear_computed_function_state() {
+    m_order_fn.clear_order_map();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/config.cpp

@@ -0,0 +1,491 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/config.h>
+
+namespace perspective {
+
+// t_ctxunit
+t_config::t_config(const std::vector<std::string>& detail_columns) :
+    t_config(detail_columns, {}, FILTER_OP_AND, {}) {}
+
+// t_ctx0
+t_config::t_config(
+    const std::vector<std::string>& detail_columns,
+    const std::vector<t_fterm>& fterms,
+    t_filter_op combiner,
+    const std::vector<std::shared_ptr<t_computed_expression>>& expressions
+) :
+    m_detail_columns(detail_columns),
+    m_fterms(fterms),
+    m_expressions(expressions),
+    m_combiner(combiner),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    setup(m_detail_columns);
+    m_is_trivial_config = m_row_pivots.empty() && m_col_pivots.empty()
+        && m_sortby.empty() && m_sortspecs.empty() && m_col_sortspecs.empty()
+        && m_detail_columns.empty() && m_fterms.empty()
+        && m_expressions.empty();
+}
+
+// t_ctx1
+t_config::t_config(
+    const std::vector<std::string>& row_pivots,
+    const std::vector<t_aggspec>& aggregates,
+    const std::vector<t_fterm>& fterms,
+    t_filter_op combiner,
+    const std::vector<std::shared_ptr<t_computed_expression>>& expressions
+) :
+    m_aggregates(aggregates),
+    m_fterms(fterms),
+    m_expressions(expressions),
+    m_combiner(combiner),
+    m_is_trivial_config(false),
+    m_totals(TOTALS_BEFORE),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    for (const auto& p : row_pivots) {
+        m_row_pivots.emplace_back(p);
+    }
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+// t_ctx2
+t_config::t_config(
+    const std::vector<std::string>& row_pivots,
+    const std::vector<std::string>& col_pivots,
+    const std::vector<t_aggspec>& aggregates,
+    const t_totals totals,
+    const std::vector<t_fterm>& fterms,
+    t_filter_op combiner,
+    const std::vector<std::shared_ptr<t_computed_expression>>& expressions,
+    bool column_only
+) :
+    m_aggregates(aggregates),
+    m_fterms(fterms),
+    m_expressions(expressions),
+    m_combiner(combiner),
+    m_column_only(column_only),
+    m_is_trivial_config(false),
+    m_totals(totals),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    for (const auto& p : row_pivots) {
+        m_row_pivots.emplace_back(p);
+    }
+    for (const auto& p : col_pivots) {
+        m_col_pivots.emplace_back(p);
+    }
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+// Constructors used for C++ tests
+t_config::t_config(
+    const std::vector<std::string>& row_pivots,
+    const std::vector<std::string>& col_pivots,
+    const std::vector<t_aggspec>& aggregates
+) :
+    t_config(
+        row_pivots, col_pivots, aggregates, TOTALS_HIDDEN, FILTER_OP_AND, {}
+    ) {}
+
+t_config::t_config(
+    const std::vector<std::string>& row_pivots,
+    const std::vector<std::string>& col_pivots,
+    const std::vector<t_aggspec>& aggregates,
+    const t_totals totals,
+    t_filter_op combiner,
+    const std::vector<t_fterm>& fterms
+) :
+    m_aggregates(aggregates),
+    m_fterms(fterms),
+    m_combiner(combiner),
+    m_is_trivial_config(false),
+    m_totals(totals),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    for (const auto& p : row_pivots) {
+        m_row_pivots.emplace_back(p);
+    }
+
+    for (const auto& p : col_pivots) {
+        m_col_pivots.emplace_back(p);
+    }
+
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+t_config::t_config(
+    const std::vector<t_pivot>& row_pivots,
+    const std::vector<t_aggspec>& aggregates
+) :
+    m_row_pivots(row_pivots),
+    m_aggregates(aggregates),
+    m_is_trivial_config(false),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+t_config::t_config(
+    const std::vector<std::string>& row_pivots,
+    const std::vector<t_aggspec>& aggregates
+) :
+    m_aggregates(aggregates),
+    m_combiner(FILTER_OP_AND),
+    m_is_trivial_config(false),
+    m_totals(TOTALS_BEFORE),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    for (const auto& p : row_pivots) {
+        m_row_pivots.emplace_back(p);
+    }
+
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+t_config::t_config(
+    const std::vector<std::string>& row_pivots, const t_aggspec& agg
+) :
+    m_aggregates(std::vector<t_aggspec>{agg}),
+    m_combiner(FILTER_OP_AND),
+    m_is_trivial_config(false),
+    m_totals(TOTALS_BEFORE),
+    m_fmode(FMODE_SIMPLE_CLAUSES) {
+    for (const auto& p : row_pivots) {
+        m_row_pivots.emplace_back(p);
+    }
+
+    setup(
+        m_detail_columns, std::vector<std::string>{}, std::vector<std::string>{}
+    );
+}
+
+t_config::t_config() = default;
+
+void
+t_config::setup(const std::vector<std::string>& detail_columns) {
+    t_index count = 0;
+    for (const auto& detail_column : detail_columns) {
+        m_detail_colmap[detail_column] = count;
+        count++;
+    }
+}
+
+void
+t_config::setup(
+    const std::vector<std::string>& detail_columns,
+    const std::vector<std::string>& sort_pivot,
+    const std::vector<std::string>& sort_pivot_by
+) {
+    t_index count = 0;
+    for (const auto& detail_column : detail_columns) {
+        m_detail_colmap[detail_column] = count;
+        count++;
+    }
+
+    m_has_pkey_agg = false;
+
+    for (const auto& m_aggregate : m_aggregates) {
+        switch (m_aggregate.agg()) {
+            case AGGTYPE_AND:
+            case AGGTYPE_OR:
+            case AGGTYPE_ANY:
+            case AGGTYPE_FIRST:
+            case AGGTYPE_LAST_BY_INDEX:
+            case AGGTYPE_LAST_MINUS_FIRST:
+            case AGGTYPE_HIGH_MINUS_LOW:
+            case AGGTYPE_MEAN:
+            case AGGTYPE_WEIGHTED_MEAN:
+            case AGGTYPE_UNIQUE:
+            case AGGTYPE_Q1:
+            case AGGTYPE_Q3:
+            case AGGTYPE_MEDIAN:
+            case AGGTYPE_JOIN:
+            case AGGTYPE_DOMINANT:
+            case AGGTYPE_PY_AGG:
+            case AGGTYPE_MIN:
+            case AGGTYPE_MAX:
+            case AGGTYPE_MAX_BY:
+            case AGGTYPE_MIN_BY:
+            case AGGTYPE_SUM_NOT_NULL:
+            case AGGTYPE_SUM_ABS:
+            case AGGTYPE_ABS_SUM:
+            case AGGTYPE_GMV:
+            case AGGTYPE_MUL:
+            case AGGTYPE_DISTINCT_COUNT:
+            case AGGTYPE_DISTINCT_LEAF:
+            case AGGTYPE_VARIANCE:
+            case AGGTYPE_STANDARD_DEVIATION:
+                m_has_pkey_agg = true;
+                break;
+            default:
+                break;
+        }
+
+        if (m_has_pkey_agg) {
+            break;
+        }
+    }
+
+    for (t_index idx = 0, loop_end = sort_pivot.size(); idx < loop_end; ++idx) {
+        m_sortby[sort_pivot[idx]] = sort_pivot_by[idx];
+    }
+
+    populate_sortby(m_row_pivots);
+    populate_sortby(m_col_pivots);
+}
+
+bool
+t_config::is_trivial_config() const {
+    return m_is_trivial_config;
+}
+
+void
+t_config::populate_sortby(const std::vector<t_pivot>& pivots) {
+    for (const auto& pivot : pivots) {
+        PSP_VERBOSE_ASSERT(
+            pivot.mode() == PIVOT_MODE_NORMAL,
+            "Only normal pivots supported for now"
+        );
+        const std::string& pstr = pivot.colname();
+        if (m_sortby.find(pstr) == m_sortby.end()) {
+            m_sortby[pstr] = pstr;
+        }
+    }
+}
+
+t_index
+t_config::get_colidx(const std::string& colname) const {
+    auto iter = m_detail_colmap.find(colname);
+    if (iter == m_detail_colmap.end()) {
+        return INVALID_INDEX;
+    }
+    return iter->second;
+}
+
+std::string
+t_config::repr() const {
+    std::stringstream ss;
+    ss << "t_config<" << this << ">";
+    return ss.str();
+}
+
+t_uindex
+t_config::get_num_aggregates() const {
+    return m_aggregates.size();
+}
+
+t_uindex
+t_config::get_num_columns() const {
+    return m_detail_columns.size();
+}
+
+std::string
+t_config::col_at(t_uindex idx) const {
+    if (idx >= m_detail_columns.size()) {
+        return "";
+    }
+    return m_detail_columns[idx];
+}
+
+bool
+t_config::has_pkey_agg() const {
+    return m_has_pkey_agg;
+}
+
+std::string
+t_config::get_totals_string() const {
+    switch (m_totals) {
+        case TOTALS_BEFORE: {
+            return "before";
+        } break;
+        case TOTALS_HIDDEN: {
+            return "hidden";
+        } break;
+        case TOTALS_AFTER: {
+            return "after";
+        } break;
+        default: {
+            return "INVALID_TOTALS";
+        } break;
+    }
+}
+
+std::string
+t_config::get_sort_by(const std::string& pivot) const {
+    std::string rval;
+    auto iter = m_sortby.find(pivot);
+
+    if (iter == m_sortby.end()) {
+        return pivot;
+    }
+    rval = iter->second;
+
+    return rval;
+}
+
+bool
+t_config::validate_colidx(t_index idx) const {
+    return idx >= 0 && idx < static_cast<t_index>(get_num_columns());
+}
+
+std::vector<std::string>
+t_config::get_column_names() const {
+    return m_detail_columns;
+}
+
+t_uindex
+t_config::get_num_rpivots() const {
+    return m_row_pivots.size();
+}
+
+t_uindex
+t_config::get_num_cpivots() const {
+    return m_col_pivots.size();
+}
+
+bool
+t_config::is_column_only() const {
+    return m_column_only;
+}
+
+const std::vector<t_pivot>&
+t_config::get_row_pivots() const {
+    return m_row_pivots;
+}
+
+const std::vector<t_pivot>&
+t_config::get_column_pivots() const {
+    return m_col_pivots;
+}
+
+std::vector<std::pair<std::string, std::string>>
+t_config::get_sortby_pairs() const {
+    std::vector<std::pair<std::string, std::string>> rval(m_sortby.size());
+    t_uindex idx = 0;
+    for (const auto& iter : m_sortby) {
+        rval[idx].first = iter.first;
+        rval[idx].second = iter.second;
+        ++idx;
+    }
+    return rval;
+}
+
+const std::vector<t_sortspec>&
+t_config::get_sortspecs() const {
+    return m_sortspecs;
+}
+
+const std::vector<t_sortspec>&
+t_config::get_col_sortspecs() const {
+    return m_col_sortspecs;
+}
+
+const std::vector<t_aggspec>&
+t_config::get_aggregates() const {
+    return m_aggregates;
+}
+
+bool
+t_config::has_filters() const {
+    switch (m_fmode) {
+        case FMODE_SIMPLE_CLAUSES: {
+            return !m_fterms.empty();
+        } break;
+        default: {
+            return false;
+        }
+    }
+    return false;
+}
+
+const std::vector<t_fterm>&
+t_config::get_fterms() const {
+    return m_fterms;
+}
+
+std::vector<std::shared_ptr<t_computed_expression>>
+t_config::get_expressions() const {
+    return m_expressions;
+}
+
+t_filter_op
+t_config::get_combiner() const {
+    return m_combiner;
+}
+
+t_totals
+t_config::get_totals() const {
+    return m_totals;
+}
+
+std::vector<t_pivot>
+t_config::get_pivots() const {
+    std::vector<t_pivot> rval = m_row_pivots;
+    for (const auto& piv : m_col_pivots) {
+        rval.push_back(piv);
+    }
+    return rval;
+}
+
+std::string
+t_config::get_parent_pkey_column() const {
+    return m_parent_pkey_column;
+}
+
+std::string
+t_config::get_child_pkey_column() const {
+    return m_child_pkey_column;
+}
+
+const std::string&
+t_config::get_grouping_label_column() const {
+    return m_grouping_label_column;
+}
+
+std::string
+t_config::unity_get_column_name(t_uindex idx) const {
+    if (m_aggregates.empty()) {
+        if (idx >= m_detail_columns.size()) {
+            return "";
+        }
+        return m_detail_columns[idx];
+    }
+
+    return m_aggregates[idx % m_aggregates.size()].name();
+}
+
+std::string
+t_config::unity_get_column_display_name(t_uindex idx) const {
+    if (m_aggregates.empty()) {
+        if (idx >= m_detail_columns.size()) {
+            return "";
+        }
+        return m_detail_columns[idx];
+    }
+
+    return m_aggregates[idx % m_aggregates.size()].disp_name();
+}
+
+t_fmode
+t_config::get_fmode() const {
+    return m_fmode;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_base.cpp

@@ -0,0 +1,16 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/context_base.h>
+
+namespace perspective {} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_grouped_pkey.cpp

@@ -0,0 +1,939 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/context_grouped_pkey.h>
+#include <perspective/first.h>
+#include <perspective/get_data_extents.h>
+#include <perspective/extract_aggregate.h>
+#include <perspective/filter.h>
+#include <perspective/sparse_tree.h>
+#include <perspective/tree_context_common.h>
+#include <perspective/sparse_tree_node.h>
+#include <perspective/traversal.h>
+#include <perspective/env_vars.h>
+#include <perspective/filter_utils.h>
+#include <memory>
+#include <queue>
+#include <tuple>
+#include <utility>
+#include <tsl/hopscotch_set.h>
+
+namespace perspective {
+
+t_ctx_grouped_pkey::t_ctx_grouped_pkey() : m_depth(0), m_depth_set(false) {}
+
+t_ctx_grouped_pkey::t_ctx_grouped_pkey(
+    const t_schema& schema, const t_config& config
+) :
+    m_depth(0),
+    m_depth_set(false) {
+    PSP_COMPLAIN_AND_ABORT("Not Implemented");
+}
+
+t_ctx_grouped_pkey::~t_ctx_grouped_pkey() = default;
+
+void
+t_ctx_grouped_pkey::init() {
+    auto pivots = m_config.get_row_pivots();
+    m_tree = std::make_shared<t_stree>(
+        pivots, m_config.get_aggregates(), m_schema, m_config
+    );
+    m_tree->init();
+    m_traversal = std::make_shared<t_traversal>(m_tree);
+
+    // Each context stores its own expression columns in separate
+    // `t_data_table`s so that each context's expressions are isolated
+    // and do not affect other contexts when they are calculated.
+    const auto& expressions = m_config.get_expressions();
+    m_expression_tables = std::make_shared<t_expression_tables>(
+        expressions, m_config.get_backing_store()
+    );
+
+    m_init = true;
+}
+
+std::shared_ptr<t_expression_tables>
+t_ctx_grouped_pkey::get_expression_tables() const {
+    return m_expression_tables;
+}
+
+t_index
+t_ctx_grouped_pkey::get_row_count() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_traversal->size();
+}
+
+t_index
+t_ctx_grouped_pkey::get_column_count() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_config.get_num_columns() + 1;
+}
+
+t_index
+t_ctx_grouped_pkey::open(t_header header, t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return open(idx);
+}
+
+std::string
+t_ctx_grouped_pkey::repr() const {
+    std::stringstream ss;
+    ss << "t_ctx_grouped_pkey<" << this << ">";
+    return ss.str();
+}
+
+t_index
+t_ctx_grouped_pkey::open(t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // If we manually open/close a node, stop automatically expanding
+    m_depth_set = false;
+    m_depth = 0;
+
+    if (idx >= t_index(m_traversal->size())) {
+        return 0;
+    }
+
+    t_index retval = m_traversal->expand_node(m_sortby, idx);
+    m_rows_changed = (retval > 0);
+    return retval;
+}
+
+t_index
+t_ctx_grouped_pkey::close(t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // If we manually open/close a node, stop automatically expanding
+    m_depth_set = false;
+    m_depth = 0;
+
+    if (idx >= t_index(m_traversal->size())) {
+        return 0;
+    }
+
+    t_index retval = m_traversal->collapse_node(idx);
+    m_rows_changed = (retval > 0);
+    return retval;
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::get_data(
+    t_index start_row, t_index end_row, t_index start_col, t_index end_col
+) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex ctx_nrows = get_row_count();
+    t_uindex ncols = get_column_count();
+    auto ext = sanitize_get_data_extents(
+        ctx_nrows, ncols, start_row, end_row, start_col, end_col
+    );
+
+    t_index nrows = ext.m_erow - ext.m_srow;
+    t_index stride = ext.m_ecol - ext.m_scol;
+    std::vector<t_tscalar> values(nrows * stride);
+    std::vector<t_tscalar> tmpvalues(nrows * ncols);
+
+    std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
+
+    if (aggcols.empty()) {
+        return values;
+    }
+
+    auto* aggtable = m_tree->get_aggtable();
+
+    for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+         ++aggidx) {
+        // resolve by column index (== aggidx); skips the per-iteration
+        // name->index map lookup + temp string and the schema deep-copy.
+        aggcols[aggidx] = aggtable->_get_const_column(aggidx);
+    }
+
+    const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
+
+    const std::string& grouping_label_col =
+        m_config.get_grouping_label_column();
+
+    for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
+        t_index nidx = m_traversal->get_tree_index(ridx);
+        t_index pnidx = m_tree->get_parent_idx(nidx);
+
+        t_uindex agg_ridx = m_tree->get_aggidx(nidx);
+        t_index agg_pridx =
+            pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
+
+        t_tscalar tree_value = m_tree->get_value(nidx);
+
+        if (m_has_label && ridx > 0) {
+            // Get pkey
+            auto iters = m_tree->get_pkeys_for_leaf(nidx);
+            tree_value.set(
+                get_value_from_gstate(grouping_label_col, iters.first->m_pkey)
+            );
+        }
+
+        tmpvalues[(ridx - ext.m_srow) * ncols] = tree_value;
+
+        for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+             ++aggidx) {
+            t_tscalar value = extract_aggregate(
+                aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
+            );
+
+            tmpvalues[(ridx - ext.m_srow) * ncols + 1 + aggidx].set(value);
+        }
+    }
+
+    for (auto ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
+        for (auto cidx = ext.m_scol; cidx < ext.m_ecol; ++cidx) {
+            auto insert_idx = (ridx - ext.m_srow) * stride + cidx - ext.m_scol;
+            auto src_idx = (ridx - ext.m_srow) * ncols + cidx;
+            values[insert_idx].set(tmpvalues[src_idx]);
+        }
+    }
+    return values;
+}
+
+void
+t_ctx_grouped_pkey::notify(
+    const t_data_table& flattened,
+    const t_data_table& delta,
+    const t_data_table& prev,
+    const t_data_table& current,
+    const t_data_table& transitions,
+    const t_data_table& existed
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    rebuild();
+}
+
+void
+t_ctx_grouped_pkey::step_begin() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    reset_step_state();
+}
+
+void
+t_ctx_grouped_pkey::step_end() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    sort_by(m_sortby);
+    if (m_depth_set) {
+        set_depth(m_depth);
+    }
+}
+
+std::vector<t_aggspec>
+t_ctx_grouped_pkey::get_aggregates() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_config.get_aggregates();
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::get_row_path(t_index idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return ctx_get_path(m_tree, m_traversal, idx);
+}
+
+void
+t_ctx_grouped_pkey::reset_sortby() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    m_sortby = std::vector<t_sortspec>();
+}
+
+std::vector<t_path>
+t_ctx_grouped_pkey::get_expansion_state() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return ctx_get_expansion_state(m_tree, m_traversal);
+}
+
+void
+t_ctx_grouped_pkey::set_expansion_state(const std::vector<t_path>& paths) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    ctx_set_expansion_state(*this, HEADER_ROW, m_tree, m_traversal, paths);
+}
+
+void
+t_ctx_grouped_pkey::expand_path(const std::vector<t_tscalar>& path) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    ctx_expand_path(*this, HEADER_ROW, m_tree, m_traversal, path);
+}
+
+t_stree*
+t_ctx_grouped_pkey::_get_tree() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_tree.get();
+}
+
+t_tscalar
+t_ctx_grouped_pkey::get_tree_value(t_index nidx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_tree->get_value(nidx);
+}
+
+std::vector<t_ftreenode>
+t_ctx_grouped_pkey::get_flattened_tree(t_index idx, t_depth stop_depth) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return ctx_get_flattened_tree(
+        idx, stop_depth, *(m_traversal), m_config, m_sortby
+    );
+}
+
+std::shared_ptr<const t_traversal>
+t_ctx_grouped_pkey::get_traversal() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_traversal;
+}
+
+void
+t_ctx_grouped_pkey::sort_by(const std::vector<t_sortspec>& sortby) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    m_sortby = sortby;
+    if (m_sortby.empty()) {
+        return;
+    }
+    m_traversal->sort_by(m_config, sortby, *this);
+}
+
+void
+t_ctx_grouped_pkey::set_depth(t_depth depth) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_depth final_depth =
+        std::min<t_depth>(m_config.get_num_rpivots() - 1, depth);
+    t_index retval = 0;
+    retval = m_traversal->set_depth(m_sortby, final_depth);
+    m_rows_changed = (retval > 0);
+    m_depth = depth;
+    m_depth_set = true;
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::get_pkeys(
+    const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (!m_traversal->validate_cells(cells)) {
+        std::vector<t_tscalar> rval;
+        return rval;
+    }
+
+    std::vector<t_tscalar> rval;
+
+    tsl::hopscotch_set<t_uindex> seen;
+
+    for (const auto& c : cells) {
+        auto ptidx = m_traversal->get_tree_index(c.first);
+
+        if (static_cast<t_uindex>(ptidx) == static_cast<t_uindex>(-1)) {
+            continue;
+        }
+
+        if (seen.find(ptidx) == seen.end()) {
+            auto iters = m_tree->get_pkeys_for_leaf(ptidx);
+            for (auto iter = iters.first; iter != iters.second; ++iter) {
+                rval.push_back(iter->m_pkey);
+            }
+            seen.insert(ptidx);
+        }
+
+        auto desc = m_tree->get_descendents(ptidx);
+
+        for (auto d : desc) {
+            if (seen.find(d) != seen.end()) {
+                continue;
+            }
+
+            auto iters = m_tree->get_pkeys_for_leaf(d);
+            for (auto iter = iters.first; iter != iters.second; ++iter) {
+                rval.push_back(iter->m_pkey);
+            }
+            seen.insert(d);
+        }
+    }
+    return rval;
+}
+
+void
+t_ctx_grouped_pkey::set_feature_state(t_ctx_feature feature, bool state) {
+    m_features[feature] = state;
+}
+
+void
+t_ctx_grouped_pkey::set_alerts_enabled(bool enabled_state) {
+    m_features[CTX_FEAT_ALERT] = enabled_state;
+    m_tree->set_alerts_enabled(enabled_state);
+}
+
+void
+t_ctx_grouped_pkey::set_deltas_enabled(bool enabled_state) {
+    m_features[CTX_FEAT_DELTA] = enabled_state;
+    m_tree->set_deltas_enabled(enabled_state);
+}
+
+t_stepdelta
+t_ctx_grouped_pkey::get_step_delta(t_index bidx, t_index eidx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    bidx = std::min(bidx, t_index(m_traversal->size()));
+    eidx = std::min(eidx, t_index(m_traversal->size()));
+
+    t_stepdelta rval(
+        m_rows_changed, m_columns_changed, get_cell_delta(bidx, eidx)
+    );
+    m_tree->clear_deltas();
+    return rval;
+}
+
+std::vector<t_cellupd>
+t_ctx_grouped_pkey::get_cell_delta(t_index bidx, t_index eidx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    eidx = std::min(eidx, t_index(m_traversal->size()));
+    std::vector<t_cellupd> rval;
+    const auto& deltas = m_tree->get_deltas();
+    for (t_index idx = bidx; idx < eidx; ++idx) {
+        t_index ptidx = m_traversal->get_tree_index(idx);
+        auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
+        for (auto iter = iterators.first; iter != iterators.second; ++iter) {
+            rval.emplace_back(
+                idx, iter->m_aggidx + 1, iter->m_old_value, iter->m_new_value
+            );
+        }
+    }
+    return rval;
+}
+
+void
+t_ctx_grouped_pkey::reset(bool reset_expressions) {
+    auto pivots = m_config.get_row_pivots();
+    m_tree = std::make_shared<t_stree>(
+        pivots, m_config.get_aggregates(), m_schema, m_config
+    );
+    m_tree->init();
+    m_tree->set_deltas_enabled(get_feature_state(CTX_FEAT_DELTA));
+    m_traversal = std::make_shared<t_traversal>(m_tree);
+
+    if (reset_expressions) {
+        m_expression_tables->reset();
+    }
+}
+
+void
+t_ctx_grouped_pkey::reset_step_state() {
+    m_rows_changed = false;
+    m_columns_changed = false;
+    if (t_env::log_progress()) {
+        std::cout << "t_ctx_grouped_pkey.reset_step_state " << repr() << '\n';
+    }
+}
+
+std::vector<t_stree*>
+t_ctx_grouped_pkey::get_trees() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    std::vector<t_stree*> rval(1);
+    rval[0] = m_tree.get();
+    return rval;
+}
+
+bool
+t_ctx_grouped_pkey::has_deltas() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return true;
+}
+
+template <typename DATA_T>
+void
+rebuild_helper(t_column* /*unused*/) {}
+
+void
+t_ctx_grouped_pkey::rebuild() {
+    auto tbl = m_gstate->get_pkeyed_table();
+
+    if (m_config.has_filters()) {
+        auto mask = filter_table_for_config(*tbl, m_config);
+        tbl = tbl->clone(mask);
+    }
+
+    std::string child_col_name = m_config.get_child_pkey_column();
+
+    std::shared_ptr<const t_column> child_col_sptr =
+        tbl->get_const_column(child_col_name);
+
+    const t_column* child_col = child_col_sptr.get();
+    auto expansion_state = get_expansion_state();
+
+    std::sort(
+        expansion_state.begin(),
+        expansion_state.end(),
+        [](const t_path& a, const t_path& b) {
+            return a.path().size() < b.path().size();
+        }
+    );
+
+    for (auto& p : expansion_state) {
+        std::reverse(p.path().begin(), p.path().end());
+    }
+
+    reset();
+
+    t_uindex nrows = child_col->size();
+
+    if (nrows == 0) {
+        return;
+    }
+
+    struct t_datum {
+        t_uindex m_pidx;
+        t_tscalar m_parent;
+        t_tscalar m_child;
+        t_tscalar m_pkey;
+        bool m_is_rchild;
+        t_uindex m_idx;
+    };
+
+    const auto* sortby_col =
+        tbl->_get_const_column(m_config.get_sort_by(child_col_name));
+
+    const auto* parent_col =
+        tbl->_get_const_column(m_config.get_parent_pkey_column());
+
+    const auto* pkey_col = tbl->_get_const_column("psp_pkey");
+
+    std::vector<t_datum> data(nrows);
+    tsl::hopscotch_map<t_tscalar, t_uindex> child_ridx_map;
+    std::vector<bool> self_pkey_eq(nrows);
+
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        data[idx].m_child.set(child_col->get_scalar(idx));
+        data[idx].m_pkey.set(pkey_col->get_scalar(idx));
+        child_ridx_map[data[idx].m_child] = idx;
+    }
+
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        auto ppkey = parent_col->get_scalar(idx);
+        data[idx].m_parent.set(ppkey);
+
+        auto p_iter = child_ridx_map.find(ppkey);
+        bool missing_parent = p_iter == child_ridx_map.end();
+
+        data[idx].m_is_rchild =
+            !ppkey.is_valid() || data[idx].m_child == ppkey || missing_parent;
+        data[idx].m_pidx =
+            data[idx].m_is_rchild ? 0 : child_ridx_map.at(data[idx].m_parent);
+        data[idx].m_idx = idx;
+    }
+
+    struct t_datumcmp {
+        bool
+        operator()(const t_datum& a, const t_datum& b) const {
+            typedef std::tuple<bool, t_tscalar, t_tscalar> t_tuple;
+            return t_tuple(!a.m_is_rchild, a.m_parent, a.m_child)
+                < t_tuple(!b.m_is_rchild, b.m_parent, b.m_child);
+        }
+    };
+
+    t_datumcmp cmp;
+
+    std::sort(data.begin(), data.end(), cmp);
+
+    std::vector<t_uindex> root_children;
+
+    std::queue<t_uindex> queue;
+    t_uindex nroot_children = 0;
+    while (nroot_children < nrows && data[nroot_children].m_is_rchild) {
+        queue.push(nroot_children);
+        ++nroot_children;
+    }
+
+    tsl::hopscotch_map<t_tscalar, std::pair<t_uindex, t_uindex>> p_range_map;
+
+    t_uindex brange = nroot_children;
+    for (t_uindex idx = nroot_children; idx < nrows; ++idx) {
+        if (data[idx].m_parent != data[idx - 1].m_parent
+            && idx > nroot_children) {
+            p_range_map[data[idx - 1].m_parent] =
+                std::pair<t_uindex, t_uindex>(brange, idx);
+            brange = idx;
+        }
+    }
+
+    p_range_map[data.back().m_parent] =
+        std::pair<t_uindex, t_uindex>(brange, nrows);
+
+    // map from unsorted space to sorted space
+    tsl::hopscotch_map<t_uindex, t_uindex> sortidx_map;
+
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        sortidx_map[data[idx].m_idx] = idx;
+    }
+
+    while (!queue.empty()) {
+        // ridx is in sorted space
+        t_uindex ridx = queue.front();
+        queue.pop();
+
+        const t_datum& rec = data[ridx];
+        t_uindex pridx = rec.m_is_rchild ? 0 : sortidx_map.at(rec.m_pidx);
+
+        auto sortby_value =
+            m_symtable.get_interned_tscalar(sortby_col->get_scalar(rec.m_idx));
+
+        t_uindex nidx = ridx + 1;
+        t_uindex pidx = rec.m_is_rchild ? 0 : pridx + 1;
+
+        auto pnode = m_tree->get_node(pidx);
+
+        auto value = m_symtable.get_interned_tscalar(rec.m_child);
+
+        t_stnode node(
+            nidx, pidx, value, pnode.m_depth + 1, sortby_value, 1, nidx
+        );
+
+        m_tree->insert_node(node);
+        m_tree->add_pkey(nidx, m_symtable.get_interned_tscalar(rec.m_pkey));
+
+        auto riter = p_range_map.find(rec.m_child);
+
+        if (riter != p_range_map.end()) {
+            auto range = riter->second;
+            t_uindex bidx = range.first;
+            t_uindex eidx = range.second;
+
+            for (t_uindex cidx = bidx; cidx < eidx; ++cidx) {
+                queue.push(cidx);
+            }
+        }
+    }
+
+    auto* aggtable = m_tree->_get_aggtable();
+    aggtable->extend(nrows + 1);
+
+    auto aggspecs = m_config.get_aggregates();
+    t_uindex naggs = aggspecs.size();
+
+    std::vector<t_uindex> aggindices(nrows);
+
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        aggindices[idx] = data[idx].m_idx;
+    }
+
+    parallel_for(
+        int(naggs),
+        [&aggtable, &aggindices, &aggspecs, &tbl](int aggnum) {
+            const t_aggspec& spec = aggspecs[aggnum];
+            if (spec.agg() == AGGTYPE_IDENTITY) {
+                auto* scol =
+                    aggtable->_get_column(spec.get_first_depname());
+                scol->copy(
+                    tbl->_get_const_column(spec.get_first_depname()),
+                    aggindices,
+                    1
+                );
+            }
+        }
+    );
+
+    m_traversal = std::make_shared<t_traversal>(m_tree);
+
+    set_expansion_state(expansion_state);
+
+    if (!m_sortby.empty()) {
+        m_traversal->sort_by(m_config, m_sortby, *this);
+    }
+}
+
+void
+t_ctx_grouped_pkey::pprint() const {
+    m_traversal->pprint();
+}
+
+void
+t_ctx_grouped_pkey::notify(
+    const t_data_table& flattened, bool /* is_registration */
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    rebuild();
+}
+
+// aggregates should be presized to be same size
+// as agg_indices
+void
+t_ctx_grouped_pkey::get_aggregates_for_sorting(
+    t_uindex nidx,
+    const std::vector<t_index>& agg_indices,
+    std::vector<t_tscalar>& aggregates,
+    t_ctx2* /*unused*/
+) const {
+    for (t_uindex idx = 0, loop_end = agg_indices.size(); idx < loop_end;
+         ++idx) {
+        auto which_agg = agg_indices[idx];
+
+        if (which_agg < 0) {
+            aggregates[idx].set(m_tree->get_sortby_value(nidx));
+        } else {
+            aggregates[idx].set(m_tree->get_aggregate(nidx, which_agg));
+        }
+    }
+}
+
+t_dtype
+t_ctx_grouped_pkey::get_column_dtype(t_uindex idx) const {
+    if (idx == 0 || idx >= static_cast<t_uindex>(get_column_count())) {
+        return DTYPE_NONE;
+    }
+
+    auto* aggtable = m_tree->_get_aggtable();
+    return aggtable->get_const_column(idx - 1)->get_dtype();
+}
+
+void
+t_ctx_grouped_pkey::compute_expressions(
+    const std::shared_ptr<t_data_table>& master,
+    const t_gstate::t_mapping& pkey_map,
+    t_expression_vocab& expression_vocab,
+    t_regex_mapping& regex_mapping
+) {
+    // Clear the transitional expression tables on the context so they are
+    // ready for the next update.
+    m_expression_tables->clear_transitional_tables();
+
+    std::shared_ptr<t_data_table> master_expression_table =
+        m_expression_tables->m_master;
+
+    // Set the master table to the right size.
+    t_uindex num_rows = master->size();
+    master_expression_table->reserve(num_rows);
+    master_expression_table->set_size(num_rows);
+
+    const auto& expressions = m_config.get_expressions();
+    for (const auto& expr : expressions) {
+        // Compute the expressions on the master table.
+        expr->compute(
+            master,
+            pkey_map,
+            master_expression_table,
+            expression_vocab,
+            regex_mapping
+        );
+    }
+}
+
+void
+t_ctx_grouped_pkey::compute_expressions(
+    const std::shared_ptr<t_data_table>& master,
+    const t_gstate::t_mapping& pkey_map,
+    const std::shared_ptr<t_data_table>& flattened,
+    const std::shared_ptr<t_data_table>& delta,
+    const std::shared_ptr<t_data_table>& prev,
+    const std::shared_ptr<t_data_table>& current,
+    const std::shared_ptr<t_data_table>& transitions,
+    const std::shared_ptr<t_data_table>& existed,
+    t_expression_vocab& expression_vocab,
+    t_regex_mapping& regex_mapping
+) {
+    // Clear the tables so they are ready for this round of updates
+    m_expression_tables->clear_transitional_tables();
+
+    // All tables are the same size
+    t_uindex flattened_num_rows = flattened->size();
+    m_expression_tables->reserve_transitional_table_size(flattened_num_rows);
+    m_expression_tables->set_transitional_table_size(flattened_num_rows);
+
+    // Update the master expression table's size
+    t_uindex master_num_rows = master->size();
+    m_expression_tables->m_master->reserve(master_num_rows);
+    m_expression_tables->m_master->set_size(master_num_rows);
+
+    const auto& expressions = m_config.get_expressions();
+    for (const auto& expr : expressions) {
+        // master: compute based on latest state of the gnode state table
+        expr->compute(
+            master,
+            pkey_map,
+            m_expression_tables->m_master,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // flattened: compute based on the latest update dataset
+        expr->compute(
+            flattened,
+            pkey_map,
+            m_expression_tables->m_flattened,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // delta: for each numerical column, the numerical delta between the
+        // previous value and the current value in the row.
+        expr->compute(
+            delta,
+            pkey_map,
+            m_expression_tables->m_delta,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // prev: the values of the updated rows before this update was applied
+        expr->compute(
+            prev,
+            pkey_map,
+            m_expression_tables->m_prev,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // current: the current values of the updated rows
+        expr->compute(
+            current,
+            pkey_map,
+            m_expression_tables->m_current,
+            expression_vocab,
+            regex_mapping
+        );
+    }
+
+    // Calculate the transitions now that the intermediate tables are computed
+    m_expression_tables->calculate_transitions(existed);
+}
+
+t_uindex
+t_ctx_grouped_pkey::num_expressions() const {
+    const auto& expressions = m_config.get_expressions();
+    return expressions.size();
+}
+
+bool
+t_ctx_grouped_pkey::is_expression_column(const std::string& colname) const {
+    const t_schema& schema = m_expression_tables->m_master->get_schema();
+    return schema.has_column(colname);
+}
+
+t_tscalar
+t_ctx_grouped_pkey::get_value_from_gstate(
+    const std::string& colname, const t_tscalar& pkey
+) const {
+    if (is_expression_column(colname)) {
+        return m_gstate->get_value(
+            *(m_expression_tables->m_master), colname, pkey
+        );
+    }
+    std::shared_ptr<t_data_table> master_table = m_gstate->get_table();
+    return m_gstate->get_value(*master_table, colname, pkey);
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::unity_get_row_data(t_uindex idx) const {
+    auto rval = get_data(idx, idx + 1, 0, get_column_count());
+    if (rval.empty()) {
+        return {};
+    }
+
+    return {rval.begin() + 1, rval.end()};
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::unity_get_column_data(t_uindex idx) const {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return {};
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::unity_get_row_path(t_uindex idx) const {
+    return get_row_path(idx);
+}
+
+std::vector<t_tscalar>
+t_ctx_grouped_pkey::unity_get_column_path(t_uindex idx) const {
+    return {};
+}
+
+t_uindex
+t_ctx_grouped_pkey::unity_get_row_depth(t_uindex ridx) const {
+    return m_traversal->get_depth(ridx);
+}
+
+t_uindex
+t_ctx_grouped_pkey::unity_get_column_depth(t_uindex cidx) const {
+    return 0;
+}
+
+std::string
+t_ctx_grouped_pkey::unity_get_column_name(t_uindex idx) const {
+    return m_config.col_at(idx);
+}
+
+std::string
+t_ctx_grouped_pkey::unity_get_column_display_name(t_uindex idx) const {
+    return m_config.col_at(idx);
+}
+
+std::vector<std::string>
+t_ctx_grouped_pkey::unity_get_column_names() const {
+    return m_config.get_column_names();
+}
+
+std::vector<std::string>
+t_ctx_grouped_pkey::unity_get_column_display_names() const {
+    return m_config.get_column_names();
+}
+
+t_uindex
+t_ctx_grouped_pkey::unity_get_column_count() const {
+    return get_column_count() - 1;
+}
+
+t_uindex
+t_ctx_grouped_pkey::unity_get_row_count() const {
+    return get_row_count();
+}
+
+bool
+t_ctx_grouped_pkey::unity_get_row_expanded(t_uindex idx) const {
+    return m_traversal->get_node_expanded(idx);
+}
+
+bool
+t_ctx_grouped_pkey::unity_get_column_expanded(t_uindex idx) const {
+    return false;
+}
+
+void
+t_ctx_grouped_pkey::clear_deltas() {}
+
+void
+t_ctx_grouped_pkey::unity_init_load_step_end() {}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_handle.cpp

@@ -0,0 +1,52 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/context_handle.h>
+
+namespace perspective {
+
+t_ctx_handle::t_ctx_handle() : m_ctx_type(ZERO_SIDED_CONTEXT), m_ctx(nullptr) {}
+
+t_ctx_handle::t_ctx_handle(void* ctx, t_ctx_type ctx_type) :
+    m_ctx_type(ctx_type),
+    m_ctx(ctx) {}
+
+std::string
+t_ctx_handle::get_type_descr() const {
+    switch (m_ctx_type) {
+        case TWO_SIDED_CONTEXT: {
+            return "TWO_SIDED_CONTEXT";
+        } break;
+        case ONE_SIDED_CONTEXT: {
+            return "ONE_SIDED_CONTEXT";
+        } break;
+        case ZERO_SIDED_CONTEXT: {
+            return "ZERO_SIDED_CONTEXT";
+        } break;
+        case UNIT_CONTEXT: {
+            return "UNIT_CONTEXT";
+        } break;
+        case GROUPED_PKEY_CONTEXT: {
+            return "GROUPED_PKEY_CONTEXT";
+        } break;
+        case GROUPED_COLUMNS_CONTEXT: {
+            return "GROUPED_COLUMNS_CONTEXT";
+        }
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Invalid context");
+        } break;
+    }
+    return "ZERO_SIDED_CONTEXT";
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_iterators.cpp

@@ -0,0 +1,246 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/sort_specification.h>
+#include <perspective/context_zero.h>
+#include <perspective/context_one.h>
+#include <perspective/context_two.h>
+#include <perspective/extract_aggregate.h>
+#include <perspective/raii.h>
+#include <perspective/sparse_tree.h>
+
+namespace perspective {
+
+t_trav_iter_node::t_trav_iter_node(
+    const t_tscalar& value,
+    bool expanded,
+    t_depth depth,
+    t_index ndesc,
+    t_index tnid,
+    t_index ridx,
+    t_index pridx
+) :
+    value(value),
+    expanded(expanded),
+    depth(depth),
+    ndesc(ndesc),
+    tree_handle(tnid),
+    idx(ridx),
+    pidx(pridx) {}
+
+t_ctx2_trav_seq_iter::t_ctx2_trav_seq_iter(t_ctx2* ctx, t_header header) :
+    m_ctx(ctx),
+    m_header(header),
+    m_curidx(0) {
+    m_nrows = ctx->get_row_count(header);
+}
+
+t_leaf_iter_node::t_leaf_iter_node(
+    const t_tscalar& value,
+    bool expanded,
+    t_depth depth,
+    t_index ndesc,
+    t_index tnid,
+    t_index ridx,
+    t_index pridx,
+    t_index nleaves
+) :
+    value(value),
+    expanded(expanded),
+    depth(depth),
+    ndesc(ndesc),
+    tree_handle(tnid),
+    idx(ridx),
+    pidx(pridx),
+    nleaves(nleaves) {}
+
+bool
+t_ctx2_trav_seq_iter::has_next() const {
+    return m_curidx < m_nrows;
+}
+
+t_trav_iter_node*
+t_ctx2_trav_seq_iter::next() {
+    auto node = m_ctx->get_trav_node(m_header, m_curidx);
+    auto iter_node = new t_trav_iter_node(
+        m_ctx->get_tree_value(m_header, node.m_tnid),
+        node.m_expanded,
+        node.m_depth,
+        node.m_ndesc,
+        node.m_tnid,
+        m_curidx,
+        m_curidx - node.m_rel_pidx
+    );
+
+    m_curidx += 1;
+    return iter_node;
+}
+
+t_ctx2_leaf_iter::t_ctx2_leaf_iter(const t_ctx2* ctx, t_header header) :
+    m_ctx(ctx),
+    m_header(header),
+    m_curidx(0) {
+    m_nrows = ctx->get_row_count(header);
+}
+
+bool
+t_ctx2_leaf_iter::has_next() const {
+    return m_curidx < m_nrows;
+}
+
+t_leaf_iter_node*
+t_ctx2_leaf_iter::next() {
+    auto node = m_ctx->get_trav_node(m_header, m_curidx);
+    t_index nleaves = m_ctx->get_trav_num_tree_leaves(m_header, m_curidx);
+
+    t_leaf_iter_node* iter_node = new t_leaf_iter_node(
+        m_ctx->get_tree_value(m_header, node.m_tnid),
+        node.m_expanded,
+        node.m_depth,
+        node.m_ndesc,
+        node.m_tnid,
+        m_curidx,
+        m_curidx - node.m_rel_pidx,
+        nleaves
+    );
+
+    m_curidx += 1;
+    return iter_node;
+}
+
+t_ctx1_tree_iter::t_ctx1_tree_iter(t_ctx1* ctx, t_index root, t_depth depth) :
+    m_curidx(0),
+    m_ctx(ctx) {
+    m_agg_indices = std::vector<t_index>(ctx->get_aggregates().size());
+
+    for (t_uindex idx = 0, loop_end = m_agg_indices.size(); idx < loop_end;
+         ++idx) {
+        m_agg_indices[idx] = idx;
+    }
+
+    m_has_next = !(m_agg_indices.empty());
+    if (m_has_next) {
+        t_depth rdepth = ctx->get_trav_depth(root);
+        m_edepth = rdepth + depth;
+        m_edepth = std::min(m_edepth, ctx->get_num_levels() - 1);
+        m_vec = m_ctx->get_flattened_tree(root, m_edepth);
+        if (m_vec.size() == 1 && m_vec[0].m_nchild == 0) {
+            m_has_next = false;
+        }
+
+        m_aggcols = ctx->_get_tree()->get_aggcols(m_agg_indices);
+    }
+}
+
+t_ctx1_tree_iter::t_ctx1_tree_iter(
+    t_ctx1* ctx,
+    t_index root,
+    t_depth depth,
+    const std::vector<t_index>& aggregates
+) :
+    m_curidx(0),
+    m_ctx(ctx),
+    m_agg_indices(aggregates) {
+    m_has_next = !(m_agg_indices.empty());
+    if (m_has_next) {
+        t_depth rdepth = ctx->get_trav_depth(root);
+        m_edepth = rdepth + depth;
+        m_edepth = std::min(m_edepth, ctx->get_num_levels() - 1);
+        m_vec = m_ctx->get_flattened_tree(root, m_edepth);
+        if (m_vec.size() == 1 && m_vec[0].m_nchild == 0) {
+            m_has_next = false;
+        }
+
+        m_aggcols = ctx->_get_tree()->get_aggcols(aggregates);
+    }
+}
+
+bool
+t_ctx1_tree_iter::has_next() const {
+    return m_has_next;
+}
+
+PyObject*
+t_ctx1_tree_iter::next() {
+    t_ftreenode& ftnode = m_vec[m_curidx];
+    t_index bidx;
+    t_index nrows;
+    if (m_ctx->get_row_pivots().size() == 0 || m_edepth == 0) {
+        bidx = 0;
+        nrows = 1;
+    } else {
+        bidx = ftnode.m_fcidx;
+        nrows = ftnode.m_nchild;
+    }
+    npy_intp dims[] = {static_cast<npy_intp>(nrows)};
+    t_py_handle dtype(
+        Py_BuildValue("[(s, s), (s, s)]", "id", "i4", "field_label", "O")
+    );
+    t_index n_aggs = m_agg_indices.size();
+
+    for (int i = 0; i < n_aggs; i++) {
+        std::stringstream field_name;
+        field_name << "field_" << i + 1;
+        t_py_handle d_field(
+            Py_BuildValue("(s,s)", field_name.str().c_str(), "f8")
+        );
+        PyList_Append(dtype.m_pyo, d_field.m_pyo);
+    }
+    PyArray_Descr* descr;
+    PyArray_DescrConverter(dtype.m_pyo, &descr);
+    t_py_handle array(PyArray_SimpleNewFromDescr(1, dims, descr));
+    PyArrayObject* array_ = reinterpret_cast<PyArrayObject*>(array.m_pyo);
+
+    t_stree* tree = m_ctx->_get_tree();
+    const std::vector<t_aggspec>& aggspecs =
+        m_ctx->get_config().get_aggregates();
+
+    for (int i = 0; i < nrows; i++) {
+        t_ftreenode& c_ftnode = m_vec[bidx + i];
+        t_index cidx = c_ftnode.m_idx;
+        void* elem_ptr = PyArray_GETPTR1(array_, i);
+        t_py_handle val(PyTuple_New(n_aggs + 2));
+        PyTuple_SetItem(val.m_pyo, 0, PyInt_FromLong(cidx));
+
+        t_tscalar tree_value = m_ctx->get_tree_value(cidx);
+        PyObject* py_tree_value = tscalar_to_python(tree_value);
+        PyTuple_SetItem(val.m_pyo, 1, py_tree_value);
+
+        t_index p_cidx = tree->get_parent_idx(cidx);
+
+        t_uindex agg_ridx = tree->get_aggidx(cidx);
+        t_index agg_pridx =
+            p_cidx == INVALID_INDEX ? INVALID_INDEX : tree->get_aggidx(p_cidx);
+
+        for (int j = 0; j < n_aggs; j++) {
+            t_tscalar agg_scalar = extract_aggregate(
+                aggspecs[j], m_aggcols[j], agg_ridx, agg_pridx
+            );
+            double aggdbl = agg_scalar.to_double();
+            PyTuple_SetItem(val.m_pyo, j + 2, PyFloat_FromDouble(aggdbl));
+        }
+        PyArray_SETITEM(array_, static_cast<char*>(elem_ptr), val.m_pyo);
+    }
+    long is_leaf = ftnode.m_depth + 1 == m_edepth;
+    PyObject* rval = Py_BuildValue("OO", PyBool_FromLong(is_leaf), array.m_pyo);
+    m_curidx++;
+
+    if (m_vec[m_curidx].m_depth == t_depth(-1)
+        || m_ctx->get_row_pivots().size() == 0 || m_edepth == 0
+        || static_cast<std::uint64_t>(m_curidx) >= m_vec.size()) {
+        m_has_next = false;
+    }
+    return rval;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_one.cpp

@@ -0,0 +1,944 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/sort_specification.h>
+#include <perspective/get_data_extents.h>
+#include <perspective/context_one.h>
+#include <perspective/extract_aggregate.h>
+#include <perspective/filter.h>
+#include <perspective/sparse_tree.h>
+#include <perspective/tree_context_common.h>
+
+#include <perspective/env_vars.h>
+#include <perspective/traversal.h>
+
+#include <memory>
+
+#include <utility>
+
+namespace perspective {
+
+t_ctx1::t_ctx1(const t_schema& schema, const t_config& pivot_config) :
+    t_ctxbase<t_ctx1>(schema, pivot_config),
+    m_depth(0),
+    m_depth_set(false) {}
+
+t_ctx1::~t_ctx1() = default;
+
+void
+t_ctx1::init() {
+    auto pivots = m_config.get_row_pivots();
+    m_tree = std::make_shared<t_stree>(
+        pivots, m_config.get_aggregates(), m_schema, m_config
+    );
+    m_tree->init();
+    m_traversal = std::make_shared<t_traversal>(m_tree);
+
+    // Each context stores its own expression columns in separate
+    // `t_data_table`s so that each context's expressions are isolated
+    // and do not affect other contexts when they are calculated.
+    const auto& expressions = m_config.get_expressions();
+    m_expression_tables = std::make_shared<t_expression_tables>(
+        expressions, m_config.get_backing_store()
+    );
+
+    m_init = true;
+}
+
+t_index
+t_ctx1::get_row_count() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_traversal->size();
+}
+
+t_index
+t_ctx1::get_column_count() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_config.get_num_aggregates() + 1;
+}
+
+t_index
+t_ctx1::open(t_header header, t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return open(idx);
+}
+
+std::string
+t_ctx1::repr() const {
+    std::stringstream ss;
+    ss << "t_ctx1<" << this << ">";
+    return ss.str();
+}
+
+t_index
+t_ctx1::open(t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // If we manually open/close a node, stop automatically expanding
+    m_depth_set = false;
+    m_depth = 0;
+
+    if (idx >= t_index(m_traversal->size())) {
+        return 0;
+    }
+
+    t_index retval = m_traversal->expand_node(m_sortby, idx);
+    m_rows_changed = (retval > 0);
+    return retval;
+}
+
+t_index
+t_ctx1::close(t_index idx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // If we manually open/close a node, stop automatically expanding
+    m_depth_set = false;
+    m_depth = 0;
+
+    if (idx >= t_index(m_traversal->size())) {
+        return 0;
+    }
+
+    t_index retval = m_traversal->collapse_node(idx);
+    m_rows_changed = (retval > 0);
+    return retval;
+}
+
+std::pair<t_tscalar, t_tscalar>
+t_ctx1::get_min_max(const std::string& colname) const {
+    auto rval = std::make_pair(mknone(), mknone());
+    auto* aggtable = m_tree->get_aggtable();
+    const t_schema& aggschema = aggtable->get_schema();
+    const auto* col = aggtable->_get_const_column(colname);
+    auto colidx = aggschema.get_colidx(colname);
+    auto depth = m_config.get_num_rpivots();
+    const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
+    bool is_finished = false;
+    while (!is_finished && depth > 0) {
+        for (std::size_t i = 0; i < m_traversal->size(); i++) {
+            t_index nidx = m_traversal->get_tree_index(i);
+            t_index pnidx = m_tree->get_parent_idx(nidx);
+            if (m_tree->get_depth(nidx) != depth) {
+                continue;
+            }
+
+            t_uindex agg_ridx = m_tree->get_aggidx(nidx);
+            t_index agg_pridx = pnidx == INVALID_INDEX
+                ? INVALID_INDEX
+                : m_tree->get_aggidx(pnidx);
+            t_tscalar val =
+                extract_aggregate(aggspecs[colidx], col, agg_ridx, agg_pridx);
+
+            if (!val.is_valid()) {
+                continue;
+            }
+
+            is_finished = true;
+            if (rval.first.is_none() || (!val.is_none() && val < rval.first)) {
+                rval.first = val;
+            }
+
+            if (val > rval.second) {
+                rval.second = val;
+            }
+        }
+
+        depth--;
+    }
+
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_ctx1::get_data(
+    t_index start_row, t_index end_row, t_index start_col, t_index end_col
+) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex ctx_nrows = get_row_count();
+    t_uindex ncols = get_column_count();
+    auto ext = sanitize_get_data_extents(
+        ctx_nrows, ncols, start_row, end_row, start_col, end_col
+    );
+
+    t_index nrows = ext.m_erow - ext.m_srow;
+    t_index stride = ext.m_ecol - ext.m_scol;
+
+    std::vector<t_tscalar> values(nrows * stride);
+
+    std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
+
+    auto* aggtable = m_tree->get_aggtable();
+    auto none = mknone();
+
+    for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+         ++aggidx) {
+        // `aggidx` is the column's position in the aggtable schema, so resolve
+        // by index — avoids the per-iteration name->index map lookup + a temp
+        // std::string, and the up-front schema deep-copy.
+        aggcols[aggidx] = aggtable->_get_const_column(aggidx);
+    }
+
+    const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
+
+    for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
+        t_index nidx = m_traversal->get_tree_index(ridx);
+        t_index pnidx = m_tree->get_parent_idx(nidx);
+
+        t_uindex agg_ridx = m_tree->get_aggidx(nidx);
+        t_index agg_pridx =
+            pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
+
+        t_index row_off = (ridx - ext.m_srow) * stride;
+
+        if (ext.m_scol == 0) {
+            t_tscalar tree_value = m_tree->get_value(nidx);
+            values[row_off] = tree_value;
+        }
+
+        for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+             ++aggidx) {
+            t_index col = 1 + aggidx;
+            if (col < ext.m_scol || col >= ext.m_ecol) {
+                continue;
+            }
+            t_tscalar value = extract_aggregate(
+                aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
+            );
+            if (!value.is_valid()) {
+                value.set(none);
+            }
+            values[row_off + col - ext.m_scol].set(value);
+        }
+    }
+
+    return values;
+}
+
+std::vector<t_tscalar>
+t_ctx1::get_data(const std::vector<t_uindex>& rows) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex nrows = rows.size();
+    t_uindex ncols = get_column_count();
+
+    std::vector<t_tscalar> values(nrows * ncols);
+
+    std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
+
+    auto* aggtable = m_tree->get_aggtable();
+    auto none = mknone();
+
+    for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+         ++aggidx) {
+        // `aggidx` is the column's position in the aggtable schema, so resolve
+        // by index — avoids the per-iteration name->index map lookup + a temp
+        // std::string, and the up-front schema deep-copy.
+        aggcols[aggidx] = aggtable->_get_const_column(aggidx);
+    }
+
+    const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
+
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        t_uindex ridx = rows[idx];
+        t_index nidx = m_traversal->get_tree_index(ridx);
+        t_index pnidx = m_tree->get_parent_idx(nidx);
+
+        t_uindex agg_ridx = m_tree->get_aggidx(nidx);
+        t_index agg_pridx =
+            pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
+
+        t_tscalar tree_value = m_tree->get_value(nidx);
+        values[idx * ncols] = tree_value;
+
+        for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+             ++aggidx) {
+            t_tscalar value = extract_aggregate(
+                aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
+            );
+            if (!value.is_valid()) {
+                value.set(none);
+            }
+            values[idx * ncols + 1 + aggidx].set(value);
+        }
+    }
+
+    return values;
+}
+
+void
+t_ctx1::notify(const t_data_table& flattened, bool /* is_registration */) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    notify_sparse_tree(
+        m_tree,
+        m_traversal,
+        true,
+        m_config.get_aggregates(),
+        m_config.get_sortby_pairs(),
+        m_sortby,
+        flattened,
+        m_config,
+        *m_gstate,
+        *(m_expression_tables->m_master)
+    );
+}
+
+void
+t_ctx1::notify(
+    const t_data_table& flattened,
+    const t_data_table& delta,
+    const t_data_table& prev,
+    const t_data_table& current,
+    const t_data_table& transitions,
+    const t_data_table& existed
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    notify_sparse_tree(
+        m_tree,
+        m_traversal,
+        true,
+        m_config.get_aggregates(),
+        m_config.get_sortby_pairs(),
+        m_sortby,
+        flattened,
+        delta,
+        prev,
+        current,
+        transitions,
+        existed,
+        m_config,
+        *m_gstate,
+        *(m_expression_tables->m_master)
+    );
+}
+
+void
+t_ctx1::step_begin() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    reset_step_state();
+}
+
+void
+t_ctx1::step_end() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    if (m_total_only) {
+        m_traversal->rebuild_for_total();
+    } else if (m_leaves_only) {
+        m_traversal->rebuild_from_leaves(m_sortby);
+    } else {
+        sort_by(m_sortby);
+        if (m_depth_set) {
+            set_depth(m_depth);
+        }
+    }
+}
+
+t_aggspec
+t_ctx1::get_aggregate(t_uindex idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    if (idx >= m_config.get_num_aggregates()) {
+        return {};
+    }
+    return m_config.get_aggregates()[idx];
+}
+
+t_tscalar
+t_ctx1::get_aggregate_name(t_uindex idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_tscalar s;
+    if (idx >= m_config.get_num_aggregates()) {
+        return s;
+    }
+    s.set(m_config.get_aggregates()[idx].name_scalar());
+    return s;
+}
+
+std::vector<t_aggspec>
+t_ctx1::get_aggregates() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_config.get_aggregates();
+}
+
+std::vector<t_tscalar>
+t_ctx1::get_row_path(t_index idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    if (idx < 0) {
+        return {};
+    }
+    return ctx_get_path(m_tree, m_traversal, idx);
+}
+
+void
+t_ctx1::reset_sortby() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    m_sortby = std::vector<t_sortspec>();
+}
+
+void
+t_ctx1::sort_by(const std::vector<t_sortspec>& sortby) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    m_sortby = sortby;
+    if (m_sortby.empty()) {
+        return;
+    }
+    m_traversal->sort_by(m_config, sortby, *(m_tree));
+}
+
+void
+t_ctx1::set_depth(t_depth depth) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    if (m_config.get_num_rpivots() == 0) {
+        return;
+    }
+    depth = std::min<t_depth>(m_config.get_num_rpivots() - 1, depth);
+    t_index retval = 0;
+    retval = m_traversal->set_depth(m_sortby, depth);
+    m_rows_changed = (retval > 0);
+    m_depth = depth;
+    m_depth_set = true;
+}
+
+void
+t_ctx1::set_leaves_only(bool enabled) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    m_leaves_only = enabled;
+    m_traversal->set_leaves_only(enabled, m_config.get_num_rpivots());
+}
+
+void
+t_ctx1::set_total_only(bool enabled) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    m_total_only = enabled;
+    m_traversal->set_total_only(enabled);
+}
+
+std::vector<t_tscalar>
+t_ctx1::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (!m_traversal->validate_cells(cells)) {
+        std::vector<t_tscalar> rval;
+        return rval;
+    }
+
+    std::vector<t_tscalar> rval;
+    std::vector<t_index> tindices(cells.size());
+    for (const auto& c : cells) {
+        auto ptidx = m_traversal->get_tree_index(c.first);
+        auto pkeys = m_tree->get_pkeys(ptidx);
+
+        rval.insert(std::end(rval), std::begin(pkeys), std::end(pkeys));
+    }
+    return rval;
+}
+
+bool
+t_ctx1::get_deltas_enabled() const {
+    return m_features[CTX_FEAT_DELTA];
+}
+
+void
+t_ctx1::set_feature_state(t_ctx_feature feature, bool state) {
+    m_features[feature] = state;
+}
+
+void
+t_ctx1::set_alerts_enabled(bool enabled_state) {
+    m_features[CTX_FEAT_ALERT] = enabled_state;
+    m_tree->set_alerts_enabled(enabled_state);
+}
+
+void
+t_ctx1::set_deltas_enabled(bool enabled_state) {
+    m_features[CTX_FEAT_DELTA] = enabled_state;
+    m_tree->set_deltas_enabled(enabled_state);
+}
+
+/**
+ * @brief Returns updated cells.
+ *
+ * @param bidx
+ * @param eidx
+ * @return t_stepdelta
+ */
+t_stepdelta
+t_ctx1::get_step_delta(t_index bidx, t_index eidx) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    bidx = std::min(bidx, t_index(m_traversal->size()));
+    eidx = std::min(eidx, t_index(m_traversal->size()));
+
+    t_stepdelta rval(
+        m_rows_changed, m_columns_changed, get_cell_delta(bidx, eidx)
+    );
+    m_tree->clear_deltas();
+    return rval;
+}
+
+/**
+ * @brief Returns a `t_rowdelta` object containing:
+ * - the row indices that have been updated
+ * - the data from those updated rows
+ *
+ * @return t_rowdelta
+ */
+t_rowdelta
+t_ctx1::get_row_delta() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    std::vector<t_uindex> rows = get_rows_changed();
+    std::vector<t_tscalar> data = get_data(rows);
+    t_rowdelta rval(m_rows_changed, rows.size(), data);
+    m_tree->clear_deltas();
+    return rval;
+}
+
+std::vector<t_uindex>
+t_ctx1::get_rows_changed() {
+    std::vector<t_uindex> rows;
+    const auto& deltas = m_tree->get_deltas();
+    auto eidx = t_uindex(m_traversal->size());
+
+    // `idx` increases monotonically and is pushed at most once, so `rows` is
+    // already unique and sorted: the per-iteration `std::find` (which made this
+    // O(n^2)) and the trailing `std::sort` were both dead work.
+    for (t_uindex idx = 0; idx < eidx; ++idx) {
+        t_index ptidx = m_traversal->get_tree_index(idx);
+        // Retrieve delta from storage and check if the row has been changed
+        auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
+        if (iterators.first != iterators.second) {
+            rows.push_back(idx);
+        }
+    }
+
+    return rows;
+}
+
+std::vector<t_cellupd>
+t_ctx1::get_cell_delta(t_index bidx, t_index eidx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    eidx = std::min(eidx, t_index(m_traversal->size()));
+    std::vector<t_cellupd> rval;
+    const auto& deltas = m_tree->get_deltas();
+    for (t_index idx = bidx; idx < eidx; ++idx) {
+        t_index ptidx = m_traversal->get_tree_index(idx);
+        auto iterators = deltas->get<by_tc_nidx_aggidx>().equal_range(ptidx);
+        for (auto iter = iterators.first; iter != iterators.second; ++iter) {
+            rval.emplace_back(
+                idx, iter->m_aggidx + 1, iter->m_old_value, iter->m_new_value
+            );
+        }
+    }
+    return rval;
+}
+
+void
+t_ctx1::reset(bool reset_expressions) {
+    auto pivots = m_config.get_row_pivots();
+    m_tree = std::make_shared<t_stree>(
+        pivots, m_config.get_aggregates(), m_schema, m_config
+    );
+    m_tree->init();
+    m_tree->set_deltas_enabled(get_feature_state(CTX_FEAT_DELTA));
+    m_traversal = std::make_shared<t_traversal>(m_tree);
+    if (m_leaves_only) {
+        m_traversal->set_leaves_only(true, m_config.get_num_rpivots());
+    }
+
+    if (reset_expressions) {
+        m_expression_tables->reset();
+    }
+}
+
+void
+t_ctx1::reset_step_state() {
+    m_rows_changed = false;
+    m_columns_changed = false;
+    if (t_env::log_progress()) {
+        std::cout << "t_ctx1.reset_step_state " << repr() << '\n';
+    }
+}
+
+t_index
+t_ctx1::sidedness() const {
+    return 1;
+}
+
+std::vector<t_stree*>
+t_ctx1::get_trees() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    std::vector<t_stree*> rval(1);
+    rval[0] = m_tree.get();
+    return rval;
+}
+
+bool
+t_ctx1::has_deltas() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_tree->has_deltas();
+}
+
+void
+t_ctx1::pprint() const {
+    std::cout << "\t" << '\n';
+    for (auto idx = 1; idx < get_column_count(); ++idx) {
+        std::cout << get_aggregate(idx - 1).agg_str() << ", " << '\n';
+    }
+
+    std::vector<const t_column*> aggcols(m_config.get_num_aggregates());
+    auto* aggtable = m_tree->get_aggtable();
+    auto none = mknone();
+
+    for (t_uindex aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+         ++aggidx) {
+        // `aggidx` is the column's position in the aggtable schema, so resolve
+        // by index — avoids the per-iteration name->index map lookup + a temp
+        // std::string, and the up-front schema deep-copy.
+        aggcols[aggidx] = aggtable->_get_const_column(aggidx);
+    }
+
+    const std::vector<t_aggspec>& aggspecs = m_config.get_aggregates();
+
+    for (auto ridx = 0; ridx < get_row_count(); ++ridx) {
+        t_index nidx = m_traversal->get_tree_index(ridx);
+        t_index pnidx = m_tree->get_parent_idx(nidx);
+
+        t_uindex agg_ridx = m_tree->get_aggidx(nidx);
+        t_index agg_pridx =
+            pnidx == INVALID_INDEX ? INVALID_INDEX : m_tree->get_aggidx(pnidx);
+
+        std::cout << get_row_path(ridx) << " => ";
+        for (t_index aggidx = 0, loop_end = aggcols.size(); aggidx < loop_end;
+             ++aggidx) {
+            t_tscalar value = extract_aggregate(
+                aggspecs[aggidx], aggcols[aggidx], agg_ridx, agg_pridx
+            );
+            if (!value.is_valid()) {
+                value.set(none); // todo: fix null handling
+            }
+
+            std::cout << value << ", ";
+        }
+
+        std::cout << "\n";
+    }
+
+    std::cout << "=================" << '\n';
+}
+
+t_index
+t_ctx1::get_row_idx(const std::vector<t_tscalar>& path) const {
+    auto nidx = m_tree->resolve_path(0, path);
+    if (nidx == INVALID_INDEX) {
+        return nidx;
+    }
+
+    return m_traversal->get_traversal_index(nidx);
+}
+
+t_dtype
+t_ctx1::get_column_dtype(t_uindex idx) const {
+    if (idx == 0 || idx >= static_cast<t_uindex>(get_column_count())) {
+        return DTYPE_NONE;
+    }
+    return m_tree->get_aggtable()->get_const_column(idx - 1)->get_dtype();
+}
+
+t_depth
+t_ctx1::get_trav_depth(t_index idx) const {
+    return m_traversal->get_depth(idx);
+}
+
+void
+t_ctx1::compute_expressions(
+    const std::shared_ptr<t_data_table>& master,
+    const t_gstate::t_mapping& pkey_map,
+    t_expression_vocab& expression_vocab,
+    t_regex_mapping& regex_mapping
+) {
+    // Clear the transitional expression tables on the context so they are
+    // ready for the next update.
+    m_expression_tables->clear_transitional_tables();
+
+    std::shared_ptr<t_data_table> master_expression_table =
+        m_expression_tables->m_master;
+
+    // Set the master table to the right size.
+    t_uindex num_rows = master->size();
+    master_expression_table->reserve(num_rows);
+    master_expression_table->set_size(num_rows);
+
+    const auto& expressions = m_config.get_expressions();
+    for (const auto& expr : expressions) {
+        // Compute the expressions on the master table.
+        expr->compute(
+            master,
+            pkey_map,
+            master_expression_table,
+            expression_vocab,
+            regex_mapping
+        );
+    }
+}
+
+void
+t_ctx1::compute_expressions(
+    const std::shared_ptr<t_data_table>& master,
+    const t_gstate::t_mapping& pkey_map,
+    const std::shared_ptr<t_data_table>& flattened,
+    const std::shared_ptr<t_data_table>& delta,
+    const std::shared_ptr<t_data_table>& prev,
+    const std::shared_ptr<t_data_table>& current,
+    const std::shared_ptr<t_data_table>& transitions,
+    const std::shared_ptr<t_data_table>& existed,
+    t_expression_vocab& expression_vocab,
+    t_regex_mapping& regex_mapping
+) {
+    // Clear the tables so they are ready for this round of updates
+    m_expression_tables->clear_transitional_tables();
+
+    // All tables are the same size
+    t_uindex flattened_num_rows = flattened->size();
+    m_expression_tables->reserve_transitional_table_size(flattened_num_rows);
+    m_expression_tables->set_transitional_table_size(flattened_num_rows);
+
+    // Update the master expression table's size
+    t_uindex master_num_rows = master->size();
+    m_expression_tables->m_master->reserve(master_num_rows);
+    m_expression_tables->m_master->set_size(master_num_rows);
+
+    const auto& expressions = m_config.get_expressions();
+    for (const auto& expr : expressions) {
+        // master: compute based on latest state of the gnode state table
+        expr->compute(
+            master,
+            pkey_map,
+            m_expression_tables->m_master,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // flattened: compute based on the latest update dataset
+        expr->compute(
+            flattened,
+            pkey_map,
+            m_expression_tables->m_flattened,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // delta: for each numerical column, the numerical delta between the
+        // previous value and the current value in the row.
+        expr->compute(
+            delta,
+            pkey_map,
+            m_expression_tables->m_delta,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // prev: the values of the updated rows before this update was applied
+        expr->compute(
+            prev,
+            pkey_map,
+            m_expression_tables->m_prev,
+            expression_vocab,
+            regex_mapping
+        );
+
+        // current: the current values of the updated rows
+        expr->compute(
+            current,
+            pkey_map,
+            m_expression_tables->m_current,
+            expression_vocab,
+            regex_mapping
+        );
+    }
+
+    // Calculate the transitions now that the intermediate tables are computed
+    m_expression_tables->calculate_transitions(existed);
+}
+
+bool
+t_ctx1::is_expression_column(const std::string& colname) const {
+    const t_schema& schema = m_expression_tables->m_master->get_schema();
+    return schema.has_column(colname);
+}
+
+t_uindex
+t_ctx1::num_expressions() const {
+    const auto& expressions = m_config.get_expressions();
+    return expressions.size();
+}
+
+std::shared_ptr<t_expression_tables>
+t_ctx1::get_expression_tables() const {
+    return m_expression_tables;
+}
+
+std::vector<t_tscalar>
+t_ctx1::unity_get_row_data(t_uindex idx) const {
+    auto rval = get_data(idx, idx + 1, 0, get_column_count());
+    if (rval.empty()) {
+        return {};
+    }
+
+    return {rval.begin() + 1, rval.end()};
+}
+
+std::vector<t_tscalar>
+t_ctx1::unity_get_column_data(t_uindex idx) const {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+    return {};
+}
+
+std::vector<t_tscalar>
+t_ctx1::unity_get_row_path(t_uindex idx) const {
+    return get_row_path(idx);
+}
+
+std::vector<t_tscalar>
+t_ctx1::unity_get_column_path(t_uindex idx) const {
+    return {};
+}
+
+t_uindex
+t_ctx1::unity_get_row_depth(t_uindex ridx) const {
+    return m_traversal->get_depth(ridx);
+}
+
+t_uindex
+t_ctx1::unity_get_column_depth(t_uindex cidx) const {
+    return 0;
+}
+
+std::string
+t_ctx1::unity_get_column_name(t_uindex idx) const {
+    return m_config.unity_get_column_name(idx);
+}
+
+std::string
+t_ctx1::unity_get_column_display_name(t_uindex idx) const {
+    return m_config.unity_get_column_display_name(idx);
+}
+
+std::vector<std::string>
+t_ctx1::unity_get_column_names() const {
+    std::vector<std::string> rv;
+
+    for (t_uindex idx = 0, loop_end = unity_get_column_count(); idx < loop_end;
+         ++idx) {
+        rv.push_back(unity_get_column_name(idx));
+    }
+    return rv;
+}
+
+std::vector<std::string>
+t_ctx1::unity_get_column_display_names() const {
+    std::vector<std::string> rv;
+
+    for (t_uindex idx = 0, loop_end = unity_get_column_count(); idx < loop_end;
+         ++idx) {
+        rv.push_back(unity_get_column_display_name(idx));
+    }
+    return rv;
+}
+
+t_uindex
+t_ctx1::unity_get_column_count() const {
+    return get_column_count() - 1;
+}
+
+t_uindex
+t_ctx1::unity_get_row_count() const {
+    return get_row_count();
+}
+
+bool
+t_ctx1::unity_get_row_expanded(t_uindex idx) const {
+    return m_traversal->get_node_expanded(idx);
+}
+
+bool
+t_ctx1::unity_get_column_expanded(t_uindex idx) const {
+    return false;
+}
+
+void
+t_ctx1::clear_deltas() {
+    m_tree->clear_deltas();
+}
+
+void
+t_ctx1::unity_init_load_step_end() {}
+
+std::shared_ptr<t_data_table>
+t_ctx1::get_table() const {
+    const t_schema& schema = m_tree->get_aggtable()->get_schema();
+    const auto& pivots = m_config.get_row_pivots();
+    auto tbl = std::make_shared<t_data_table>(schema, m_tree->size());
+    tbl->init();
+    tbl->extend(m_tree->size());
+
+    std::vector<t_column*> aggcols = tbl->get_columns();
+    auto n_aggs = aggcols.size();
+    std::vector<t_column*> pivcols;
+
+    std::stringstream ss;
+    pivcols.reserve(pivots.size());
+    for (const auto& c : pivots) {
+        pivcols.push_back(
+            tbl->add_column(c.colname(), m_schema.get_dtype(c.colname()), true)
+        );
+    }
+
+    auto idx = 0;
+    for (auto nidx : m_tree->dfs()) {
+        auto depth = m_tree->get_depth(nidx);
+        if (depth > 0) {
+            pivcols[depth - 1]->set_scalar(idx, m_tree->get_value(nidx));
+        }
+        for (t_uindex aggnum = 0; aggnum < n_aggs; ++aggnum) {
+            auto aggscalar = m_tree->get_aggregate(nidx, aggnum);
+            aggcols[aggnum]->set_scalar(idx, aggscalar);
+        }
+        ++idx;
+    }
+    return tbl;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/context_unit.cpp

@@ -0,0 +1,497 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/context_base.h>
+#include <perspective/get_data_extents.h>
+#include <perspective/context_unit.h>
+#include <perspective/flat_traversal.h>
+#include <perspective/sym_table.h>
+
+#include <perspective/filter_utils.h>
+
+namespace perspective {
+
+t_ctxunit::t_ctxunit() = default;
+
+t_ctxunit::t_ctxunit(const t_schema& schema, const t_config& config) :
+    t_ctxbase<t_ctxunit>(schema, config),
+    m_has_delta(false) {}
+
+t_ctxunit::~t_ctxunit() = default;
+
+void
+t_ctxunit::init() {
+    m_init = true;
+}
+
+void
+t_ctxunit::step_begin() {
+    if (!m_init) {
+        return;
+    }
+
+    m_delta_pkeys.clear();
+    m_rows_changed = false;
+    m_columns_changed = false;
+}
+
+void
+t_ctxunit::step_end() {}
+
+/**
+ * @brief Notify the context with new data when the `t_gstate` master table is
+ * not empty, and being updated with new data.
+ *
+ * @param flattened
+ * @param delta
+ * @param prev
+ * @param curr
+ * @param transitions
+ * @param existed
+ */
+void
+t_ctxunit::notify(
+    const t_data_table& flattened,
+    const t_data_table& delta,
+    const t_data_table& prev,
+    const t_data_table& curr,
+    const t_data_table& transitions,
+    const t_data_table& existed
+) {
+    t_uindex nrecs = flattened.size();
+
+    std::shared_ptr<const t_column> pkey_sptr =
+        flattened.get_const_column("psp_pkey");
+    std::shared_ptr<const t_column> op_sptr =
+        flattened.get_const_column("psp_op");
+    const t_column* pkey_col = pkey_sptr.get();
+    const t_column* op_col = op_sptr.get();
+
+    bool delete_encountered = false;
+
+    for (t_uindex idx = 0; idx < nrecs; ++idx) {
+        // pkeys are always integers >= 0 - no need to use internal
+        // symtable to dereference.
+        t_tscalar pkey = pkey_col->get_scalar(idx);
+        std::uint8_t op_ = *(op_col->get_nth<std::uint8_t>(idx));
+        t_op op = static_cast<t_op>(op_);
+
+        switch (op) {
+            case OP_INSERT: {
+            } break;
+            case OP_DELETE: {
+                delete_encountered = true;
+            } break;
+            default: {
+                PSP_COMPLAIN_AND_ABORT("Unexpected OP");
+            } break;
+        }
+
+        // add the pkey for row delta
+        add_delta_pkey(pkey);
+    }
+
+    m_has_delta = !m_delta_pkeys.empty() || delete_encountered;
+}
+
+/**
+ * @brief Notify the context with new data after the `t_gstate`'s master table
+ * has been updated for the first time with data.
+ *
+ * @param flattened
+ */
+void
+t_ctxunit::notify(const t_data_table& flattened, bool is_registration) {
+    t_uindex nrecs = flattened.size();
+    std::shared_ptr<const t_column> pkey_sptr =
+        flattened.get_const_column("psp_pkey");
+    const t_column* pkey_col = pkey_sptr.get();
+
+    m_has_delta = true;
+
+    // During `_register_context`, no subscriber exists yet — skip the
+    // hopscotch_set insertions no observer can see. A unit context has no
+    // traversal/sort state to build either, so this is effectively O(1).
+    if (is_registration) {
+        return;
+    }
+
+    // TODO: pkey and idx are equal, except idx is not a t_tscalar. I don't
+    // think there is a difference between accessing the pkey column and
+    // creating a brand new scalar, as get_scalar always returns a copy. We
+    // could also simply store the row indices but there are some problems
+    // with correctness that I couldn't nail down.
+    for (t_uindex idx = 0; idx < nrecs; ++idx) {
+        t_tscalar pkey = pkey_col->get_scalar(idx);
+
+        // Add primary key to track row delta
+        add_delta_pkey(pkey);
+    }
+}
+
+std::pair<t_tscalar, t_tscalar>
+t_ctxunit::get_min_max(const std::string& colname) const {
+    auto col = m_gstate->get_table()->get_const_column(colname);
+    auto rval = std::make_pair(mknone(), mknone());
+    for (std::size_t i = 0; i < col->size(); i++) {
+        t_tscalar val = col->get_scalar(i);
+        if (!val.is_valid()) {
+            continue;
+        }
+
+        if (rval.first.is_none() || (!val.is_none() && val < rval.first)) {
+            rval.first = val;
+        }
+
+        if (val > rval.second) {
+            rval.second = val;
+        }
+    }
+
+    return rval;
+}
+
+/**
+ * @brief Given a start/end row and column, return the data for the subset.
+ *
+ * @param start_row
+ * @param end_row
+ * @param start_col
+ * @param end_col
+ * @return std::vector<t_tscalar>
+ */
+std::vector<t_tscalar>
+t_ctxunit::get_data(
+    t_index start_row, t_index end_row, t_index start_col, t_index end_col
+) const {
+    t_uindex ctx_nrows = get_row_count();
+    t_uindex ctx_ncols = get_column_count();
+
+    auto ext = sanitize_get_data_extents(
+        ctx_nrows, ctx_ncols, start_row, end_row, start_col, end_col
+    );
+
+    t_index num_rows = ext.m_erow - ext.m_srow;
+    t_index stride = ext.m_ecol - ext.m_scol;
+    std::vector<t_tscalar> values(num_rows * stride);
+
+    auto none = mknone();
+
+    const t_data_table& master_table = *(m_gstate->get_table());
+
+    for (t_index cidx = ext.m_scol; cidx < ext.m_ecol; ++cidx) {
+        const std::string& colname = m_config.col_at(cidx);
+
+        std::vector<t_tscalar> out_data(num_rows);
+
+        // Read directly from the row indices on the table - they will
+        // always correspond exactly.
+        m_gstate->read_column(
+            master_table, colname, start_row, end_row, out_data
+        );
+
+        for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) {
+            auto v = out_data[ridx - ext.m_srow];
+
+            // todo: fix null handling
+            if (!v.is_valid()) {
+                v.set(none);
+            }
+
+            values[(ridx - ext.m_srow) * stride + (cidx - ext.m_scol)] = v;
+        }
+    }
+
+    return values;
+}
+
+/**
+ * @brief Given a vector of row indices, which may not be contiguous,
+ * return the underlying data for these rows.
+ *
+ * @param rows a vector of row indices
+ * @return std::vector<t_tscalar> a vector of scalars containing data
+ */
+std::vector<t_tscalar>
+t_ctxunit::get_data(const std::vector<t_uindex>& rows) const {
+    t_uindex stride = get_column_count();
+    std::vector<t_tscalar> values(rows.size() * stride);
+
+    auto none = mknone();
+
+    const t_data_table& master_table = *(m_gstate->get_table());
+
+    for (t_uindex cidx = 0; cidx < stride; ++cidx) {
+        std::vector<t_tscalar> out_data(rows.size());
+        const std::string& colname = m_config.col_at(cidx);
+        m_gstate->read_column(master_table, colname, rows, out_data);
+
+        for (t_uindex ridx = 0; ridx < rows.size(); ++ridx) {
+            auto v = out_data[ridx];
+
+            if (!v.is_valid()) {
+                v.set(none);
+            }
+
+            values[(ridx)*stride + (cidx)] = v;
+        }
+    }
+
+    return values;
+}
+
+std::vector<t_tscalar>
+t_ctxunit::get_data(const std::vector<t_tscalar>& pkeys) const {
+    t_uindex stride = get_column_count();
+    std::vector<t_tscalar> values(pkeys.size() * stride);
+
+    auto none = mknone();
+
+    const t_data_table& master_table = *(m_gstate->get_table());
+
+    for (t_uindex cidx = 0; cidx < stride; ++cidx) {
+        std::vector<t_tscalar> out_data(pkeys.size());
+        const std::string& colname = m_config.col_at(cidx);
+
+        m_gstate->read_column(master_table, colname, pkeys, out_data);
+
+        for (t_uindex ridx = 0; ridx < pkeys.size(); ++ridx) {
+            auto v = out_data[ridx];
+
+            if (!v.is_valid()) {
+                v.set(none);
+            }
+
+            values[(ridx)*stride + (cidx)] = v;
+        }
+    }
+
+    return values;
+}
+
+/**
+ * @brief Returns a vector of primary keys for the specified cells,
+ * reading from the gnode_state's master table instead of from a traversal.
+ *
+ * @param cells
+ * @return std::vector<t_tscalar>
+ */
+std::vector<t_tscalar>
+t_ctxunit::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    // Validate cells
+    t_index num_rows = get_row_count();
+
+    for (const auto& cell : cells) {
+        t_index ridx = cell.first;
+        if (ridx >= num_rows) {
+            return {};
+        }
+    }
+
+    std::set<t_index> all_rows;
+
+    for (const auto& cell : cells) {
+        all_rows.insert(cell.first);
+    }
+
+    const t_data_table& master_table = *(m_gstate->get_table());
+    std::shared_ptr<const t_column> pkey_sptr =
+        master_table.get_const_column("psp_pkey");
+
+    std::vector<t_tscalar> rval(all_rows.size());
+
+    t_uindex i = 0;
+    for (auto ridx : all_rows) {
+        rval[i] = pkey_sptr->get_scalar(ridx);
+        i++;
+    }
+
+    return rval;
+}
+
+/**
+ * @brief Returns a string column name using the context's config.
+ *
+ * @param idx
+ * @return t_tscalar
+ */
+t_tscalar
+t_ctxunit::get_column_name(t_index idx) {
+    std::string empty;
+
+    if (idx >= get_column_count()) {
+        return m_symtable.get_interned_tscalar(empty.c_str());
+    }
+
+    return m_symtable.get_interned_tscalar(m_config.col_at(idx).c_str());
+}
+
+/**
+ * @brief Returns a `t_rowdelta` struct containing data from updated rows
+ * and the updated row indices.
+ *
+ * @return t_rowdelta
+ */
+t_rowdelta
+t_ctxunit::get_row_delta() {
+    bool rows_changed = m_rows_changed;
+    std::vector<t_tscalar> pkey_vector(
+        m_delta_pkeys.begin(), m_delta_pkeys.end()
+    );
+
+    // Sort pkeys - they will always be integers >= 0, as the table has
+    // no index set.
+    std::sort(pkey_vector.begin(), pkey_vector.end());
+
+    std::vector<t_tscalar> data = get_data(pkey_vector);
+    t_rowdelta rval(rows_changed, pkey_vector.size(), data);
+    clear_deltas();
+
+    return rval;
+}
+
+const tsl::hopscotch_set<t_tscalar>&
+t_ctxunit::get_delta_pkeys() const {
+    return m_delta_pkeys;
+}
+
+std::vector<std::string>
+t_ctxunit::get_column_names() const {
+    return m_schema.columns();
+}
+
+void
+t_ctxunit::reset() {
+    m_has_delta = false;
+}
+
+bool
+t_ctxunit::get_deltas_enabled() const {
+    return true;
+}
+
+void
+t_ctxunit::set_deltas_enabled(bool enabled_state) {}
+
+t_index
+t_ctxunit::sidedness() const {
+    return 0;
+}
+
+t_index
+t_ctxunit::get_row_count() const {
+    return m_gstate->num_rows();
+}
+
+t_index
+t_ctxunit::get_column_count() const {
+    return m_config.get_num_columns();
+}
+
+std::vector<t_tscalar>
+t_ctxunit::unity_get_row_data(t_uindex idx) const {
+    return get_data(idx, idx + 1, 0, get_column_count());
+}
+
+std::vector<t_tscalar>
+t_ctxunit::unity_get_row_path(t_uindex idx) const {
+    return {};
+}
+
+std::vector<t_tscalar>
+t_ctxunit::unity_get_column_path(t_uindex idx) const {
+    return {};
+}
+
+t_uindex
+t_ctxunit::unity_get_row_depth(t_uindex ridx) const {
+    return 0;
+}
+
+t_uindex
+t_ctxunit::unity_get_column_depth(t_uindex cidx) const {
+    return 0;
+}
+
+std::vector<std::string>
+t_ctxunit::unity_get_column_names() const {
+    return get_column_names();
+}
+
+t_uindex
+t_ctxunit::unity_get_column_count() const {
+    return get_column_count();
+}
+
+t_uindex
+t_ctxunit::unity_get_row_count() const {
+    return get_row_count();
+}
+
+bool
+t_ctxunit::unity_get_row_expanded(t_uindex idx) const {
+    return false;
+}
+
+bool
+t_ctxunit::unity_get_column_expanded(t_uindex idx) const {
+    return false;
+}
+
+/**
+ * @brief Mark a primary key as updated by adding it to the tracking set.
+ *
+ * @param pkey
+ */
+void
+t_ctxunit::add_delta_pkey(t_tscalar pkey) {
+    m_delta_pkeys.insert(pkey);
+}
+
+bool
+t_ctxunit::has_deltas() const {
+    return m_has_delta;
+}
+
+t_dtype
+t_ctxunit::get_column_dtype(t_uindex idx) const {
+    if (idx >= static_cast<t_uindex>(get_column_count())) {
+        return DTYPE_NONE;
+    }
+
+    auto cname = m_config.col_at(idx);
+
+    if (!m_schema.has_column(cname)) {
+        return DTYPE_NONE;
+    }
+
+    return m_schema.get_dtype(cname);
+}
+
+void
+t_ctxunit::clear_deltas() {
+    m_has_delta = false;
+}
+
+std::string
+t_ctxunit::repr() const {
+    std::stringstream ss;
+    ss << "t_ctxunit<" << this << ">";
+    return ss.str();
+}
+
+void
+t_ctxunit::pprint() const {}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/data.cpp

@@ -0,0 +1,30 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/data.h>
+
+namespace perspective {
+t_data::t_data() = default;
+
+t_data::t_data(const std::vector<t_tscalar>& data) : m_data(data) {}
+
+const std::vector<t_tscalar>&
+t_data::data() const {
+    return m_data;
+}
+
+std::vector<t_tscalar>&
+t_data::data() {
+    return m_data;
+}
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/data_slice.cpp

@@ -0,0 +1,201 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/data_slice.h>
+
+namespace perspective {
+
+template <typename CTX_T>
+t_data_slice<CTX_T>::t_data_slice(
+    std::shared_ptr<CTX_T> ctx,
+    t_uindex start_row,
+    t_uindex end_row,
+    t_uindex start_col,
+    t_uindex end_col,
+    t_uindex row_offset,
+    t_uindex col_offset,
+    const std::vector<t_tscalar>& slice,
+    const std::vector<std::vector<t_tscalar>>& column_names
+) :
+    m_ctx(ctx),
+    m_start_row(start_row),
+    m_end_row(end_row),
+    m_start_col(start_col),
+    m_end_col(end_col),
+    m_row_offset(row_offset),
+    m_col_offset(col_offset),
+    m_slice(slice),
+    m_column_names(column_names) {
+    m_stride = m_end_col - m_start_col;
+}
+
+template <typename CTX_T>
+t_data_slice<CTX_T>::t_data_slice(
+    std::shared_ptr<CTX_T> ctx,
+    t_uindex start_row,
+    t_uindex end_row,
+    t_uindex start_col,
+    t_uindex end_col,
+    t_uindex row_offset,
+    t_uindex col_offset,
+    const std::vector<t_tscalar>& slice,
+    const std::vector<std::vector<t_tscalar>>& column_names,
+    const std::vector<t_uindex>& column_indices
+) :
+    m_ctx(ctx),
+    m_start_row(start_row),
+    m_end_row(end_row),
+    m_start_col(start_col),
+    m_end_col(end_col),
+    m_row_offset(row_offset),
+    m_col_offset(col_offset),
+    m_slice(slice),
+    m_column_names(column_names),
+    m_column_indices(column_indices) {
+    m_stride = m_end_col - m_start_col;
+}
+
+template <typename CTX_T>
+t_data_slice<CTX_T>::~t_data_slice() = default;
+
+// Public API
+template <typename CTX_T>
+t_tscalar
+t_data_slice<CTX_T>::get(t_uindex ridx, t_uindex cidx) const {
+    ridx += m_row_offset;
+    t_uindex idx = get_slice_idx(ridx, cidx);
+    t_tscalar rv;
+    if (idx >= m_slice.size()) {
+        rv.clear();
+    } else {
+        rv = m_slice.operator[](idx);
+    }
+    return rv;
+}
+
+template <typename CTX_T>
+std::vector<t_tscalar>
+t_data_slice<CTX_T>::get_pkeys(t_uindex ridx, t_uindex cidx) const {
+    std::pair<t_uindex, t_uindex> pair{ridx, cidx};
+    std::vector<std::pair<t_uindex, t_uindex>> vec{pair};
+    return m_ctx->get_pkeys(vec);
+}
+
+template <typename CTX_T>
+std::vector<t_tscalar>
+t_data_slice<CTX_T>::get_column_slice(t_uindex cidx) const {
+    std::vector<t_tscalar> column_data;
+    column_data.reserve(m_end_row);
+
+    for (t_uindex ridx = 0; ridx < m_end_row; ++ridx) {
+        ridx += m_row_offset;
+        t_tscalar value = get(ridx, cidx);
+        column_data.push_back(value);
+    }
+
+    return column_data;
+}
+
+template <typename CTX_T>
+std::vector<t_tscalar>
+t_data_slice<CTX_T>::get_row_path(t_uindex ridx) const {
+    return m_ctx->unity_get_row_path(ridx);
+}
+
+template <typename CTX_T>
+t_uindex
+t_data_slice<CTX_T>::num_rows() const {
+    return m_end_row - m_start_row;
+}
+
+// Getters
+template <typename CTX_T>
+std::shared_ptr<CTX_T>
+t_data_slice<CTX_T>::get_context() const {
+    return m_ctx;
+}
+
+template <typename CTX_T>
+const std::vector<t_tscalar>&
+t_data_slice<CTX_T>::get_slice() const {
+    return m_slice;
+}
+
+template <typename CTX_T>
+const std::vector<std::vector<t_tscalar>>&
+t_data_slice<CTX_T>::get_column_names() const {
+    return m_column_names;
+}
+
+template <typename CTX_T>
+const std::vector<t_uindex>&
+t_data_slice<CTX_T>::get_column_indices() const {
+    return m_column_indices;
+}
+
+template <typename CTX_T>
+bool
+t_data_slice<CTX_T>::is_column_only() const {
+    return false;
+}
+
+template <>
+bool
+t_data_slice<t_ctx2>::is_column_only() const {
+    auto config = m_ctx->get_config();
+    return config.is_column_only();
+}
+
+template <typename CTX_T>
+t_uindex
+t_data_slice<CTX_T>::get_stride() const {
+    return m_stride;
+}
+
+template <typename CTX_T>
+t_uindex
+t_data_slice<CTX_T>::get_row_offset() const {
+    return m_row_offset;
+}
+
+template <typename CTX_T>
+t_uindex
+t_data_slice<CTX_T>::get_col_offset() const {
+    return m_col_offset;
+}
+
+template <typename CTX_T>
+t_get_data_extents
+t_data_slice<CTX_T>::get_data_extents() const {
+    auto nrows = m_ctx->get_row_count();
+    auto ncols = m_ctx->get_column_count();
+    t_get_data_extents ext = sanitize_get_data_extents(
+        nrows, ncols, m_start_row, m_end_row, m_start_col, m_end_col
+    );
+    return ext;
+}
+
+// Private
+template <typename CTX_T>
+t_uindex
+t_data_slice<CTX_T>::get_slice_idx(t_uindex ridx, t_uindex cidx) const {
+    t_uindex idx = (ridx - m_start_row) * m_stride + (cidx - m_start_col);
+    return idx;
+}
+
+// Explicitly instantiate data slice for each context
+template class t_data_slice<t_ctxunit>;
+template class t_data_slice<t_ctx0>;
+template class t_data_slice<t_ctx1>;
+template class t_data_slice<t_ctx2>;
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/data_table.cpp

@@ -0,0 +1,936 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/data_table.h>
+#include <perspective/column.h>
+#include <perspective/storage.h>
+#include <perspective/scalar.h>
+#include <perspective/tracing.h>
+#include <perspective/utils.h>
+
+#include <sstream>
+#include <utility>
+namespace perspective {
+
+void
+t_data_table::set_capacity(t_uindex idx) {
+    m_capacity = idx;
+#ifdef PSP_TABLE_VERIFY
+    if (m_init) {
+        for (auto c : m_columns) {
+            c->verify_size();
+            c->verify_size(m_capacity);
+        }
+    }
+
+#endif
+}
+
+t_data_table::t_data_table(t_schema s, t_uindex init_cap) :
+    m_schema(std::move(s)),
+    m_size(0),
+    m_backing_store(BACKING_STORE_MEMORY),
+    m_init(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_data_table");
+    set_capacity(init_cap);
+}
+
+t_data_table::t_data_table(
+    std::string name,
+    std::string dirname,
+    t_schema s,
+    t_uindex init_cap,
+    t_backing_store backing_store
+) :
+    m_name(std::move(name)),
+    m_dirname(std::move(dirname)),
+    m_schema(std::move(s)),
+    m_size(0),
+    m_backing_store(backing_store),
+    m_init(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_data_table");
+    set_capacity(init_cap);
+}
+
+// THIS CONSTRUCTOR INITS. Do not use in production.
+t_data_table::t_data_table(
+    const t_schema& s, const std::vector<std::vector<t_tscalar>>& v
+) :
+    m_schema(s),
+    m_size(0),
+    m_backing_store(BACKING_STORE_MEMORY),
+    m_init(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_data_table");
+    auto ncols = s.size();
+    PSP_VERBOSE_ASSERT(
+        std::all_of(
+            v.begin(),
+            v.end(),
+            [ncols](const std::vector<t_tscalar>& vec) {
+                return vec.size() == ncols;
+            }
+        ),
+        "Mismatched row size found"
+    );
+    set_capacity(v.size());
+    init();
+    extend(v.size());
+    std::vector<t_column*> cols = get_columns();
+    for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
+        auto* col = cols[cidx];
+        for (t_uindex ridx = 0, loop_end = v.size(); ridx < loop_end; ++ridx) {
+            col->set_scalar(ridx, v[ridx][cidx]);
+        }
+    }
+}
+
+t_data_table::~t_data_table() {
+    PSP_TRACE_SENTINEL();
+    LOG_DESTRUCTOR("t_data_table");
+}
+
+const std::string&
+t_data_table::name() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_name;
+}
+
+void
+t_data_table::init(bool make_columns) {
+    PSP_TRACE_SENTINEL();
+    LOG_INIT("t_data_table");
+    m_columns = std::vector<std::shared_ptr<t_column>>(m_schema.size());
+
+    if (make_columns) {
+        for (t_uindex idx = 0; idx < int(m_schema.size()); ++idx) {
+            const std::string& colname = m_schema.m_columns[idx];
+            t_dtype dtype = m_schema.m_types[idx];
+            m_columns[idx] =
+                make_column(colname, dtype, m_schema.m_status_enabled[idx]);
+            m_columns[idx]->init();
+        }
+    }
+
+    m_init = true;
+}
+
+std::shared_ptr<t_column>
+t_data_table::make_column(
+    const std::string& colname, t_dtype dtype, bool status_enabled
+) {
+    t_lstore_recipe a(
+        m_dirname,
+        m_name + std::string("_") + colname,
+        m_capacity * get_dtype_size(dtype),
+        m_backing_store
+    );
+    return std::make_shared<t_column>(dtype, status_enabled, a, m_capacity);
+}
+
+t_uindex
+t_data_table::num_columns() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_schema.size();
+}
+
+t_uindex
+t_data_table::num_rows() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_size;
+}
+
+t_uindex
+t_data_table::size() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return num_rows();
+}
+
+t_dtype
+t_data_table::get_dtype(const std::string& colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_schema.get_dtype(colname);
+}
+
+std::shared_ptr<t_column>
+t_data_table::get_column_safe(std::string_view colname) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx_safe(colname.data());
+    if (idx == -1) {
+        return nullptr;
+    }
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<t_column>
+t_data_table::get_column(std::string_view colname) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx(colname.data());
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<t_column>
+t_data_table::get_column(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx(colname.data());
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<t_column>
+t_data_table::get_column_safe(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx_safe(colname.data());
+    if (idx == -1) {
+        return nullptr;
+    }
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<const t_column>
+t_data_table::get_const_column(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx(colname.data());
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<const t_column>
+t_data_table::get_const_column_safe(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx_safe(colname.data());
+    if (idx == -1) {
+        return nullptr;
+    }
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<const t_column>
+t_data_table::get_const_column(t_uindex idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::shared_ptr<const t_column>
+t_data_table::get_const_column_safe(t_uindex idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    if (idx == -1) {
+        return nullptr;
+    }
+    ensure_col_resident(idx);
+    return m_columns[idx];
+}
+
+std::vector<t_column*>
+t_data_table::get_columns() {
+    std::vector<t_column*> rval(m_columns.size());
+    t_uindex idx = 0;
+    for (const auto& c : m_columns) {
+        ensure_col_resident(idx);
+        rval[idx] = c.get();
+        ++idx;
+    }
+    return rval;
+}
+
+std::vector<const t_column*>
+t_data_table::get_const_columns() const {
+    std::vector<const t_column*> rval(m_columns.size());
+    t_uindex idx = 0;
+    for (const auto& c : m_columns) {
+        ensure_col_resident(idx);
+        rval[idx] = c.get();
+        ++idx;
+    }
+    return rval;
+}
+
+void
+t_data_table::extend(t_uindex nelems) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    PSP_VERBOSE_ASSERT(m_init, "Table not inited");
+    for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
+        // Restore before reallocating: extend/reserve realloc `m_base`, which on
+        // an evicted store would discard its disk-resident data.
+        ensure_col_resident(idx);
+        m_columns[idx]->extend_dtype(nelems);
+    }
+    m_size = std::max(nelems, m_size);
+    set_capacity(std::max(nelems, m_capacity));
+}
+
+void
+t_data_table::set_size(t_uindex size) {
+    PSP_TRACE_SENTINEL();
+    for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
+        ensure_col_resident(idx);
+        m_columns[idx]->set_size(size);
+    }
+    m_size = size;
+}
+
+void
+t_data_table::set_table_size(t_uindex size) {
+    m_size = size;
+}
+
+void
+t_data_table::reserve(t_uindex capacity) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    for (t_uindex idx = 0, loop_end = m_schema.size(); idx < loop_end; ++idx) {
+        ensure_col_resident(idx);
+        m_columns[idx]->reserve(capacity);
+    }
+    set_capacity(std::max(capacity, m_capacity));
+}
+
+t_column*
+t_data_table::_get_column(std::string_view colname) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx(colname.data());
+    ensure_col_resident(idx);
+    return m_columns[idx].get();
+}
+
+const t_column*
+t_data_table::_get_const_column(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx(colname.data());
+    ensure_col_resident(idx);
+    return m_columns[idx].get();
+}
+
+const t_column*
+t_data_table::_get_const_column(t_uindex idx) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    ensure_col_resident(idx);
+    return m_columns[idx].get();
+}
+
+const t_column*
+t_data_table::_get_const_column_safe(std::string_view colname) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex idx = m_schema.get_colidx_safe(colname.data());
+    if (idx == t_uindex(-1)) {
+        return nullptr;
+    }
+    ensure_col_resident(idx);
+    return m_columns[idx].get();
+}
+
+const t_schema&
+t_data_table::get_schema() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_schema;
+}
+
+std::shared_ptr<t_data_table>
+t_data_table::flatten() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    PSP_VERBOSE_ASSERT(is_pkey_table(), "Not a pkeyed table");
+    std::shared_ptr<t_data_table> flattened = std::make_shared<t_data_table>(
+        "", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    flattened->init();
+    flatten_body<std::shared_ptr<t_data_table>>(flattened);
+    return flattened;
+}
+
+bool
+t_data_table::is_pkey_table() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_schema.is_pkey();
+}
+
+bool
+t_data_table::is_same_shape(t_data_table& tbl) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_schema == tbl.m_schema;
+}
+
+void
+t_data_table::pprint() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    pprint(size(), &std::cout);
+}
+
+// void
+// t_data_table::pprint(const std::string& fname) const {
+
+//     PSP_TRACE_SENTINEL();
+//     PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+//     std::ofstream file;
+//     file.open(fname);
+//     pprint(size(), &file);
+// }
+
+void
+t_data_table::pprint(t_uindex nrows, std::ostream* os) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (os == nullptr) {
+        os = &std::cout;
+    }
+
+    t_uindex nrows_ = nrows != 0U ? nrows : num_rows();
+    nrows_ = std::min(nrows_, num_rows());
+
+    t_uindex ncols = num_columns();
+
+    std::vector<const t_column*> columns(ncols);
+    for (t_uindex idx = 0; idx < ncols; ++idx) {
+        columns[idx] = m_columns[idx].get();
+        (*os) << m_schema.m_columns[idx] << ", ";
+    }
+
+    (*os) << '\n';
+    (*os) << "==========================" << '\n';
+
+    for (t_uindex ridx = 0; ridx < nrows_; ++ridx) {
+        for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
+            (*os) << columns[cidx]->get_scalar(ridx).to_string() << ", ";
+        }
+        (*os) << '\n';
+    }
+}
+
+void
+t_data_table::pprint(const std::vector<t_uindex>& vec) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_uindex nrows = vec.size();
+    t_uindex ncols = num_columns();
+
+    std::vector<const t_column*> columns(ncols);
+    for (t_uindex idx = 0; idx < ncols; ++idx) {
+        columns[idx] = m_columns[idx].get();
+        std::cout << m_schema.m_columns[idx] << ", ";
+    }
+
+    std::cout << '\n';
+    std::cout << "==========================" << '\n';
+
+    for (t_uindex ridx = 0; ridx < nrows; ++ridx) {
+        for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
+            std::cout << columns[cidx]->get_scalar(vec[ridx]) << ", ";
+        }
+        std::cout << '\n';
+    }
+}
+
+void
+t_data_table::append(const t_data_table& other) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    t_uindex cursize = size();
+
+    std::vector<const t_column*> src_cols;
+    std::vector<t_column*> dst_cols;
+
+    src_cols.reserve(other.m_schema.m_columns.size());
+    dst_cols.reserve(m_schema.m_columns.size());
+
+    std::set<std::string> incoming;
+
+    for (const auto& cname : other.m_schema.m_columns) {
+        t_dtype col_dtype = get_column(cname)->get_dtype();
+        t_dtype other_col_dtype = other.get_const_column(cname)->get_dtype();
+        if (col_dtype != other_col_dtype) {
+            std::stringstream ss;
+            ss << "Mismatched dtypes for `" << cname
+               << "`: attempted to append column of dtype `"
+               << get_dtype_descr(other_col_dtype)
+               << "` to existing column of dtype `"
+               << get_dtype_descr(col_dtype) << "`" << '\n';
+            std::cout << ss.str();
+            PSP_COMPLAIN_AND_ABORT(ss.str())
+        }
+        src_cols.push_back(other._get_const_column(cname));
+        dst_cols.push_back(_get_column(cname));
+        incoming.insert(cname);
+    }
+    t_uindex other_size = other.num_rows();
+
+    for (const auto& cname : m_schema.m_columns) {
+        if (incoming.find(cname) == incoming.end()) {
+            get_column(cname)->extend_dtype(cursize + other_size);
+        }
+    }
+
+    for (t_uindex colidx = 0; colidx < int(src_cols.size()); ++colidx) {
+        dst_cols[colidx]->append(*(src_cols[colidx]));
+    }
+
+    set_capacity(std::max(m_capacity, m_size + other.num_rows()));
+    set_size(m_size + other.num_rows());
+}
+
+void
+t_data_table::clear() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    for (const auto& m_column : m_columns) {
+        m_column->clear();
+    }
+    m_size = 0;
+}
+
+t_mask
+t_data_table::filter_cpp(
+    t_filter_op combiner, const std::vector<t_fterm>& fterms_
+) const {
+    auto* self = const_cast<t_data_table*>(this);
+    auto fterms = fterms_;
+
+    t_mask mask(size());
+    t_uindex fterm_size = fterms.size();
+    std::vector<t_uindex> indices(fterm_size);
+    std::vector<const t_column*> columns(fterm_size);
+
+    for (t_uindex idx = 0; idx < fterm_size; ++idx) {
+        indices[idx] = m_schema.get_colidx(fterms[idx].m_colname);
+        columns[idx] = _get_const_column(fterms[idx].m_colname);
+        fterms[idx].coerce_numeric(columns[idx]->get_dtype());
+        if (fterms[idx].m_use_interned) {
+            t_tscalar& thr = fterms[idx].m_threshold;
+            auto col = self->get_column(fterms[idx].m_colname);
+            auto interned = col->get_interned(thr.get_char_ptr());
+            thr.set(interned);
+        }
+    }
+
+    switch (combiner) {
+        case FILTER_OP_AND: {
+            t_tscalar cell_val;
+
+            for (t_uindex ridx = 0, rloop_end = size(); ridx < rloop_end;
+                 ++ridx) {
+                bool pass = true;
+
+                for (t_uindex cidx = 0; cidx < fterm_size; ++cidx) {
+                    if (!pass) {
+                        break;
+                    }
+
+                    // TODO we can make this faster by not constructing these on
+                    // every iteration?
+
+                    const auto& ft = fterms[cidx];
+                    bool tval;
+
+                    if (ft.m_use_interned) {
+                        cell_val.set(*(columns[cidx]->get_nth<t_uindex>(ridx)));
+                        cell_val.set_status(*(columns[cidx]->get_nth_status(ridx
+                        )));
+                    } else {
+                        cell_val = columns[cidx]->get_scalar(ridx);
+                    }
+
+                    tval = ft(cell_val);
+                    if (!tval) {
+                        pass = false;
+                        break;
+                    }
+                }
+
+                mask.set(ridx, pass);
+            }
+        } break;
+        case FILTER_OP_OR: {
+            for (t_uindex ridx = 0, rloop_end = size(); ridx < rloop_end;
+                 ++ridx) {
+                bool pass = false;
+                for (t_uindex cidx = 0; cidx < fterm_size; ++cidx) {
+                    t_tscalar cell_val = columns[cidx]->get_scalar(ridx);
+                    if (fterms[cidx](cell_val)) {
+                        pass = true;
+                        break;
+                    }
+                }
+                mask.set(ridx, pass);
+            }
+        } break;
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unknown filter op");
+        } break;
+    }
+
+    return mask;
+}
+
+t_uindex
+t_data_table::get_capacity() const {
+    return m_capacity;
+}
+
+t_data_table*
+t_data_table::clone_(const t_mask& mask) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_schema schema = m_schema;
+
+    auto* rval = new t_data_table("", "", schema, 5, BACKING_STORE_MEMORY);
+    rval->init();
+
+    for (const auto& cname : schema.m_columns) {
+        rval->set_column(cname, get_const_column(cname)->clone(mask));
+    }
+
+    rval->set_size(mask.count());
+    return rval;
+}
+
+std::shared_ptr<t_data_table>
+t_data_table::clone(const t_mask& mask) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    auto* tbl = clone_(mask);
+    return std::shared_ptr<t_data_table>(tbl);
+}
+
+std::shared_ptr<t_data_table>
+t_data_table::clone() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    t_schema schema = m_schema;
+    auto rval =
+        std::make_shared<t_data_table>("", "", schema, 5, BACKING_STORE_MEMORY);
+    rval->init();
+
+    for (const auto& cname : schema.m_columns) {
+        rval->set_column(cname, get_const_column(cname)->clone());
+    }
+    rval->set_size(size());
+    return rval;
+}
+
+std::shared_ptr<t_data_table>
+t_data_table::borrow(const std::vector<std::string>& columns) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    std::vector<t_dtype> dtypes;
+    dtypes.reserve(columns.size());
+
+    for (const auto& col : columns) {
+        dtypes.push_back(m_schema.get_dtype(col));
+    }
+
+    t_schema borrowed_schema = t_schema(columns, dtypes);
+    auto rval = std::make_shared<t_data_table>(
+        "", "", borrowed_schema, 5, BACKING_STORE_MEMORY
+    );
+    rval->init();
+
+    for (const auto& cname : borrowed_schema.m_columns) {
+        rval->set_column(cname, get_column(cname));
+    }
+
+    rval->set_size(size());
+    return rval;
+}
+
+std::shared_ptr<t_data_table>
+t_data_table::join(const std::shared_ptr<t_data_table>& other_table) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (size() != other_table->size()) {
+        std::stringstream ss;
+        ss << "[t_data_table::join] Cannot join two tables of unequal sizes! "
+              "Current size: "
+           << size() << ", size of other table: " << other_table->size()
+           << '\n';
+        PSP_COMPLAIN_AND_ABORT(ss.str());
+    }
+
+    // join both schemas
+    t_schema schema = m_schema;
+    const t_schema& other_schema = other_table->get_schema();
+    std::vector<std::string> other_columns;
+
+    for (const std::string& column : other_schema.m_columns) {
+        // Only append unique columns
+        if (!schema.has_column(column)) {
+            schema.add_column(column, other_schema.get_dtype(column));
+            other_columns.push_back(column);
+        }
+    }
+
+    std::shared_ptr<t_data_table> rval = std::make_shared<t_data_table>(
+        "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+
+    // init() without initializing the t_columns on the returned table, as
+    // they will be immediately replaced by the columns from the tables
+    // we are joining together.
+    rval->init(false);
+
+    // borrow columns from the current table
+    for (const std::string& column : m_schema.m_columns) {
+        rval->set_column(column, get_column(column));
+    }
+
+    // and the columns we need from the other table
+    for (const std::string& column : other_columns) {
+        rval->set_column(column, other_table->get_column(column));
+    }
+
+    // Set size and capacity on the table only - don't mutate any of the
+    // columns.
+    rval->set_table_size(size());
+    rval->set_capacity(std::max(get_capacity(), other_table->get_capacity()));
+
+    return rval;
+}
+
+std::shared_ptr<t_column>
+t_data_table::add_column_sptr(
+    const std::string& name, t_dtype dtype, bool status_enabled
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (m_schema.has_column(name)) {
+        return m_columns.at(m_schema.get_colidx(name));
+    }
+    m_schema.add_column(name, dtype);
+    m_columns.push_back(make_column(name, dtype, status_enabled));
+    m_columns.back()->init();
+    m_columns.back()->reserve(
+        std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
+    );
+    m_columns.back()->set_size(size());
+    return m_columns.back();
+}
+
+t_column*
+t_data_table::add_column(
+    const std::string& name, t_dtype dtype, bool status_enabled
+) {
+    return add_column_sptr(name, dtype, status_enabled).get();
+}
+
+void
+t_data_table::drop_column(const std::string& name) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (m_schema.has_column(name)) {
+        auto idx = m_schema.get_colidx(name);
+        auto col = m_columns[idx];
+        // FIXME: make sure that we can erase columns from m_schema eventually.
+        m_columns[idx]->clear();
+    }
+}
+
+void
+t_data_table::promote_column(
+    std::string_view col_name,
+    t_dtype new_dtype,
+    std::int32_t iter_limit,
+    bool fill
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    const auto* name = col_name.data();
+
+    if (!m_schema.has_column(name)) {
+        std::cout << "Cannot promote a column that does not exist." << '\n';
+        return;
+    }
+
+    t_dtype current_dtype = m_schema.get_dtype(name);
+    if (current_dtype == new_dtype) {
+        return;
+    }
+
+    t_uindex idx = m_schema.get_colidx(name);
+    std::shared_ptr<t_column> current_col = m_columns[idx];
+
+    // create the new column and copy data
+    std::shared_ptr<t_column> promoted_col =
+        make_column(name, new_dtype, current_col->is_status_enabled());
+    promoted_col->init();
+    promoted_col->reserve(
+        std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
+    );
+    promoted_col->set_size(size());
+
+    if (fill) {
+        for (auto i = 0; i < iter_limit; ++i) {
+            switch (new_dtype) {
+                case DTYPE_INT64: {
+                    auto* val = current_col->get_nth<std::int32_t>(i);
+                    auto fval = static_cast<std::int64_t>(*val);
+                    promoted_col->set_nth(i, fval);
+                } break;
+                case DTYPE_FLOAT64: {
+                    auto* val = current_col->get_nth<std::int32_t>(i);
+                    auto fval = static_cast<double>(*val);
+                    promoted_col->set_nth(i, fval);
+                } break;
+                case DTYPE_STR: {
+                    auto* val = current_col->get_nth<std::int32_t>(i);
+                    std::string fval = std::to_string(*val);
+                    promoted_col->set_nth(i, fval);
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT(
+                        "Columns can only be promoted to "
+                        "int64, float64, or string type."
+                    );
+                }
+            }
+        }
+    }
+
+    // finally, mutate schema and columns
+    m_schema.retype_column(name, new_dtype);
+    set_column(idx, promoted_col);
+}
+
+void
+t_data_table::set_column(t_uindex idx, std::shared_ptr<t_column> col) {
+    m_columns[idx] = std::move(col);
+}
+
+void
+t_data_table::set_column(
+    const std::string& name, std::shared_ptr<t_column> col
+) {
+    t_uindex idx = m_schema.get_colidx(name);
+    set_column(idx, std::move(col));
+}
+
+t_column*
+t_data_table::clone_column(
+    const std::string& existing_col, const std::string& new_colname
+) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    if (!m_schema.has_column(existing_col)) {
+        std::cout << "Cannot clone non existing column: " << existing_col
+                  << '\n';
+        return nullptr;
+    }
+
+    t_uindex idx = m_schema.get_colidx(existing_col);
+
+    m_schema.add_column(new_colname, m_columns[idx]->get_dtype());
+    m_columns.push_back(m_columns[idx]->clone());
+    m_columns.back()->reserve(
+        std::max(size(), std::max(static_cast<t_uindex>(8), m_capacity))
+    );
+    m_columns.back()->set_size(size());
+    return m_columns.back().get();
+}
+
+std::string
+t_data_table::repr() const {
+    std::stringstream ss;
+    ss << "t_data_table<" << this << ">";
+    return ss.str();
+}
+
+void
+t_data_table::verify() const {
+    for (const auto& c : m_columns) {
+        c->verify_size(m_capacity);
+        c->verify();
+    }
+
+    for (const auto& c : m_columns) {
+        PSP_VERBOSE_ASSERT(
+            c, || (size() == c->size()), "Ragged table encountered"
+        );
+    }
+}
+
+void
+t_data_table::reset() {
+    for (const auto& m_column : m_columns) {
+        if (m_column->get_dtype() == DTYPE_OBJECT) {
+            m_column->clear_objects();
+        }
+        m_column->clear();
+    }
+    m_size = 0;
+    m_capacity = DEFAULT_EMPTY_CAPACITY;
+    init();
+}
+
+std::vector<t_tscalar>
+t_data_table::get_scalvec() const {
+    auto nrows = size();
+    auto cols = get_const_columns();
+    auto ncols = cols.size();
+    std::vector<t_tscalar> rv;
+    rv.reserve(nrows * ncols);
+    for (t_uindex idx = 0; idx < nrows; ++idx) {
+        for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
+            rv.push_back(cols[cidx]->get_scalar(idx));
+        }
+    }
+    return rv;
+}
+
+std::shared_ptr<t_column>
+t_data_table::operator[](const std::string& name) {
+    if (!m_schema.has_column(name)) {
+        return {nullptr};
+    }
+    return m_columns[m_schema.get_colidx(name)];
+}
+
+bool
+operator==(const t_data_table& lhs, const t_data_table& rhs) {
+    return lhs.get_scalvec() == rhs.get_scalvec();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/date.cpp

@@ -0,0 +1,172 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/date.h>
+#include <perspective/time.h>
+
+namespace perspective {
+
+t_date::t_date() : m_storage(0) {}
+
+void
+t_date::set_psp_date(t_uindex dt)
+
+{
+    m_storage = t_rawtype(dt);
+}
+
+t_date::t_date(std::int16_t year, std::int8_t month, std::int8_t day) {
+    set_year_month_day(year, month, day);
+}
+
+void
+t_date::set_year_month_day(
+    std::int16_t year, std::int8_t month, std::int8_t day
+) {
+    set_year(year);
+    set_month(month);
+    set_day(day);
+}
+
+void
+t_date::set_year(std::int16_t year) {
+    m_storage = (m_storage & ~YEAR_MASK) | (year << YEAR_SHIFT);
+}
+void
+t_date::set_month(std::int8_t month) {
+    m_storage = (m_storage & ~MONTH_MASK) | (month << MONTH_SHIFT);
+}
+void
+t_date::set_day(std::int8_t day) {
+    m_storage = (m_storage & ~DAY_MASK) | (day << DAY_SHIFT);
+}
+
+t_date::t_date(std::uint32_t raw_val) : m_storage(raw_val) {}
+
+std::uint32_t
+t_date::raw_value() const {
+    return m_storage;
+}
+// Index such that
+// a.consecutive_day_idx()-b.consecutive_day_idx() is
+// number of days a is after b.
+//(Start point is unspecified, may not be stable and
+// only works for dates after 1900AD.)
+std::int32_t
+t_date::consecutive_day_idx() const {
+    std::int32_t m = month();
+    std::int32_t y = year();
+    std::int32_t yP = y - 1;
+    std::int32_t leap_selector =
+        (y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 1 : 0;
+    return day() + 365 * y + yP / 4 - yP / 100 + yP / 400
+        + CUMULATIVE_DAYS[leap_selector][m - 1];
+}
+
+std::int64_t
+t_date::as_epoch_ms() const {
+    // `to_gmtime` months are [1-12], `t_date` months are [0-11]
+    return to_gmtime(year(), month() + 1, day(), 0, 0, 0) * 1000;
+}
+
+t_date
+t_date::from_epoch_ms(std::int64_t ms) {
+    std::tm t = gmtime_from_epoch_ms(ms);
+    return {
+        static_cast<std::int16_t>(t.tm_year + 1900),
+        static_cast<std::int8_t>(t.tm_mon),
+        static_cast<std::int8_t>(t.tm_mday)
+    };
+}
+
+t_date
+from_consecutive_day_idx(std::int32_t idx) {
+    auto y = static_cast<std::int32_t>(idx / 365.2425);
+    std::int32_t yP = y - 1;
+    std::int32_t idx_year_removed =
+        idx - (y * 365 + yP / 4 - yP / 100 + yP / 400);
+    std::int32_t tgt_substraction =
+        (y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 366 : 365;
+    if (idx_year_removed > tgt_substraction) {
+        idx_year_removed -= tgt_substraction;
+        y += 1;
+    }
+    std::int32_t yearkind =
+        (y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)) ? 1 : 0;
+    const std::int32_t* const pos = std::lower_bound(
+                                        CUMULATIVE_DAYS[yearkind],
+                                        CUMULATIVE_DAYS[yearkind] + 13,
+                                        idx_year_removed
+                                    )
+        - 1;
+    // NOLINTNEXTLINE
+    return t_date(
+        y,
+        std::distance(CUMULATIVE_DAYS[yearkind], pos) + 1,
+        idx_year_removed - *pos /*+1*/
+    );
+}
+
+bool
+operator<(const t_date& a, const t_date& b) {
+    return a.m_storage < b.m_storage;
+}
+bool
+operator<=(const t_date& a, const t_date& b) {
+    return a.m_storage <= b.m_storage;
+}
+bool
+operator>(const t_date& a, const t_date& b) {
+    return a.m_storage > b.m_storage;
+}
+bool
+operator>=(const t_date& a, const t_date& b) {
+    return a.m_storage >= b.m_storage;
+}
+bool
+operator==(const t_date& a, const t_date& b) {
+    return a.m_storage == b.m_storage;
+}
+bool
+operator!=(const t_date& a, const t_date& b) {
+    return a.m_storage != b.m_storage;
+}
+
+std::int32_t
+t_date::year() const {
+    return ((m_storage & YEAR_MASK) >> YEAR_SHIFT);
+}
+std::int32_t
+t_date::month() const {
+    return ((m_storage & MONTH_MASK) >> MONTH_SHIFT);
+}
+std::int32_t
+t_date::day() const {
+    return ((m_storage & DAY_MASK) >> DAY_SHIFT);
+}
+std::string
+t_date::str() const {
+    std::stringstream ss;
+    ss << year() << "-" << str_(month() + 1) << "-" << str_(day());
+    return ss.str();
+}
+
+} // end namespace perspective
+
+namespace std {
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_date& t) {
+    os << t.str();
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/date_parser.cpp

@@ -0,0 +1,54 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <sstream>
+#include <perspective/first.h>
+#include <perspective/date_parser.h>
+#include <locale>
+#include <iomanip>
+
+namespace perspective {
+
+// Milliseconds & timezones are not currently handled
+const std::string t_date_parser::VALID_FORMATS[12] = {
+    "%Y%m%dT%H%M%S", // ISO "%Y%m%dT%H%M%S%F%q"
+    "%Y-%m-%dT%H:%M:%S",
+    "%Y-%m-%d %H:%M:%S",     // ISO extended
+    "%A, %d %b %Y %H:%M:%S", // RFC 0822
+    "%Y-%m-%d\\%H:%M:%S"
+    "%m-%d-%Y",
+    "%m/%d/%Y",
+    "%m-%d-%Y",
+    "%m %d %Y",
+    "%m/%d/%Y",
+    "%m/%d/%y",
+    "%d %m %Y"
+};
+
+t_date_parser::t_date_parser() {}
+
+bool
+t_date_parser::is_valid(std::string const& datestring) {
+    for (const std::string& fmt : VALID_FORMATS) {
+        if (fmt != "") {
+            std::tm t = {};
+            std::stringstream ss(datestring);
+            ss.imbue(std::locale::classic());
+            ss >> std::get_time(&t, fmt.c_str());
+            if (!ss.fail()) {
+                return true;
+            }
+        }
+    }
+    return false;
+}
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/dense_nodes.cpp

@@ -0,0 +1,46 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/dense_nodes.h>
+
+namespace std {
+
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_dense_tnode& s) {
+    std::cout << "t_dtnode<idx:" << s.m_idx << " pidx:" << s.m_pidx
+              << " fcidx:" << s.m_fcidx << " nchild:" << s.m_nchild
+              << " flidx:" << s.m_flidx << " nleaves:" << s.m_nleaves << ">";
+    return os;
+}
+} // namespace std
+namespace perspective {
+
+void
+fill_dense_tnode(
+    t_dense_tnode* node,
+    t_uindex idx,
+    t_uindex pidx,
+    t_uindex fcidx,
+    t_uindex nchild,
+    t_uindex flidx,
+    t_uindex nleaves
+) {
+    node->m_idx = idx;
+    node->m_pidx = pidx;
+    node->m_fcidx = fcidx;
+    node->m_nchild = nchild;
+    node->m_flidx = flidx;
+    node->m_nleaves = nleaves;
+}
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/dense_tree.cpp

@@ -0,0 +1,503 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/dense_tree.h>
+#include <perspective/filter.h>
+#include <perspective/node_processor.h>
+#include <perspective/comparators.h>
+#include <perspective/sort_specification.h>
+#include <perspective/data_table.h>
+
+#include <numeric>
+#include <utility>
+
+namespace perspective {
+
+t_dtree::t_dtree(
+    t_dssptr ds,
+    const std::vector<t_pivot>& pivots,
+    const std::vector<std::pair<std::string, std::string>>& sortby_colvec
+) :
+    m_levels_pivoted(0),
+    m_ds(std::move(ds)),
+    m_pivots(pivots),
+    m_nidx(0),
+    m_backing_store(BACKING_STORE_MEMORY),
+    m_init(false),
+    m_sortby_colvec(sortby_colvec) {}
+
+t_uindex
+t_dtree::size() const {
+    return m_nodes.size();
+}
+
+std::pair<t_index, t_index>
+t_dtree::get_level_markers(t_uindex idx) const {
+    PSP_VERBOSE_ASSERT(idx < m_levels.size(), "Unexpected lvlidx");
+    return m_levels[idx];
+}
+
+t_dtree::t_dtree(
+    std::string dirname,
+    t_dssptr ds,
+    const std::vector<t_pivot>& pivots,
+    t_backing_store backing_store,
+    const std::vector<std::pair<std::string, std::string>>& sortby_colvec
+) :
+    m_dirname(std::move(dirname)),
+    m_levels_pivoted(0),
+    m_ds(std::move(ds)),
+    m_pivots(pivots),
+    m_nidx(0),
+    m_backing_store(backing_store),
+    m_init(false),
+    m_sortby_colvec(sortby_colvec) {}
+
+void
+t_dtree::init() {
+    t_lstore_recipe leaf_args(
+        m_dirname, leaves_colname(), DEFAULT_CAPACITY, m_backing_store
+    );
+    m_leaves = t_column(DTYPE_UINT64, false, leaf_args, DEFAULT_CAPACITY);
+    m_leaves.init();
+
+    t_lstore_recipe node_args(
+        m_dirname, nodes_colname(), DEFAULT_CAPACITY, m_backing_store
+    );
+
+    m_values = std::vector<t_column>(m_pivots.size() + 1);
+
+    m_has_sortby = std::vector<bool>(m_values.size());
+    m_has_sortby[0] = false;
+
+    m_sortby_columns.clear();
+    for (auto& sidx : m_sortby_colvec) {
+        m_sortby_columns[sidx.first] = sidx.second;
+    }
+
+    t_lstore_recipe root_args(
+        m_dirname, values_colname("_root_"), DEFAULT_CAPACITY, m_backing_store
+    );
+
+    m_values[0] = t_column(DTYPE_STR, true, leaf_args, DEFAULT_CAPACITY);
+    m_values[0].init();
+
+    m_sortby_dpthcol.emplace_back("");
+
+    for (t_uindex idx = 0, loop_end = m_pivots.size(); idx < loop_end; ++idx) {
+        auto colname = m_pivots[idx].colname();
+        t_lstore_recipe leaf_args(
+            m_dirname,
+            values_colname(colname),
+            DEFAULT_CAPACITY,
+            m_backing_store
+        );
+
+        auto siter = m_sortby_columns.find(colname);
+        bool has_sortby =
+            siter != m_sortby_columns.end() && siter->second != colname;
+        m_has_sortby[idx + 1] = has_sortby;
+        std::string sortby_column = has_sortby ? siter->second : colname;
+        m_sortby_dpthcol.push_back(sortby_column);
+        t_dtype dtype = m_ds->get_dtype(colname);
+        m_values[idx + 1] = t_column(dtype, true, leaf_args, DEFAULT_CAPACITY);
+        m_values[idx + 1].init();
+    }
+
+    m_init = true;
+}
+
+std::string
+t_dtree::repr() const {
+    std::stringstream ss;
+    ss << m_ds->name() << "_tree_" << this;
+    return ss.str();
+}
+
+std::string
+t_dtree::leaves_colname() const {
+    return repr() + std::string("_leaves");
+}
+
+std::string
+t_dtree::nodes_colname() const {
+    return repr() + std::string("_nodes");
+}
+
+std::string
+t_dtree::values_colname(const std::string& tbl_colname) const {
+    return repr() + std::string("_valuespan_") + tbl_colname;
+}
+
+void
+t_dtree::check_pivot(const t_filter& filter, t_uindex level) {
+    if (level <= m_levels_pivoted) {
+        return;
+    }
+
+    PSP_VERBOSE_ASSERT(
+        level <= m_pivots.size() + 1, "Erroneous level passed in"
+    );
+
+    pivot(filter, level);
+}
+
+void
+t_dtree::pivot(const t_filter& filter, t_uindex level) {
+    if (level <= m_levels_pivoted) {
+        return;
+    }
+
+    PSP_VERBOSE_ASSERT(
+        level <= m_pivots.size() + 1, "Erroneous level passed in"
+    );
+
+    t_uindex ncols = m_pivots.size();
+
+    t_uindex nidx = m_nidx;
+
+    t_uindex nrows;
+    const t_mask* mask = nullptr;
+
+    if (ncols > 0) {
+        if (filter.has_filter()) {
+            nrows = filter.count();
+            mask = filter.cmask().get();
+        } else {
+            nrows = m_ds->num_rows();
+        }
+    } else {
+        nrows = m_pivots.empty() ? m_ds->num_rows() : 1;
+    }
+
+    t_uindex nbidx;
+    t_uindex neidx;
+
+    if (m_levels_pivoted == 0) {
+        m_leaves.extend<t_uindex>(nrows);
+        auto* leaves = m_leaves.get_nth<t_uindex>(0);
+        std::iota(leaves, leaves + nrows, t_uindex(0));
+
+        nbidx = 0;
+        neidx = 1;
+    } else {
+        nbidx = m_levels[m_levels_pivoted].first;
+        neidx = m_levels[m_levels_pivoted].second;
+    }
+
+    for (t_uindex pidx = m_levels_pivoted; pidx < level; pidx++) {
+        const t_column* pivcol;
+        if (pidx == 0) {
+            m_nodes.emplace_back();
+            t_tnode* root = &m_nodes.back();
+            fill_dense_tnode(root, nidx, nidx, 1, 0, 0, nrows);
+            nidx++;
+            m_values[0].push_back(std::string("Grand Aggregate"));
+            m_levels.emplace_back(nbidx, neidx);
+        } else {
+            const t_pivot& pivot = m_pivots[pidx - 1];
+            std::string pivot_colname = pivot.colname();
+            pivcol = m_ds->_get_const_column(pivot_colname);
+            t_dtype piv_dtype = pivcol->get_dtype();
+
+            t_uindex next_neidx = 0;
+
+            switch (piv_dtype) {
+                case DTYPE_STR: {
+                    next_neidx = t_pivot_processor<DTYPE_STR>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_INT64: {
+                    next_neidx = t_pivot_processor<DTYPE_INT64>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_INT32: {
+                    next_neidx = t_pivot_processor<DTYPE_INT32>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_INT16: {
+                    next_neidx = t_pivot_processor<DTYPE_INT16>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_INT8: {
+                    next_neidx = t_pivot_processor<DTYPE_INT8>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_UINT64: {
+                    next_neidx = t_pivot_processor<DTYPE_UINT64>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_UINT32: {
+                    next_neidx = t_pivot_processor<DTYPE_UINT32>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_UINT16: {
+                    next_neidx = t_pivot_processor<DTYPE_UINT16>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_UINT8: {
+                    next_neidx = t_pivot_processor<DTYPE_UINT8>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_FLOAT64: {
+                    next_neidx = t_pivot_processor<DTYPE_FLOAT64>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_FLOAT32: {
+                    next_neidx = t_pivot_processor<DTYPE_FLOAT32>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_BOOL: {
+
+                    next_neidx = t_pivot_processor<DTYPE_BOOL>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_TIME: {
+                    next_neidx = t_pivot_processor<DTYPE_INT64>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                case DTYPE_DATE: {
+                    next_neidx = t_pivot_processor<DTYPE_UINT32>()(
+                        pivcol,
+                        &m_nodes,
+                        &(m_values[pidx]),
+                        &m_leaves,
+                        nbidx,
+                        neidx,
+                        mask
+                    );
+                } break;
+                default: {
+                    PSP_COMPLAIN_AND_ABORT("Not supported yet");
+                } break;
+            }
+            nbidx = neidx;
+            neidx = next_neidx;
+            m_levels.emplace_back(nbidx, neidx);
+        }
+
+        m_levels_pivoted = pidx;
+    }
+
+    m_nidx = neidx;
+}
+
+t_uindex
+t_dtree::get_depth(t_index idx) const {
+    return get_span_index(idx).first;
+}
+
+void
+t_dtree::pprint(const t_filter& filter) const {
+    std::string indent("  ");
+
+    for (auto idx : dfs()) {
+        t_uindex depth = get_depth(idx);
+
+        for (t_uindex didx = 0; didx < depth; ++didx) {
+            std::cout << indent;
+        }
+
+        const t_dense_tnode* node = get_node_ptr(idx);
+
+        std::cout << get_value(filter, idx) << " idx => " << node->m_idx
+                  << " pidx => " << node->m_pidx << " fcidx => "
+                  << node->m_fcidx << " nchild => " << node->m_nchild
+                  << " flidx => " << node->m_flidx << " nleaves => "
+                  << node->m_nleaves << '\n';
+    }
+}
+
+t_uindex
+t_dtree::last_level() const {
+    return m_pivots.size();
+}
+
+const t_dtree::t_tnode*
+t_dtree::get_node_ptr(t_index nidx) const {
+    return &m_nodes.at(nidx);
+}
+
+t_tscalar
+t_dtree::_get_value(const t_filter& filter, t_index nidx, bool sort_value)
+    const {
+
+    std::pair<t_uindex, t_uindex> spi = get_span_index(nidx);
+    t_index dpth = spi.first;
+    t_uindex scalar_idx = spi.second;
+
+    if (sort_value || nidx == 0) {
+        const t_column& col = m_values[dpth];
+        return col.get_scalar(scalar_idx);
+    }
+    const std::string& colname = m_sortby_dpthcol[dpth];
+    const t_column& col = *(m_ds->get_const_column(colname));
+    const auto* node = get_node_ptr(nidx);
+    t_uindex lfidx = *(m_leaves.get_nth<t_uindex>(node->m_flidx));
+    return col.get_scalar(lfidx);
+}
+
+t_tscalar
+t_dtree::get_value(const t_filter& filter, t_index nidx) const {
+    return _get_value(filter, nidx, true);
+}
+
+t_tscalar
+t_dtree::get_sortby_value(const t_filter& filter, t_index nidx) const {
+    return _get_value(filter, nidx, false);
+}
+
+std::pair<t_uindex, t_uindex>
+t_dtree::get_span_index(t_index idx) const {
+    for (t_uindex i = 0, loop_end = m_levels.size(); i < loop_end; i++) {
+        t_index bidx = m_levels[i].first;
+        t_index eidx = m_levels[i].second;
+
+        if ((idx >= bidx) && (idx < eidx)) {
+            return std::pair<t_uindex, t_uindex>(i, idx - bidx);
+        }
+    }
+
+    PSP_COMPLAIN_AND_ABORT("Reached unreachable.");
+    return std::pair<t_uindex, t_uindex>(0, 0);
+}
+
+const t_column*
+t_dtree::get_leaf_cptr() const {
+    return &m_leaves;
+}
+
+t_bfs_iter<t_dtree>
+t_dtree::bfs() const {
+    return {this};
+}
+
+t_dfs_iter<t_dtree>
+t_dtree::dfs() const {
+    return {this};
+}
+
+t_index
+t_dtree::get_parent(t_index idx) const {
+    const t_tnode* n = get_node_ptr(idx);
+    return n->m_pidx;
+}
+
+const std::vector<t_pivot>&
+t_dtree::get_pivots() const {
+    return m_pivots;
+}
+
+void
+t_dtree::get_child_indices(t_index nidx, std::vector<t_index>& v) const {
+    const auto* nptr = get_node_ptr(nidx);
+    for (t_index idx = nptr->m_fcidx + nptr->m_nchild - 1,
+                 loop_end = nptr->m_fcidx;
+         idx >= loop_end;
+         --idx) {
+        v.push_back(idx);
+    }
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/dense_tree_context.cpp

@@ -0,0 +1,290 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <iomanip>
+#include <utility>
+#include <perspective/dense_tree_context.h>
+#include <perspective/dependency.h>
+#include <perspective/schema.h>
+
+namespace perspective {
+
+t_dtree_ctx::t_dtree_ctx(
+    std::shared_ptr<const t_data_table> strands,
+    std::shared_ptr<const t_data_table> strand_deltas,
+    const t_dtree& tree,
+    const std::vector<t_aggspec>& aggspecs
+) :
+    m_strands(std::move(std::move(strands))),
+    m_strand_deltas(std::move(std::move(strand_deltas))),
+    m_tree(tree),
+    m_aggspecs(aggspecs),
+    m_init(false) {
+    std::vector<t_dep> depvec = {t_dep("psp_strand_count", DEPTYPE_COLUMN)};
+
+    m_aggspecs.emplace_back("psp_strand_count_sum", AGGTYPE_SUM, depvec);
+
+    t_uindex aggidx = 0;
+    for (const auto& spec : m_aggspecs) {
+        m_aggspecmap[spec.name()] = aggidx;
+        ++aggidx;
+    }
+}
+
+void
+t_dtree_ctx::init() {
+    build_aggregates();
+    m_init = true;
+}
+
+t_uindex
+t_dtree_ctx::get_num_aggcols() const {
+    return m_aggspecs.size();
+}
+
+void
+t_dtree_ctx::build_aggregates() {
+    std::vector<std::string> columns;
+    std::vector<t_dtype> dtypes;
+
+    const t_schema& delta_schema = m_strand_deltas->get_schema();
+
+    for (const auto& spec : m_aggspecs) {
+        auto cinfo = spec.get_output_specs(delta_schema);
+        for (const auto& ci : cinfo) {
+            PSP_VERBOSE_ASSERT(
+                ci.m_type != DTYPE_NONE, "NULL type encountered"
+            );
+            columns.push_back(ci.m_name);
+            dtypes.push_back(ci.m_type);
+        }
+    }
+
+    t_schema aggschema(columns, dtypes);
+
+    m_aggregates = std::make_shared<t_data_table>(aggschema, m_tree.size());
+
+    m_aggregates->init();
+    m_aggregates->set_size(m_tree.size());
+
+    for (const auto& aggspec : m_aggspecs) {
+        const std::vector<t_dep>& deps = aggspec.get_dependencies();
+
+        const t_data_table* tbl =
+            aggspec.is_non_delta() ? m_strands.get() : m_strand_deltas.get();
+
+        std::vector<std::shared_ptr<const t_column>> icolumns;
+        icolumns.reserve(deps.size());
+        for (const auto& d : deps) {
+            icolumns.push_back(tbl->get_const_column(d.name()));
+        }
+
+        auto output_col = m_aggregates->get_column(aggspec.name());
+
+        t_aggregate agg(m_tree, aggspec.agg(), icolumns, output_col);
+        agg.init();
+    }
+}
+
+const t_data_table&
+t_dtree_ctx::get_aggtable() const {
+    return *(m_aggregates);
+}
+
+const t_dtree&
+t_dtree_ctx::get_tree() const {
+    return m_tree;
+}
+
+const std::vector<t_aggspec>&
+t_dtree_ctx::get_aggspecs() const {
+    return m_aggspecs;
+}
+
+const t_aggspec&
+t_dtree_ctx::get_aggspec(const std::string& aggname) const {
+    auto iter = m_aggspecmap.find(aggname);
+    PSP_VERBOSE_ASSERT(iter != m_aggspecmap.end(), "Failed to find aggspec");
+    PSP_VERBOSE_ASSERT(
+        static_cast<t_uindex>(iter->second) < m_aggspecs.size(),
+        "Invalid aggspec index"
+    );
+    return m_aggspecs[iter->second];
+}
+
+void
+t_dtree_ctx::pprint(const t_filter& fltr) const {
+    std::vector<const t_column*> aggcols;
+
+    t_uindex ncols = 0;
+    for (const std::string& colname : m_aggregates->get_schema().m_columns) {
+        aggcols.push_back(m_aggregates->_get_const_column(colname));
+        std::cout << colname << ", ";
+        ++ncols;
+    }
+    std::cout << "\n====================================\n";
+
+    for (auto idx : m_tree.dfs()) {
+        auto depth = m_tree.get_depth(idx);
+        for (t_uindex spidx = 0; spidx < static_cast<t_uindex>(depth);
+             ++spidx) {
+            std::cout << "\t";
+        }
+
+        auto value = m_tree.get_value(fltr, idx);
+        std::cout << "(" << idx << "). " << value << " => ";
+
+        for (t_uindex aggidx = 0; aggidx < ncols; ++aggidx) {
+            std::cout << aggcols[aggidx]->get_scalar(idx) << ", ";
+        }
+        std::cout << "\n";
+    }
+}
+
+std::pair<const t_uindex*, const t_uindex*>
+t_dtree_ctx::get_leaf_iterators(t_index idx) const {
+    const t_dense_tnode* node = m_tree.get_node_ptr(idx);
+    const auto* lbaseptr = m_tree.get_leaf_cptr()->get_nth<t_uindex>(0);
+    return std::pair<const t_uindex*, const t_uindex*>(
+        lbaseptr + node->m_flidx, lbaseptr + node->m_flidx + node->m_nleaves
+    );
+}
+
+std::shared_ptr<const t_column>
+t_dtree_ctx::get_pkey_col() const {
+    return m_strands->get_const_column("psp_pkey");
+}
+
+std::shared_ptr<const t_column>
+t_dtree_ctx::get_strand_count_col() const {
+    return m_strand_deltas->get_const_column("psp_strand_count");
+}
+
+std::shared_ptr<const t_data_table>
+t_dtree_ctx::get_strands() const {
+    return m_strands;
+}
+
+std::shared_ptr<const t_data_table>
+t_dtree_ctx::get_strand_deltas() const {
+    return m_strand_deltas;
+}
+
+void
+t_dtree_ctx::pprint_strands() const {
+    std::vector<const t_column*> columns;
+    const auto* scount_col =
+        m_strand_deltas->_get_const_column("psp_strand_count");
+    const auto* pkey_col = m_strands->_get_const_column("psp_pkey");
+    auto strand_schema = m_strands->get_schema();
+
+    t_uindex width = 18;
+
+    std::vector<std::string> colnames = {"psp_pkey", "psp_strand_count"};
+
+    for (const auto& colname : strand_schema.m_columns) {
+        const auto* col = m_strands->_get_const_column(colname);
+        if (col != pkey_col) {
+            columns.push_back(col);
+            colnames.push_back(colname);
+        }
+    }
+
+    auto strand_delta_schema = m_strand_deltas->get_schema();
+    for (const auto& colname : strand_delta_schema.m_columns) {
+        const auto* col = m_strand_deltas->_get_const_column(colname);
+        if (col != scount_col) {
+            columns.push_back(col);
+            std::stringstream ss;
+            ss << "delta(" << colname << ")";
+            colnames.push_back(ss.str());
+        }
+    }
+
+    for (const auto& c : colnames) {
+        std::cout << std::setw(width) << c;
+    }
+
+    std::cout << "\n====================================\n";
+
+    for (t_uindex idx = 0, loop_end = m_strands->size(); idx < loop_end;
+         ++idx) {
+
+        std::vector<t_tscalar> vec;
+        vec.push_back(pkey_col->get_scalar(idx));
+        t_tscalar sc;
+        sc.set(t_index(*(scount_col->get<std::int8_t>(idx))));
+        vec.push_back(sc);
+
+        for (const auto* col : columns) {
+            vec.push_back(col->get_scalar(idx));
+        }
+
+        std::cout << idx << ".";
+        for (const auto& val : vec) {
+            std::cout << std::setw(width) << val;
+        }
+
+        std::cout << '\n';
+    }
+}
+
+void
+t_dtree_ctx::pprint_strands_tree() const {
+    typedef std::pair<std::string, const t_column*> t_colname_cptr_pair;
+
+    std::vector<t_colname_cptr_pair> columns;
+
+    columns.emplace_back(
+        "psp_pkey", m_strands->_get_const_column("psp_pkey")
+    );
+
+    columns.emplace_back(
+        "psp_strand_count",
+        m_strand_deltas->_get_const_column("psp_strand_count")
+    );
+
+    for (const auto& piv : m_tree.get_pivots()) {
+        columns.emplace_back(
+            piv.colname(), m_strands->_get_const_column(piv.colname())
+        );
+    }
+
+    for (auto dptidx : m_tree.dfs()) {
+        std::cout << "nidx(" << dptidx << ") => " << '\n';
+
+        t_depth ndepth = m_tree.get_depth(dptidx);
+
+        auto liters = get_leaf_iterators(dptidx);
+
+        for (const auto* lfiter = liters.first; lfiter != liters.second;
+             ++lfiter) {
+            for (t_uindex idx = 0; idx < ndepth; ++idx) {
+                std::cout << "\t";
+            }
+
+            std::cout << "\tleaf# " << *lfiter << "\n";
+
+            for (const auto& colinfo : columns) {
+                for (t_uindex idx = 0; idx < ndepth + 1; ++idx) {
+                    std::cout << "\t";
+                }
+                std::cout << "    " << colinfo.first << ": "
+                          << colinfo.second->get_scalar(*lfiter) << "\n";
+            }
+        }
+
+        std::cout << '\n';
+    }
+}
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/dependency.cpp

@@ -0,0 +1,82 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/dependency.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_dep::t_dep(const t_dep_recipe& v) :
+    m_name(v.m_name),
+    m_disp_name(v.m_disp_name),
+    m_type(v.m_type),
+    m_imm(v.m_imm),
+    m_dtype(v.m_dtype) {}
+
+t_dep::t_dep(const std::string& name, t_deptype type) :
+    m_name(name),
+    m_disp_name(name),
+    m_type(type),
+    m_dtype(DTYPE_NONE) {}
+
+t_dep::t_dep(t_tscalar imm) :
+    m_type(DEPTYPE_SCALAR),
+    m_imm(imm),
+    m_dtype(DTYPE_NONE) {}
+
+t_dep::t_dep(
+    std::string name, std::string disp_name, t_deptype type, t_dtype dtype
+) :
+    m_name(std::move(name)),
+    m_disp_name(std::move(disp_name)),
+    m_type(type),
+    m_dtype(dtype) {}
+
+const std::string&
+t_dep::name() const {
+    return m_name;
+}
+
+const std::string&
+t_dep::disp_name() const {
+    return m_disp_name;
+}
+
+t_deptype
+t_dep::type() const {
+    return m_type;
+}
+
+t_tscalar
+t_dep::imm() const {
+    return m_imm;
+}
+
+t_dtype
+t_dep::dtype() const {
+    return m_dtype;
+}
+
+t_dep_recipe
+t_dep::get_recipe() const {
+    t_dep_recipe rv;
+    rv.m_name = m_name;
+    rv.m_disp_name = m_disp_name;
+    rv.m_type = m_type;
+    rv.m_imm = m_imm;
+    rv.m_dtype = m_dtype;
+    return rv;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/expression_tables.cpp

@@ -0,0 +1,198 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/expression_tables.h>
+#include <perspective/utils.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_expression_tables::t_expression_tables(
+    const std::vector<std::shared_ptr<t_computed_expression>>& expressions,
+    t_backing_store backing_store
+) {
+    t_schema schema;
+    t_schema transitions_schema;
+
+    for (const auto& expr : expressions) {
+        const std::string& alias = expr->get_expression_alias();
+        schema.add_column(alias, expr->get_dtype());
+        transitions_schema.add_column(alias, DTYPE_UINT8);
+    }
+
+    // Only the persistent `m_master` table honors on-disk backing; the
+    // transitional tables are per-update scratch (cleared every update, sized
+    // to the update batch) so disk-backing them is pure I/O churn with no
+    // memory-relief benefit, and they stay in memory.
+    std::string master_dirname;
+    if (backing_store == BACKING_STORE_DISK && !expressions.empty()) {
+        master_dirname = create_backing_store_dir("perspective_expr_");
+    }
+
+    m_master = std::make_shared<t_data_table>(
+        "", master_dirname, schema, DEFAULT_EMPTY_CAPACITY, backing_store
+    );
+    m_flattened = std::make_shared<t_data_table>(
+        "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_prev = std::make_shared<t_data_table>(
+        "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_current = std::make_shared<t_data_table>(
+        "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_delta = std::make_shared<t_data_table>(
+        "", "", schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_transitions = std::make_shared<t_data_table>(
+        "", "", transitions_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+
+    m_master->init();
+    m_flattened->init();
+    m_prev->init();
+    m_current->init();
+    m_delta->init();
+    m_transitions->init();
+}
+
+t_data_table*
+t_expression_tables::get_table() const {
+    return m_master.get();
+}
+
+void
+t_expression_tables::set_flattened(
+    const std::shared_ptr<t_data_table>& flattened
+) const {
+    t_uindex flattened_num_rows = flattened->size();
+    reserve_transitional_table_size(flattened_num_rows);
+    set_transitional_table_size(flattened_num_rows);
+    const t_schema& schema = m_flattened->get_schema();
+    const std::vector<std::string>& column_names = schema.m_columns;
+    for (const auto& colname : column_names) {
+        // Deep-copy into the (in-memory) transitional column in place rather
+        // than `clone()`-ing the source. `clone()` inherits the source's
+        // backing store via its recipe, and for an on-disk expression
+        // `m_master` that produces a broken disk clone which aliases the
+        // master's backing file. `copy_from` preserves `m_flattened`'s own
+        // (memory) backing store.
+        m_flattened->get_column(colname)->copy_from(
+            *flattened->get_column(colname)
+        );
+    }
+}
+
+void
+t_expression_tables::calculate_transitions(
+    const std::shared_ptr<t_data_table>& existed
+) {
+    const t_schema& schema = m_transitions->get_schema();
+    const std::vector<std::string>& column_names = schema.m_columns;
+    const t_column& existed_column =
+        *(existed->get_const_column("psp_existed"));
+
+    auto num_cols = column_names.size();
+
+    parallel_for(
+        int(num_cols),
+        [&column_names, &existed_column, this](int cidx) {
+            const std::string& cname = column_names[cidx];
+            const t_column& prev_column = *(m_prev->get_const_column(cname));
+            const t_column& current_column =
+                *(m_current->get_const_column(cname));
+            std::shared_ptr<t_column> transition_column =
+                m_transitions->get_column(cname);
+
+            for (t_uindex ridx = 0; ridx < transition_column->size(); ++ridx) {
+                bool row_existed =
+                    existed_column.get_nth<bool>(ridx) != nullptr;
+
+                t_tscalar prev_value = prev_column.get_scalar(ridx);
+                t_tscalar curr_value = current_column.get_scalar(ridx);
+
+                bool prev_valid = prev_column.is_valid(ridx);
+                bool curr_valid = current_column.is_valid(ridx);
+                bool prev_curr_eq =
+                    prev_valid && curr_valid && (prev_value == curr_value);
+
+                t_value_transition transition;
+
+                // Use a small subset of `t_value_transitions` that are
+                // relevant - I have not implemented the code paths in
+                // `calc_transitions` that are not referenced elsewhere, i.e.
+                // by a context or by a tree implementation.
+                if (row_existed) {
+                    if (prev_curr_eq) {
+                        // Row existed before, and the current value is
+                        // the same as the previous value.
+                        transition = VALUE_TRANSITION_EQ_TT;
+                    } else {
+                        if (!prev_valid && curr_valid) {
+                            // Previous value was a null, new value is valid.
+                            transition = VALUE_TRANSITION_NEQ_FT;
+                        } else {
+                            // Previous value was not null, new value is
+                            // not null, and previous value != new value
+                            transition = VALUE_TRANSITION_NEQ_TT;
+                        }
+                    }
+                } else {
+                    // Row did not exist before and was added
+                    transition = VALUE_TRANSITION_NEQ_FT;
+                }
+
+                transition_column->set_nth<std::uint8_t>(ridx, transition);
+            }
+        }
+    );
+}
+
+void
+t_expression_tables::reserve_transitional_table_size(t_uindex size) const {
+    m_flattened->reserve(size);
+    m_prev->reserve(size);
+    m_current->reserve(size);
+    m_delta->reserve(size);
+    m_transitions->reserve(size);
+}
+
+void
+t_expression_tables::set_transitional_table_size(t_uindex size) const {
+    m_flattened->set_size(size);
+    m_prev->set_size(size);
+    m_current->set_size(size);
+    m_delta->set_size(size);
+    m_transitions->set_size(size);
+}
+
+void
+t_expression_tables::clear_transitional_tables() const {
+    m_flattened->clear();
+    m_prev->clear();
+    m_current->clear();
+    m_delta->clear();
+    m_transitions->clear();
+}
+
+void
+t_expression_tables::reset() const {
+    m_master->reset();
+    m_flattened->reset();
+    m_prev->reset();
+    m_current->reset();
+    m_delta->reset();
+    m_transitions->reset();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/expression_vocab.cpp

@@ -0,0 +1,81 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/expression_vocab.h>
+
+namespace perspective {
+
+t_expression_vocab::t_expression_vocab() {
+    // Pre-reserve each page to its full size so that `intern()` never
+    // triggers a realloc inside `t_vocab`; previously-returned `const char*`
+    // pointers must remain valid for the lifetime of the vocab.
+    m_max_vocab_size = 16 * 64 * 64;
+
+    // Always start with one vocab
+    allocate_new_vocab();
+}
+
+const char*
+t_expression_vocab::intern(const char* str) {
+    std::size_t bytelength = strlen(str);
+    std::size_t hash = boost::hash_range(str, str + bytelength);
+    t_uindex existing_idx;
+
+    for (auto& current_vocab : m_vocabs) {
+        if (current_vocab.string_exists(str, hash, existing_idx)) {
+            return current_vocab.unintern_c(existing_idx);
+        }
+    }
+
+    if (m_current_vocab_size + bytelength + 1 > m_max_vocab_size) {
+        allocate_new_vocab();
+    }
+
+    t_vocab& current_vocab = m_vocabs.front();
+    m_current_vocab_size += bytelength + 1;
+    t_uindex interned_idx = current_vocab.get_interned(str, bytelength, hash);
+    return current_vocab.unintern_c(interned_idx);
+}
+
+const char*
+t_expression_vocab::intern(const std::string& str) {
+    return intern(str.c_str());
+}
+
+void
+t_expression_vocab::clear() {
+    m_vocabs.clear();
+    allocate_new_vocab();
+}
+
+const char*
+t_expression_vocab::get_empty_string() const {
+    return m_empty_string.c_str();
+}
+
+void
+t_expression_vocab::pprint() const {
+    for (const auto& vocab : m_vocabs) {
+        vocab.pprint_vocabulary();
+    }
+}
+
+void
+t_expression_vocab::allocate_new_vocab() {
+    t_vocab vocab;
+    vocab.init(false);
+    vocab.reserve(m_max_vocab_size, m_max_vocab_size / 64);
+    m_vocabs.insert(m_vocabs.begin(), std::move(vocab));
+    m_current_vocab_size = 0;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/extract_aggregate.cpp

@@ -0,0 +1,124 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/extract_aggregate.h>
+
+namespace perspective {
+
+t_tscalar
+extract_aggregate(
+    const t_aggspec& aggspec,
+    const t_column* aggcol,
+    t_uindex ridx,
+    t_index pridx
+) {
+
+    switch (aggspec.agg()) {
+        case AGGTYPE_PCT_SUM_PARENT: {
+            t_tscalar cv = aggcol->get_scalar(ridx);
+
+            if (pridx == INVALID_INDEX) {
+                return mktscalar<double>(100.0);
+            }
+
+            double pv = aggcol->get_scalar(pridx).to_double();
+            t_tscalar ret;
+            if (pv == 0) {
+                ret.set(t_none());
+            } else {
+                ret.set(100.0 * (cv.to_double() / pv));
+            }
+            return ret;
+        } break;
+        case AGGTYPE_PCT_SUM_GRAND_TOTAL: {
+            t_tscalar cv = aggcol->get_scalar(ridx);
+            double pv = aggcol->get_scalar(ROOT_AGGIDX).to_double();
+            t_tscalar ret;
+            if (pv == 0) {
+                ret.set(t_none());
+            } else {
+                ret.set(100.0 * (cv.to_double() / pv));
+            }
+            return ret;
+        } break;
+        case AGGTYPE_SUM:
+        case AGGTYPE_SUM_ABS:
+        case AGGTYPE_ABS_SUM:
+        case AGGTYPE_GMV:
+        case AGGTYPE_SUM_NOT_NULL:
+        case AGGTYPE_MUL:
+        case AGGTYPE_COUNT:
+        case AGGTYPE_ANY:
+        case AGGTYPE_DOMINANT:
+        case AGGTYPE_Q1:
+        case AGGTYPE_Q3:
+        case AGGTYPE_MEDIAN:
+        case AGGTYPE_FIRST:
+        case AGGTYPE_AND:
+        case AGGTYPE_OR:
+        case AGGTYPE_LAST_BY_INDEX:
+        case AGGTYPE_LAST_VALUE:
+        case AGGTYPE_LAST_MINUS_FIRST:
+        case AGGTYPE_MAX:
+        case AGGTYPE_MAX_BY:
+        case AGGTYPE_MIN_BY:
+        case AGGTYPE_MIN:
+        case AGGTYPE_HIGH_WATER_MARK:
+        case AGGTYPE_LOW_WATER_MARK:
+        case AGGTYPE_HIGH_MINUS_LOW:
+        case AGGTYPE_SCALED_DIV:
+        case AGGTYPE_SCALED_ADD:
+        case AGGTYPE_SCALED_MUL:
+        case AGGTYPE_UDF_COMBINER:
+        case AGGTYPE_UDF_REDUCER:
+        case AGGTYPE_JOIN:
+        case AGGTYPE_IDENTITY:
+        case AGGTYPE_DISTINCT_COUNT:
+        case AGGTYPE_DISTINCT_LEAF:
+        case AGGTYPE_VARIANCE:
+        case AGGTYPE_STANDARD_DEVIATION: {
+            t_tscalar rval = aggcol->get_scalar(ridx);
+            return rval;
+        } break;
+        case AGGTYPE_UNIQUE: {
+            t_tscalar rval = aggcol->get_scalar(ridx);
+            if (!rval.is_valid()) {
+                t_tscalar rv;
+                rv.set(t_none());
+                return rv;
+            }
+            return rval;
+        } break;
+        case AGGTYPE_MEAN_BY_COUNT:
+        case AGGTYPE_WEIGHTED_MEAN:
+        case AGGTYPE_MEAN: {
+            const auto* pair = aggcol->get_nth<std::pair<double, double>>(ridx);
+            t_tscalar rval;
+            double second = pair->second;
+            if (second != 0) {
+                double mean = pair->first / second;
+                rval.set(mean);
+            } else {
+                rval.set(t_none());
+            }
+            return rval;
+        } break;
+        default: {
+            PSP_COMPLAIN_AND_ABORT("Unexpected agg type");
+        }
+    }
+
+    return mknone();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/filter.cpp

@@ -0,0 +1,168 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/filter.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_fterm::t_fterm() = default;
+
+t_fterm::t_fterm(
+    std::string colname,
+    t_filter_op op,
+    t_tscalar threshold,
+    const std::vector<t_tscalar>& bag,
+    bool negated,
+    bool is_primary
+) :
+    m_colname(std::move(colname)),
+    m_op(op),
+    m_threshold(threshold),
+    m_bag(bag),
+    m_negated(negated),
+    m_is_primary(is_primary) {
+    m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE)
+        && threshold.m_type == DTYPE_STR;
+}
+
+t_fterm::t_fterm(
+    std::string colname,
+    t_filter_op op,
+    t_tscalar threshold,
+    const std::vector<t_tscalar>& bag
+) :
+    m_colname(std::move(colname)),
+    m_op(op),
+    m_threshold(threshold),
+    m_bag(bag),
+    m_negated(false),
+    m_is_primary(false) {
+    m_use_interned = (op == FILTER_OP_EQ || op == FILTER_OP_NE)
+        && threshold.m_type == DTYPE_STR;
+}
+
+void
+t_fterm::coerce_numeric(t_dtype dtype) {
+    m_threshold.set(m_threshold.coerce_numeric_dtype(dtype));
+    for (auto& f : m_bag) {
+        f.set(f.coerce_numeric_dtype(dtype));
+    }
+}
+
+std::string
+t_fterm::get_expr() const {
+    std::stringstream ss;
+
+    ss << m_colname << " ";
+
+    switch (m_op) {
+        case FILTER_OP_LT:
+        case FILTER_OP_LTEQ:
+        case FILTER_OP_GT:
+        case FILTER_OP_GTEQ:
+        case FILTER_OP_EQ:
+        case FILTER_OP_NE:
+        case FILTER_OP_CONTAINS: {
+            ss << filter_op_to_str(m_op) << " ";
+            ss << m_threshold.to_string(true);
+        } break;
+        case FILTER_OP_NOT_IN:
+        case FILTER_OP_IN: {
+            ss << " " << filter_op_to_str(m_op) << " (";
+            for (auto v : m_bag) {
+                ss << v.to_string(true) << ", ";
+            }
+            ss << " )";
+        } break;
+        case FILTER_OP_BEGINS_WITH:
+        case FILTER_OP_ENDS_WITH: {
+            ss << "." << filter_op_to_str(m_op) << "( "
+               << m_threshold.to_string(true) << " )";
+        } break;
+        default: {
+            ss << " is failed_compilation";
+        }
+    }
+
+    return ss.str();
+}
+
+t_filter::t_filter() : m_mode(SELECT_MODE_ALL) {}
+
+t_filter::t_filter(const std::vector<std::string>& columns) :
+    m_mode(SELECT_MODE_ALL),
+    m_columns(columns) {}
+
+t_filter::t_filter(
+    const std::vector<std::string>& columns, t_uindex bidx, t_uindex eidx
+) :
+    m_mode(SELECT_MODE_RANGE),
+    m_bidx(bidx),
+    m_eidx(eidx),
+    m_columns(columns) {}
+
+t_filter::t_filter(
+    const std::vector<std::string>& columns, t_uindex mask_size
+) :
+    m_mode(SELECT_MODE_MASK),
+    m_columns(columns),
+    m_mask(std::make_shared<t_mask>(mask_size)) {}
+
+t_uindex
+t_filter::num_cols() const {
+    return m_columns.size();
+}
+
+bool
+t_filter::has_filter() const {
+    return m_mode != SELECT_MODE_ALL;
+}
+
+const std::vector<std::string>&
+t_filter::columns() const {
+    return m_columns;
+}
+
+t_select_mode
+t_filter::mode() const {
+    return m_mode;
+}
+
+t_uindex
+t_filter::bidx() const {
+    return m_bidx;
+}
+
+t_uindex
+t_filter::eidx() const {
+    return m_eidx;
+}
+
+t_maskcsptr
+t_filter::cmask() const {
+    return t_maskcsptr(m_mask);
+}
+
+t_masksptr
+t_filter::mask() const {
+    return t_masksptr(m_mask);
+}
+
+t_uindex
+t_filter::count() const {
+    return m_mask->count();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/flat_traversal.cpp

@@ -0,0 +1,497 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/config.h>
+#include <perspective/flat_traversal.h>
+#include <perspective/scalar.h>
+#include <perspective/schema.h>
+
+#include <algorithm>
+
+namespace perspective {
+
+t_ftrav::t_ftrav() : m_step_deletes(0), m_step_inserts(0) {
+    m_index = std::make_shared<std::vector<t_mselem>>();
+}
+
+void
+t_ftrav::init() {
+    m_index = std::make_shared<std::vector<t_mselem>>();
+}
+
+std::vector<t_tscalar>
+t_ftrav::get_all_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    // assumes the code calling this has already validated
+    // cells
+    std::vector<t_tscalar> rval;
+    rval.reserve(cells.size());
+    std::vector<t_mselem>* index = m_index.get();
+    for (const auto& cell : cells) {
+        rval.push_back((*index)[cell.first].m_pkey);
+    }
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_ftrav::get_pkeys(const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    tsl::hopscotch_set<t_tscalar> all_pkeys;
+
+    std::set<t_index> all_rows;
+
+    for (const auto& cell : cells) {
+        all_rows.insert(cell.first);
+    }
+
+    std::vector<t_tscalar> rval(all_rows.size());
+    std::set<t_index>::iterator it;
+    t_index count = 0;
+    for (it = all_rows.begin(); it != all_rows.end(); ++it) {
+        rval[count] = (*m_index)[*it].m_pkey;
+        ++count;
+    }
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_ftrav::get_pkeys(t_index begin_row, t_index end_row) const {
+    t_index index_size = m_index->size();
+    end_row = std::min(end_row, index_size);
+    std::vector<t_tscalar> rval(end_row - begin_row);
+    for (t_index ridx = begin_row; ridx < end_row; ++ridx) {
+        rval[ridx - begin_row] = (*m_index)[ridx].m_pkey;
+    }
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_ftrav::get_pkeys(const std::vector<t_uindex>& rows) const {
+    std::vector<t_tscalar> rval;
+    rval.reserve(rows.size());
+    for (unsigned long long ridx : rows) {
+        rval.push_back((*m_index)[ridx].m_pkey);
+    }
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_ftrav::get_pkeys() const {
+    return get_pkeys(0, size());
+}
+
+t_tscalar
+t_ftrav::get_pkey(t_index idx) const {
+    return (*m_index)[idx].m_pkey;
+}
+
+void
+t_ftrav::fill_sort_elem(
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table,
+    const t_config& config,
+    t_tscalar pkey,
+    t_mselem& out_elem
+) {
+    t_index sortby_size = m_sortby.size();
+    out_elem.m_row.reserve(sortby_size);
+    out_elem.m_pkey = pkey;
+
+    for (const t_sortspec& sort : m_sortby) {
+        // maintain backwards compatibility
+        std::string colname;
+
+        if (!sort.m_colname.empty()) {
+            colname = config.get_sort_by(sort.m_colname);
+        } else {
+            colname = config.col_at(sort.m_agg_index);
+        }
+
+        const std::string& sortby_colname = config.get_sort_by(colname);
+
+        out_elem.m_row.push_back(
+            m_symtable.get_interned_tscalar(get_from_gstate(
+                gstate, expression_master_table, sortby_colname, pkey
+            ))
+        );
+    }
+}
+
+void
+t_ftrav::fill_sort_elem(
+    const t_gstate& gstate,
+    const t_config& config,
+    const std::vector<t_tscalar>& row,
+    t_mselem& out_elem
+) const {
+    t_index sortby_size = m_sortby.size();
+    out_elem.m_row.reserve(sortby_size);
+    out_elem.m_pkey = mknone();
+
+    for (const t_sortspec& sort : m_sortby) {
+        std::string colname;
+
+        if (!sort.m_colname.empty()) {
+            colname = config.get_sort_by(sort.m_colname);
+        } else {
+            colname = config.col_at(sort.m_agg_index);
+        }
+
+        const std::string& sortby_colname = config.get_sort_by(colname);
+
+        out_elem.m_row.push_back(
+            get_interned_tscalar(row.at(config.get_colidx(sortby_colname)))
+        );
+    }
+}
+
+void
+t_ftrav::sort_by(
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table,
+    const t_config& config,
+    const std::vector<t_sortspec>& sortby
+) {
+    if (sortby.empty()) {
+        return;
+    }
+    t_multisorter sorter(get_sort_orders(sortby));
+    t_index size = m_index->size();
+    auto sort_elems =
+        std::make_shared<std::vector<t_mselem>>(static_cast<size_t>(size));
+    m_sortby = sortby;
+
+    for (t_index idx = 0; idx < size; ++idx) {
+        t_mselem& elem = (*sort_elems)[idx];
+        t_tscalar pkey = (*m_index)[idx].m_pkey;
+        fill_sort_elem(gstate, expression_master_table, config, pkey, elem);
+    }
+
+    std::swap(m_index, sort_elems);
+    std::sort(m_index->begin(), m_index->end(), sorter);
+    m_pkeyidx.clear();
+    for (t_index idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) {
+        m_pkeyidx[(*m_index)[idx].m_pkey] = idx;
+    }
+}
+
+t_index
+t_ftrav::size() const {
+    return m_index->size();
+}
+
+void
+t_ftrav::get_row_indices(
+    const tsl::hopscotch_set<t_tscalar>& pkeys,
+    tsl::hopscotch_map<t_tscalar, t_index>& out_map
+) const {
+    for (t_index idx = 0, loop_end = size(); idx < loop_end; ++idx) {
+        const t_tscalar& pkey = (*m_index)[idx].m_pkey;
+        if (pkeys.find(pkey) != pkeys.end()) {
+            out_map[pkey] = idx;
+        }
+    }
+}
+
+void
+t_ftrav::get_row_indices(
+    t_index bidx,
+    t_index eidx,
+    const tsl::hopscotch_set<t_tscalar>& pkeys,
+    tsl::hopscotch_map<t_tscalar, t_index>& out_map
+) const {
+    for (t_index idx = bidx; idx < eidx; ++idx) {
+        const t_tscalar& pkey = (*m_index)[idx].m_pkey;
+        if (pkeys.find(pkey) != pkeys.end()) {
+            out_map[pkey] = idx;
+        }
+    }
+}
+
+/**
+ * @brief Given a set of primary keys, return the corresponding row indices.
+ *
+ * @param pkeys
+ * @return std::vector<t_index>
+ */
+std::vector<t_uindex>
+t_ftrav::get_row_indices(const tsl::hopscotch_set<t_tscalar>& pkeys) const {
+    std::vector<t_uindex> rows;
+    for (t_uindex idx = 0, loop_end = size(); idx < loop_end; ++idx) {
+        const t_tscalar& pkey = (*m_index)[idx].m_pkey;
+        if (pkeys.find(pkey) != pkeys.end()) {
+            rows.push_back(idx);
+        }
+    }
+    return rows;
+}
+
+void
+t_ftrav::reset() {
+    if (m_index != nullptr) {
+        m_index->clear();
+    }
+}
+
+void
+t_ftrav::check_size() {
+    tsl::hopscotch_set<t_tscalar> pkey_set;
+    for (t_index idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) {
+        if (pkey_set.find((*m_index)[idx].m_pkey) != pkey_set.end()) {
+            std::cout << "Duplicate entry for " << (*m_index)[idx].m_pkey
+                      << '\n';
+            PSP_COMPLAIN_AND_ABORT("Exiting");
+        }
+
+        pkey_set.insert((*m_index)[idx].m_pkey);
+    }
+}
+
+bool
+t_ftrav::validate_cells(const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    t_index trav_size = size();
+
+    for (const auto& cell : cells) {
+        t_index ridx = cell.first;
+        if (ridx >= trav_size) {
+            return false;
+        }
+    }
+    return true;
+}
+
+void
+t_ftrav::step_begin() {
+    m_step_deletes = 0;
+    m_step_inserts = 0;
+    m_new_elems.clear();
+}
+
+void
+t_ftrav::step_end() {
+    // Fast path: if no incremental work happened this step, `m_index` and
+    // `m_pkeyidx` are already in their final shape (either unchanged, or
+    // populated directly via `bulk_load_append`). Skip the O(N) rebuild.
+    if (m_step_inserts == 0 && m_step_deletes == 0 && m_new_elems.empty()) {
+        return;
+    }
+
+    // The new number of rows in this traversal
+    t_index new_size = m_index->size() + m_step_inserts - m_step_deletes;
+
+    auto new_index = std::make_shared<std::vector<t_mselem>>();
+    new_index->reserve(new_size);
+
+    t_uindex i = 0;
+    t_multisorter sorter(get_sort_orders(m_sortby));
+
+    std::vector<t_mselem> new_rows;
+    new_rows.reserve(m_new_elems.size());
+
+    for (tsl::hopscotch_map<t_tscalar, t_mselem>::const_iterator pkelem_iter =
+             m_new_elems.begin();
+         pkelem_iter != m_new_elems.end();
+         ++pkelem_iter) {
+        new_rows.push_back(pkelem_iter->second);
+    }
+
+    // TODO: int/float/date/datetime pkeys are already sorted here, so if
+    // there was a way to assert that `psp_pkey` is a string typed column,
+    // we can conditional the sort on whether m_sortby.size() > 0 or if
+    // psp_pkey is a string column.
+    std::sort(new_rows.begin(), new_rows.end(), sorter);
+
+    for (auto& new_elem : new_rows) {
+        while (i < m_index->size()) {
+            const t_mselem& old_elem = (*m_index)[i];
+
+            if (old_elem.m_deleted) {
+                i++;
+                m_pkeyidx.erase(old_elem.m_pkey);
+            } else if (old_elem.m_updated) {
+                i++;
+            } else if (sorter(old_elem, new_elem)) {
+                m_pkeyidx[old_elem.m_pkey] = new_index->size();
+                new_index->push_back(old_elem);
+                i++;
+            } else {
+                break;
+            }
+        }
+
+        m_pkeyidx[new_elem.m_pkey] = new_index->size();
+        new_index->push_back(new_elem);
+    }
+
+    // reconcile old rows that are marked as removed or updated.
+    while (i < m_index->size()) {
+        const t_mselem& old_elem = (*m_index)[i++];
+
+        if (old_elem.m_deleted) {
+            m_pkeyidx.erase(old_elem.m_pkey);
+        } else if (!old_elem.m_updated) {
+            // Add back cells that have not changed during this step.
+            m_pkeyidx[old_elem.m_pkey] = new_index->size();
+            new_index->push_back(old_elem);
+        }
+    }
+
+    std::swap(new_index, m_index);
+    m_new_elems.clear();
+}
+
+void
+t_ftrav::add_row(
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table,
+    const t_config& config,
+    t_tscalar pkey
+) {
+    t_mselem mselem;
+    fill_sort_elem(gstate, expression_master_table, config, pkey, mselem);
+    m_new_elems[pkey] = mselem;
+    ++m_step_inserts;
+}
+
+void
+t_ftrav::update_row(
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table,
+    const t_config& config,
+    t_tscalar pkey
+) {
+    if (m_sortby.empty()) {
+        return;
+    }
+    auto pkiter = m_pkeyidx.find(pkey);
+    if (pkiter == m_pkeyidx.end()) {
+        add_row(gstate, expression_master_table, config, pkey);
+        return;
+    }
+    t_mselem mselem;
+    fill_sort_elem(gstate, expression_master_table, config, pkey, mselem);
+    (*m_index)[pkiter->second].m_updated = true;
+    m_new_elems[pkey] = mselem;
+}
+
+void
+t_ftrav::delete_row(t_tscalar pkey) {
+    auto pkiter = m_pkeyidx.find(pkey);
+    if (pkiter == m_pkeyidx.end()) {
+        return;
+    }
+    (*m_index)[pkiter->second].m_deleted = true;
+    m_new_elems.erase(pkey);
+    ++m_step_deletes;
+}
+
+std::vector<t_sortspec>
+t_ftrav::get_sort_by() const {
+    return m_sortby;
+}
+
+bool
+t_ftrav::empty_sort_by() const {
+    return m_sortby.empty();
+}
+
+void
+t_ftrav::reset_step_state() {
+    m_step_deletes = 0;
+    m_step_inserts = 0;
+    m_new_elems.clear();
+}
+
+t_uindex
+t_ftrav::lower_bound_row_idx(
+    const t_gstate& gstate,
+    const t_config& config,
+    const std::vector<t_tscalar>& row
+) const {
+    t_multisorter sorter(get_sort_orders(m_sortby));
+    t_mselem target_val;
+
+    fill_sort_elem(gstate, config, row, target_val);
+
+    auto iter =
+        std::lower_bound(m_index->begin(), m_index->end(), target_val, sorter);
+
+    return std::distance(m_index->begin(), iter);
+}
+
+t_index
+t_ftrav::get_row_idx(t_tscalar pkey) const {
+    auto pkiter = m_pkeyidx.find(pkey);
+    if (pkiter == m_pkeyidx.end()) {
+        return -1;
+    }
+    return pkiter->second;
+}
+
+t_tscalar
+t_ftrav::get_from_gstate(
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table,
+    const std::string& colname,
+    t_tscalar pkey
+) const {
+    const t_schema& expression_schema = expression_master_table.get_schema();
+
+    if (expression_schema.has_column(colname)) {
+        return gstate.get(expression_master_table, colname, pkey);
+    }
+    std::shared_ptr<t_data_table> master_table = gstate.get_table();
+    return gstate.get(*master_table, colname, pkey);
+}
+
+void
+t_ftrav::bulk_load_reserve(t_uindex n) {
+    m_index->reserve(n);
+    m_pkeyidx.reserve(n);
+}
+
+void
+t_ftrav::bulk_load_append(t_tscalar pkey) {
+    // Pre-condition: `empty_sort_by()` and the caller has ownership of
+    // step framing (i.e. we're inside a `step_begin`/`step_end` pair for
+    // an initial registration). `m_step_inserts` is intentionally NOT
+    // incremented so that `step_end` takes its short-circuit path.
+    // `m_pkeyidx` is populated in `bulk_load_finalize` after sorting,
+    // not here — the pkey-to-index mapping only has meaning once the
+    // final sort order is determined.
+    m_index->emplace_back();
+    m_index->back().m_pkey = pkey;
+}
+
+void
+t_ftrav::bulk_load_finalize() {
+    // Match the order the existing `add_row`/`step_end` path would
+    // produce for an empty sort spec: `cmp_mselem` with zero sort
+    // columns falls through to `a.m_pkey < b.m_pkey`, so sort
+    // `m_index` by pkey and then rebuild `m_pkeyidx` against the final
+    // row positions.
+    std::sort(
+        m_index->begin(),
+        m_index->end(),
+        [](const t_mselem& a, const t_mselem& b) {
+            return a.m_pkey < b.m_pkey;
+        }
+    );
+    for (t_uindex i = 0, loop_end = m_index->size(); i < loop_end; ++i) {
+        m_pkeyidx[(*m_index)[i].m_pkey] = i;
+    }
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/get_data_extents.cpp

@@ -0,0 +1,48 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/get_data_extents.h>
+
+namespace perspective {
+t_get_data_extents
+sanitize_get_data_extents(
+    t_index nrows,
+    t_index ncols,
+    t_index start_row,
+    t_index end_row,
+    t_index start_col,
+    t_index end_col
+) {
+    start_row = std::min(start_row, nrows);
+    end_row = std::min(end_row, nrows);
+
+    start_row = std::max(t_index(0), start_row);
+    end_row = std::max(t_index(0), end_row);
+    end_row = std::max(start_row, end_row);
+
+    start_col = std::min(start_col, ncols);
+    end_col = std::min(end_col, ncols);
+
+    start_col = std::max(t_index(0), start_col);
+    end_col = std::max(t_index(0), end_col);
+    end_col = std::max(start_col, end_col);
+
+    t_get_data_extents rval;
+    rval.m_srow = start_row;
+    rval.m_erow = end_row;
+    rval.m_scol = start_col;
+    rval.m_ecol = end_col;
+
+    return rval;
+}
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/gnode_state.cpp

@@ -0,0 +1,886 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/context_unit.h>
+#include <perspective/context_zero.h>
+#include <perspective/context_one.h>
+#include <perspective/context_two.h>
+#include <perspective/gnode_state.h>
+#include <perspective/mask.h>
+#include <perspective/sym_table.h>
+#include <perspective/parallel_for.h>
+#include <perspective/utils.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_gstate::t_gstate(
+    t_schema input_schema,
+    t_schema output_schema,
+    t_backing_store backing_store
+) :
+    m_input_schema(std::move(input_schema)),
+    m_output_schema(std::move(output_schema)),
+    m_backing_store(backing_store),
+    m_init(false) {
+    LOG_CONSTRUCTOR("t_gstate");
+}
+
+t_gstate::~t_gstate() { LOG_DESTRUCTOR("t_gstate"); }
+
+void
+t_gstate::init() {
+    // The master `t_data_table` is the canonical copy of the dataset and is the
+    // only table eligible for on-disk backing. When `BACKING_STORE_DISK` is
+    // requested, the column files live in a unique per-table directory under
+    // the OS temp directory (native only; WASM/OPFS is handled separately).
+    std::string dirname;
+    if (m_backing_store == BACKING_STORE_DISK) {
+        dirname = create_backing_store_dir("perspective_");
+    }
+    m_table = std::make_shared<t_data_table>(
+        "", dirname, m_input_schema, DEFAULT_EMPTY_CAPACITY, m_backing_store
+    );
+    m_table->init();
+    m_pkcol = m_table->get_column("psp_pkey");
+    m_opcol = m_table->get_column("psp_op");
+    m_init = true;
+}
+
+t_rlookup
+t_gstate::lookup(t_tscalar pkey) const {
+    t_rlookup rval(0, false);
+
+    t_mapping::const_iterator iter = m_mapping.find(pkey);
+
+    if (iter == m_mapping.end()) {
+        return rval;
+    }
+
+    rval.m_idx = iter->second;
+    rval.m_exists = true;
+    return rval;
+}
+
+void
+t_gstate::_mark_deleted(t_uindex idx) {
+    m_free.insert(idx);
+}
+
+void
+t_gstate::erase(const t_tscalar& pkey) {
+    t_mapping::const_iterator iter = m_mapping.find(pkey);
+
+    if (iter == m_mapping.end()) {
+        return;
+    }
+
+    auto columns = m_table->get_columns();
+
+    t_uindex idx = iter->second;
+
+    for (auto* c : columns) {
+        c->clear(idx);
+    }
+
+    m_mapping.erase(iter);
+    _mark_deleted(idx);
+}
+
+t_uindex
+t_gstate::lookup_or_create(const t_tscalar& pkey) {
+    auto pkey_ = m_symtable.get_interned_tscalar(pkey);
+    t_mapping::const_iterator iter = m_mapping.find(pkey_);
+
+    if (iter != m_mapping.end()) {
+        return iter->second;
+    }
+
+    if (!m_free.empty()) {
+        t_free_items::const_iterator iter = m_free.begin();
+        t_uindex idx = *iter;
+        m_free.erase(iter);
+        m_mapping.emplace(pkey_, idx);
+        return idx;
+    }
+
+    t_uindex nrows = m_table->num_rows();
+    if (nrows >= m_table->get_capacity() - 1) {
+        m_table->reserve(std::max(
+            nrows + 1,
+            static_cast<t_uindex>(
+                m_table->get_capacity() * PSP_TABLE_GROW_RATIO
+            )
+        ));
+    }
+
+    m_table->set_size(nrows + 1);
+    m_opcol->set_nth<std::uint8_t>(nrows, OP_INSERT);
+    m_pkcol->set_scalar(nrows, pkey);
+    m_mapping.emplace(pkey_, nrows);
+    return nrows;
+}
+
+void
+t_gstate::fill_master_table(const std::shared_ptr<t_data_table>& flattened) {
+    // insert into empty `m_table`
+    m_free.clear();
+    m_mapping.clear();
+
+    const t_schema& master_table_schema = m_table->get_schema();
+
+    const t_column* flattened_pkey_col =
+        flattened->_get_const_column("psp_pkey");
+    const t_column* flattened_op_col =
+        flattened->_get_const_column("psp_op");
+
+    t_uindex ncols = m_table->num_columns();
+    auto* master_table = m_table.get();
+    const bool page_to_disk = m_backing_store == BACKING_STORE_DISK;
+
+    parallel_for(
+        int(ncols),
+        [&master_table, &master_table_schema, &flattened, page_to_disk](int idx) {
+            const std::string& column_name = master_table_schema.m_columns[idx];
+            auto flattened_column = flattened->get_column_safe(column_name);
+            if (!flattened_column) {
+                return;
+            }
+            if (page_to_disk) {
+                // The master table is on-disk; preserve its disk-backed column
+                // and deep-copy the flattened (in-memory) column's data into
+                // it rather than aliasing the in-memory `shared_ptr`.
+                master_table->_get_column(column_name)
+                    ->copy_from(*flattened_column);
+            } else {
+                // Alias each column `shared_ptr` from flattened into `m_table`
+                // rather than deep-cloning.
+                master_table->set_column(idx, std::move(flattened_column));
+            }
+        }
+    );
+
+    m_pkcol = master_table->get_column("psp_pkey");
+    m_opcol = master_table->get_column("psp_op");
+
+    master_table->set_capacity(flattened->get_capacity());
+    master_table->set_size(flattened->size());
+
+    const std::uint8_t* flattened_op_base =
+        flattened_op_col->get_nth_base<std::uint8_t>();
+    for (t_uindex idx = 0, loop_end = flattened->num_rows(); idx < loop_end;
+         ++idx) {
+        t_tscalar pkey = flattened_pkey_col->get_scalar(idx);
+        t_op op = static_cast<t_op>(flattened_op_base[idx]);
+
+        switch (op) {
+            case OP_INSERT: {
+                // Write new primary keys into `m_mapping`
+                m_mapping.emplace(
+                    m_symtable.get_interned_tscalar(pkey), idx
+                );
+                m_opcol->set_nth<std::uint8_t>(idx, OP_INSERT);
+                m_pkcol->set_scalar(idx, pkey);
+            } break;
+            case OP_DELETE: {
+                _mark_deleted(idx);
+            } break;
+            default: {
+                PSP_COMPLAIN_AND_ABORT("Unexpected OP");
+            } break;
+        }
+    }
+
+#ifdef PSP_TABLE_VERIFY
+    master_table->verify();
+#endif
+}
+
+void
+t_gstate::init_from_table(const std::shared_ptr<t_data_table>& source) {
+    // Bulk-init the master table directly from `source`. Assumes:
+    //   - master table is empty
+    //   - all rows are `OP_INSERT`
+    //   - `psp_pkey` has no duplicates
+    //
+    // Column `shared_ptr`s are aliased rather than deep-cloned.
+    m_free.clear();
+    m_mapping.clear();
+
+    const t_schema& master_table_schema = m_table->get_schema();
+    t_uindex ncols = m_table->num_columns();
+    auto* master_table = m_table.get();
+    const bool page_to_disk = m_backing_store == BACKING_STORE_DISK;
+
+    parallel_for(
+        int(ncols),
+        [&master_table, &master_table_schema, &source, page_to_disk](int idx) {
+            const std::string& column_name = master_table_schema.m_columns[idx];
+            auto src_col = source->get_column_safe(column_name);
+            if (!src_col) {
+                return;
+            }
+            if (page_to_disk) {
+                // Preserve the disk-backed master column; deep-copy rather than
+                // aliasing the in-memory source column.
+                master_table->_get_column(column_name)->copy_from(*src_col);
+            } else {
+                master_table->set_column(idx, std::move(src_col));
+            }
+        }
+    );
+
+    m_pkcol = master_table->get_column("psp_pkey");
+    m_opcol = master_table->get_column("psp_op");
+
+    master_table->set_capacity(source->get_capacity());
+    master_table->set_size(source->size());
+
+    const t_column* pkey_col = master_table->_get_const_column("psp_pkey");
+    for (t_uindex idx = 0, loop_end = source->num_rows(); idx < loop_end;
+         ++idx) {
+        t_tscalar pkey = pkey_col->get_scalar(idx);
+        m_mapping.emplace(m_symtable.get_interned_tscalar(pkey), idx);
+    }
+
+#ifdef PSP_TABLE_VERIFY
+    master_table->verify();
+#endif
+}
+
+void
+t_gstate::update_master_table(
+    const std::shared_ptr<t_data_table>& flattened
+) {
+    if (num_rows() == 0) {
+        fill_master_table(flattened);
+        return;
+    }
+
+    // Update existing `m_table`
+    const t_column* flattened_pkey_col =
+        flattened->_get_const_column("psp_pkey");
+    const t_column* flattened_op_col =
+        flattened->_get_const_column("psp_op");
+
+    t_data_table* master_table = m_table.get();
+    std::vector<t_uindex> master_table_indexes(flattened->num_rows());
+    const std::uint8_t* flattened_op_base =
+        flattened_op_col->get_nth_base<std::uint8_t>();
+
+    for (t_uindex idx = 0, loop_end = flattened->num_rows(); idx < loop_end;
+         ++idx) {
+        t_tscalar pkey = flattened_pkey_col->get_scalar(idx);
+        t_op op = static_cast<t_op>(flattened_op_base[idx]);
+
+        switch (op) {
+            case OP_INSERT: {
+                // Lookup/create the row index in `m_table` based on pkey
+                master_table_indexes[idx] = lookup_or_create(pkey);
+
+                // Write the op and pkey to `m_table`
+                m_opcol->set_nth<std::uint8_t>(
+                    master_table_indexes[idx], OP_INSERT
+                );
+                m_pkcol->set_scalar(master_table_indexes[idx], pkey);
+            } break;
+            case OP_DELETE: {
+                // Erase the pkey from the master table, but this does not
+                // change the size as the row isn't removed, just cleared out.
+                erase(pkey);
+            } break;
+            default: {
+                PSP_COMPLAIN_AND_ABORT("Unexpected OP");
+            } break;
+        }
+    }
+
+    const t_schema& master_schema = m_table->get_schema();
+    t_uindex ncols = master_table->num_columns();
+
+    parallel_for(
+        int(ncols),
+        [flattened,
+         flattened_op_col,
+         &master_schema,
+         &master_table,
+         &master_table_indexes,
+         this](int idx) {
+            const std::string& column_name = master_schema.m_columns[idx];
+            t_column* master_column =
+                master_table->_get_column(column_name);
+            const t_column* flattened_column =
+                flattened->_get_const_column_safe(column_name);
+            if (!flattened_column) {
+                return;
+            }
+            update_master_column(
+                master_column,
+                flattened_column,
+                flattened_op_col,
+                master_table_indexes,
+                flattened->num_rows()
+            );
+        }
+    );
+}
+
+void
+t_gstate::update_master_column(
+    t_column* master_column,
+    const t_column* flattened_column,
+    const t_column* op_column,
+    const std::vector<t_uindex>& master_table_indexes,
+    t_uindex num_rows
+) {
+    const std::uint8_t* op_base = op_column->get_nth_base<std::uint8_t>();
+    for (t_uindex idx = 0, loop_end = num_rows; idx < loop_end; ++idx) {
+        bool is_valid = flattened_column->is_valid(idx);
+        t_uindex master_table_idx = master_table_indexes[idx];
+
+        if (!is_valid) {
+            bool is_cleared = flattened_column->is_cleared(idx);
+            if (is_cleared) {
+                master_column->clear(master_table_idx);
+            }
+            continue;
+        }
+
+        t_op op = static_cast<t_op>(op_base[idx]);
+
+        if (op == OP_DELETE) {
+            continue;
+        }
+
+        switch (flattened_column->get_dtype()) {
+            case DTYPE_NONE: {
+            } break;
+            case DTYPE_INT64: {
+                master_column->set_nth<std::int64_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::int64_t>(idx))
+                );
+            } break;
+            case DTYPE_INT32: {
+                master_column->set_nth<std::int32_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::int32_t>(idx))
+                );
+            } break;
+            case DTYPE_INT16: {
+                master_column->set_nth<std::int16_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::int16_t>(idx))
+                );
+            } break;
+            case DTYPE_INT8: {
+                master_column->set_nth<std::int8_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::int8_t>(idx))
+                );
+            } break;
+            case DTYPE_UINT64: {
+                master_column->set_nth<std::uint64_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint64_t>(idx))
+                );
+            } break;
+            case DTYPE_UINT32: {
+                master_column->set_nth<std::uint32_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint32_t>(idx))
+                );
+            } break;
+            case DTYPE_UINT16: {
+                master_column->set_nth<std::uint16_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint16_t>(idx))
+                );
+            } break;
+            case DTYPE_UINT8: {
+                master_column->set_nth<std::uint8_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint8_t>(idx))
+                );
+            } break;
+            case DTYPE_FLOAT64: {
+                master_column->set_nth<double>(
+                    master_table_idx, *(flattened_column->get_nth<double>(idx))
+                );
+            } break;
+            case DTYPE_FLOAT32: {
+                master_column->set_nth<float>(
+                    master_table_idx, *(flattened_column->get_nth<float>(idx))
+                );
+            } break;
+            case DTYPE_BOOL: {
+                master_column->set_nth<std::uint8_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint8_t>(idx))
+                );
+            } break;
+            case DTYPE_TIME: {
+                master_column->set_nth<std::int64_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::int64_t>(idx))
+                );
+            } break;
+            case DTYPE_DATE: {
+                master_column->set_nth<std::uint32_t>(
+                    master_table_idx,
+                    *(flattened_column->get_nth<std::uint32_t>(idx))
+                );
+            } break;
+            case DTYPE_STR: {
+                master_column->set_nth<const char*>(
+                    master_table_idx, flattened_column->get_nth<const char>(idx)
+                );
+            } break;
+            case DTYPE_OBJECT:
+            default: {
+                PSP_COMPLAIN_AND_ABORT("Unexpected type");
+            }
+        }
+    }
+}
+
+void
+t_gstate::pprint() const {
+    std::vector<t_uindex> indices(m_mapping.size());
+    t_uindex idx = 0;
+    for (const auto& iter : m_mapping) {
+        indices[idx] = iter.second;
+        ++idx;
+    }
+    m_table->pprint(indices);
+}
+
+t_mask
+t_gstate::get_cpp_mask() const {
+    t_uindex sz = m_table->size();
+    t_mask msk(sz);
+    for (const auto& iter : m_mapping) {
+        msk.set(iter.second, true);
+    }
+    return msk;
+}
+
+std::shared_ptr<t_data_table>
+t_gstate::get_table() const {
+    return m_table;
+}
+
+t_tscalar
+t_gstate::read_by_pkey(
+    const t_data_table& table, const std::string& colname, t_tscalar& pkey
+) const {
+    const t_column* col_ = table._get_const_column(colname);
+    t_mapping::const_iterator iter = m_mapping.find(pkey);
+    if (iter != m_mapping.end()) {
+        return col_->get_scalar(iter->second);
+    }
+    PSP_COMPLAIN_AND_ABORT("Called without pkey");
+}
+
+void
+t_gstate::resolve_pkeys(
+    const std::vector<t_tscalar>& pkeys,
+    std::vector<t_uindex>& out_row_indices
+) const {
+    t_uindex num_rows = pkeys.size();
+    out_row_indices.resize(num_rows);
+    constexpr t_uindex INVALID_ROW = t_uindex(-1);
+    for (t_uindex idx = 0; idx < num_rows; ++idx) {
+        t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
+        if (iter != m_mapping.end()) {
+            out_row_indices[idx] = iter->second;
+        } else {
+            out_row_indices[idx] = INVALID_ROW;
+        }
+    }
+}
+
+void
+t_gstate::read_column(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_tscalar>& pkeys,
+    std::vector<t_tscalar>& out_data
+) const {
+    t_index num_rows = pkeys.size();
+    const t_column* col_ = table._get_const_column(colname);
+    out_data.resize(num_rows);
+
+    for (t_index idx = 0; idx < num_rows; ++idx) {
+        t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
+        if (iter != m_mapping.end()) {
+            out_data[idx].set(col_->get_scalar(iter->second));
+        } else {
+            out_data[idx] = t_tscalar();
+        }
+    }
+}
+
+void
+t_gstate::read_column(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_tscalar>& pkeys,
+    std::vector<double>& out_data
+) const {
+    read_column(table, colname, pkeys, out_data, true);
+}
+
+void
+t_gstate::read_column(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_tscalar>& pkeys,
+    std::vector<double>& out_data,
+    bool include_nones
+) const {
+    t_index num_rows = pkeys.size();
+    const t_column* col_ = table._get_const_column(colname);
+
+    out_data.clear();
+    out_data.reserve(num_rows);
+    for (t_index idx = 0; idx < num_rows; ++idx) {
+        t_mapping::const_iterator iter = m_mapping.find(pkeys[idx]);
+        if (iter != m_mapping.end()) {
+            auto tscalar = col_->get_scalar(iter->second);
+            if (include_nones || tscalar.is_valid()) {
+                out_data.push_back(tscalar.to_double());
+            }
+        }
+    }
+}
+
+void
+t_gstate::read_column(
+    const t_data_table& table,
+    const std::string& colname,
+    t_uindex start_idx,
+    t_uindex end_idx,
+    std::vector<t_tscalar>& out_data
+) const {
+    t_index num_rows = end_idx - start_idx;
+
+    // Don't read invalid row indices.
+    if (num_rows <= 0) {
+        return;
+    }
+
+    const t_column* col_ = table._get_const_column(colname);
+
+    out_data.resize(num_rows);
+
+    t_uindex i = 0;
+    for (t_uindex idx = start_idx; idx < end_idx; ++idx) {
+        out_data[i] = col_->get_scalar(idx);
+        i++;
+    }
+}
+
+void
+t_gstate::read_column(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_uindex>& row_indices,
+    std::vector<t_tscalar>& out_data
+) const {
+    const t_column* col_ = table._get_const_column(colname);
+
+    t_index num_rows = row_indices.size();
+    out_data.resize(num_rows);
+
+    t_uindex i = 0;
+    for (auto idx : row_indices) {
+        out_data[i] = col_->get_scalar(idx);
+        i++;
+    }
+}
+
+t_tscalar
+t_gstate::get(
+    const t_data_table& table, const std::string& colname, t_tscalar pkey
+) const {
+    t_mapping::const_iterator iter = m_mapping.find(pkey);
+    if (iter != m_mapping.end()) {
+        return table._get_const_column(colname)->get_scalar(iter->second);
+    }
+
+    return {};
+}
+
+t_tscalar
+t_gstate::get_value(
+    const t_data_table& table, const std::string& colname, const t_tscalar& pkey
+) const {
+    const t_column* col_ = table._get_const_column(colname);
+    t_tscalar rval = mknone();
+
+    auto iter = m_mapping.find(pkey);
+    if (iter != m_mapping.end()) {
+        rval.set(col_->get_scalar(iter->second));
+    }
+
+    return rval;
+}
+
+bool
+t_gstate::is_unique(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_tscalar>& pkeys,
+    t_tscalar& value
+) const {
+    const t_column* col_ = table._get_const_column(colname);
+    value = mknone();
+
+    for (const auto& pkey : pkeys) {
+        t_mapping::const_iterator iter = m_mapping.find(pkey);
+        if (iter != m_mapping.end()) {
+            auto tmp = col_->get_scalar(iter->second);
+            if (!value.is_none() && value != tmp) {
+                return false;
+            }
+            value = tmp;
+        }
+    }
+
+    return true;
+}
+
+bool
+t_gstate::apply(
+    const t_data_table& table,
+    const std::string& colname,
+    const std::vector<t_tscalar>& pkeys,
+    t_tscalar& value,
+    const std::function<bool(const t_tscalar&, t_tscalar&)>& fn
+) const {
+    const t_column* col_ = table._get_const_column(colname);
+
+    value = mknone();
+
+    for (const auto& pkey : pkeys) {
+        t_mapping::const_iterator iter = m_mapping.find(pkey);
+        if (iter != m_mapping.end()) {
+            auto tmp = col_->get_scalar(iter->second);
+            bool done = fn(tmp, value);
+            if (done) {
+                value = tmp;
+                return done;
+            }
+        }
+    }
+
+    return false;
+}
+
+const t_schema&
+t_gstate::get_output_schema() const {
+    return m_output_schema;
+}
+
+const t_gstate::t_mapping&
+t_gstate::get_pkey_map() const {
+    return m_mapping;
+}
+
+t_dtype
+t_gstate::get_pkey_dtype() const {
+    if (m_mapping.empty()) {
+        return DTYPE_STR;
+    }
+    auto iter = m_mapping.begin();
+    return iter->first.get_dtype();
+}
+
+std::shared_ptr<t_data_table>
+t_gstate::get_pkeyed_table() const {
+    return get_pkeyed_table(m_input_schema, m_table);
+}
+
+std::shared_ptr<t_data_table>
+t_gstate::get_pkeyed_table(
+    const t_schema& schema, const std::shared_ptr<t_data_table>& table
+) const {
+    // If there are no removes, just return the gstate table. Removes would
+    // cause m_mapping to be smaller than m_table.
+    if (m_mapping.size() == table->size()) {
+        return table;
+    }
+
+    // Otherwise mask out the removed rows and return the table.
+    auto mask = get_cpp_mask();
+
+    // count = total number of rows - number of removed rows
+    t_uindex table_size = mask.count();
+
+    const auto& schema_columns = schema.m_columns;
+    t_uindex num_columns = schema_columns.size();
+
+    std::shared_ptr<t_data_table> rval =
+        std::make_shared<t_data_table>(schema, table_size);
+    rval->init();
+    rval->set_size(table_size);
+
+    parallel_for(
+        int(num_columns),
+        [&schema_columns, rval, table, &mask](int colidx) {
+            const std::string& colname = schema_columns[colidx];
+            rval->set_column(
+                colname, table->_get_const_column(colname)->clone(mask)
+            );
+        }
+
+    );
+
+    return rval;
+}
+
+t_uindex
+t_gstate::num_rows() const {
+    return m_table->num_rows();
+}
+
+t_uindex
+t_gstate::num_columns() const {
+    return m_table->num_columns();
+}
+
+std::vector<t_tscalar>
+t_gstate::get_row_data_pkeys(const std::vector<t_tscalar>& pkeys) const {
+    t_uindex ncols = m_table->num_columns();
+    const t_schema& schema = m_table->get_schema();
+    std::vector<t_tscalar> rval;
+    rval.reserve(pkeys.size() * ncols);
+
+    std::vector<const t_column*> columns(ncols);
+    for (t_uindex idx = 0, loop_end = schema.size(); idx < loop_end; ++idx) {
+        const std::string& cname = schema.m_columns[idx];
+        columns[idx] = m_table->_get_const_column(cname);
+    }
+
+    auto none = mknone();
+
+    for (const auto& pkey : pkeys) {
+        t_mapping::const_iterator iter = m_mapping.find(pkey);
+        if (iter == m_mapping.end()) {
+            continue;
+        }
+
+        for (t_uindex cidx = 0; cidx < ncols; ++cidx) {
+            auto v = columns[cidx]->get_scalar(iter->second);
+            if (v.is_valid()) {
+                rval.push_back(v);
+            } else {
+                rval.push_back(none);
+            }
+        }
+    }
+    return rval;
+}
+
+bool
+t_gstate::has_pkey(t_tscalar pkey) const {
+    return m_mapping.find(pkey) != m_mapping.end();
+}
+
+std::vector<t_tscalar>
+t_gstate::has_pkeys(const std::vector<t_tscalar>& pkeys) const {
+    if (pkeys.empty()) {
+        return {};
+    }
+
+    std::vector<t_tscalar> rval(pkeys.size());
+    t_uindex idx = 0;
+
+    for (const auto& p : pkeys) {
+        t_tscalar tval;
+        tval.set(m_mapping.find(p) != m_mapping.end());
+        rval[idx].set(tval);
+        ++idx;
+    }
+
+    return rval;
+}
+
+std::vector<t_tscalar>
+t_gstate::get_pkeys() const {
+    std::vector<t_tscalar> rval(m_mapping.size());
+    t_uindex idx = 0;
+    for (const auto& kv : m_mapping) {
+        rval[idx].set(kv.first);
+        ++idx;
+    }
+    return rval;
+}
+
+std::pair<t_tscalar, t_tscalar>
+get_vec_min_max(const std::vector<t_tscalar>& vec) {
+    t_tscalar min = mknone();
+    t_tscalar max = mknone();
+
+    for (const auto& v : vec) {
+        if (min.is_none()) {
+            min = v;
+        } else {
+            min = std::min(v, min);
+        }
+
+        if (max.is_none()) {
+            max = v;
+        } else {
+            max = std::max(v, max);
+        }
+    }
+
+    return std::pair<t_tscalar, t_tscalar>(min, max);
+}
+
+t_uindex
+t_gstate::mapping_size() const {
+    return m_mapping.size();
+}
+
+void
+t_gstate::reset() {
+    m_table->reset();
+    m_mapping.clear();
+    m_free.clear();
+}
+
+const t_schema&
+t_gstate::get_input_schema() const {
+    return m_input_schema;
+}
+
+std::vector<t_uindex>
+t_gstate::get_pkeys_idx(const std::vector<t_tscalar>& pkeys) const {
+    std::vector<t_uindex> rv;
+    rv.reserve(pkeys.size());
+
+    for (const auto& p : pkeys) {
+        auto lk = lookup(p);
+        std::cout << "pkey " << p << " exists " << lk.m_exists << '\n';
+        if (lk.m_exists) {
+            rv.push_back(lk.m_idx);
+        }
+    }
+    return rv;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/heap_instruments.cpp

@@ -0,0 +1,234 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include "perspective/base.h"
+#include "perspective/heap_instruments.h"
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <emscripten/emscripten.h>
+#include <emscripten/heap.h>
+#include <emscripten/em_asm.h>
+#include <emscripten/stack.h>
+#include <string>
+
+static std::uint64_t USED_MEMORY = 0;
+
+static constexpr std::uint64_t MIN_RELEVANT_SIZE = 5 * 1024 * 1024;
+
+extern "C" void
+psp_print_used_memory() {
+    printf("Used memory: %llu\n", USED_MEMORY);
+}
+
+struct Header {
+    std::uint64_t size;
+};
+
+using UnderlyingString =
+    std::basic_string<char, std::char_traits<char>, UnderlyingAllocator<char>>;
+
+using UnderlyingIStringStream = std::basic_istringstream<
+    char,
+    std::char_traits<char>,
+    UnderlyingAllocator<char>>;
+
+struct AllocMeta {
+    Header header;
+    const UnderlyingString* trace;
+    std::uint64_t size;
+};
+
+static std::unordered_map<
+    UnderlyingString,
+    AllocMeta,
+    std::hash<UnderlyingString>,
+    std::equal_to<>,
+    UnderlyingAllocator<std::pair<UnderlyingString const, AllocMeta>>>
+    stack_traces;
+
+static UnderlyingString IRRELEVANT = "irrelevant";
+
+static inline void
+record_stack_trace(Header* header, std::uint64_t size) {
+    if (size >= MIN_RELEVANT_SIZE) {
+        const char* stack_c_str = perspective::psp_stack_trace();
+        UnderlyingIStringStream stack(stack_c_str);
+        UnderlyingString line;
+        UnderlyingString out;
+
+        while (std::getline(stack, line)) {
+            line = line.substr(0, line.find_last_of(" ("));
+            out += line + "\n";
+        }
+
+        emscripten_builtin_free(const_cast<char*>(stack_c_str));
+
+        // stack_traces[ptr] = {.header = *header, .trace = out};
+        if (stack_traces.find(out) == stack_traces.end()) {
+            stack_traces[out] =
+                AllocMeta{.header = *header, .trace = nullptr, .size = size};
+
+            stack_traces[out].trace = &stack_traces.find(out)->first;
+        } else {
+            stack_traces[out].size += size;
+        }
+    } else {
+        if (stack_traces.find(IRRELEVANT) == stack_traces.end()) {
+            stack_traces[IRRELEVANT] = AllocMeta{
+                .header = *header, .trace = &IRRELEVANT, .size = size
+            };
+        } else {
+            stack_traces[IRRELEVANT].size += size;
+        }
+    }
+}
+
+extern "C" void
+psp_dump_stack_traces() {
+    std::vector<AllocMeta, UnderlyingAllocator<AllocMeta>> metas;
+    metas.reserve(stack_traces.size());
+    for (const auto& [_, meta] : stack_traces) {
+        metas.push_back(meta);
+    }
+    std::sort(
+        metas.begin(),
+        metas.end(),
+        [](const AllocMeta& a, const AllocMeta& b) {
+            return a.header.size > b.header.size;
+        }
+    );
+    for (const auto& meta : metas) {
+        printf("Allocated %llu bytes\n", meta.header.size);
+        printf("Stacktrace:\n%s\n", meta.trace->c_str());
+    }
+}
+
+extern "C" void
+psp_clear_stack_traces() {
+    stack_traces.clear();
+}
+
+void*
+malloc(size_t size) {
+    if (size > MIN_RELEVANT_SIZE) {
+        printf("Allocating %zu bytes\n", size);
+    }
+    USED_MEMORY += size;
+    const size_t total_size = size + sizeof(Header);
+    auto* header = static_cast<Header*>(emscripten_builtin_malloc(total_size));
+    if (header == nullptr) {
+        fprintf(stderr, "Failed to allocate %zu bytes\n", size);
+    }
+    header->size = size;
+    record_stack_trace(header, size);
+    return header + 1;
+}
+
+void*
+calloc(size_t nmemb, size_t size) {
+    // printf("Allocating array: %zu elements of size %zu\n", nmemb, size);
+    USED_MEMORY += nmemb * size;
+    // return emscripten_builtin_calloc(nmemb, size);
+    const size_t total_size = (nmemb * size) + sizeof(Header);
+    auto* header = static_cast<Header*>(emscripten_builtin_malloc(total_size));
+    if (header == nullptr) {
+        fprintf(stderr, "Failed to allocate %zu bytes\n", size);
+    }
+    header->size = nmemb * size;
+    memset(header + 1, 0, nmemb * size);
+    record_stack_trace(header, size);
+    return header + 1;
+}
+
+void
+free(void* ptr) {
+    // printf("Freeing memory at %p\n", ptr);
+
+    if (ptr == nullptr) {
+        emscripten_builtin_free(ptr);
+    } else {
+        auto* header = static_cast<Header*>(ptr) - 1;
+        auto old_memory = USED_MEMORY;
+        USED_MEMORY -= header->size;
+        if (USED_MEMORY > old_memory) {
+            std::abort();
+        }
+        emscripten_builtin_free(header);
+    }
+}
+
+void*
+memalign(size_t alignment, size_t size) {
+    const size_t total_size = size + sizeof(Header);
+    auto* header =
+        static_cast<Header*>(emscripten_builtin_memalign(alignment, total_size)
+        );
+    if (header == nullptr) {
+        fprintf(stderr, "Failed to allocate %zu bytes\n", size);
+    }
+    header->size = size;
+    record_stack_trace(header, size);
+    return header + 1;
+}
+
+int
+posix_memalign(void** memptr, size_t alignment, size_t size) {
+    auto* header = static_cast<Header*>(
+        emscripten_builtin_memalign(alignment, size + sizeof(Header))
+    );
+    if (header == nullptr) {
+        fprintf(stderr, "Failed to allocate %zu bytes\n", size);
+    }
+    header->size = size;
+    USED_MEMORY += size;
+    record_stack_trace(header, size);
+    *memptr = header + 1;
+    return 0;
+}
+
+void*
+realloc(void* ptr, size_t new_size) {
+    if (ptr == nullptr) {
+        // If ptr is nullptr, realloc behaves like malloc
+        return malloc(new_size);
+    }
+
+    if (new_size == 0) {
+        // If new_size is 0, realloc behaves like free
+        free(ptr);
+        return nullptr;
+    }
+
+    auto* header = static_cast<Header*>(ptr) - 1;
+    const size_t old_size = header->size;
+
+    if (new_size <= old_size) {
+        USED_MEMORY -= old_size - new_size;
+        // If the new size is smaller or equal, we can potentially shrink the
+        // block in place. For simplicity, we don't actually shrink the block
+        // here.
+        header->size = new_size; // Update the size in the header
+        return ptr;              // Return the same pointer
+    }
+
+    // If the new size is larger, allocate a new block
+    void* new_ptr = malloc(new_size);
+    if (new_ptr == nullptr) {
+        return nullptr;
+    }
+
+    memcpy(new_ptr, ptr, old_size);
+    free(ptr);
+
+    return new_ptr;
+}

rust/perspective-server/cpp/perspective/src/cpp/init.cpp

@@ -0,0 +1,24 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/init.h>
+
+namespace perspective {
+
+void
+perspective_init() {}
+
+void
+perspective_finalize() {}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/join_engine.cpp

@@ -0,0 +1,451 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include "perspective/join_engine.h"
+#include "perspective/column.h"
+#include "perspective/data_table.h"
+#include "perspective/gnode.h"
+#include "perspective/scalar.h"
+#include <algorithm>
+#include <sstream>
+#include <tsl/hopscotch_map.h>
+#include <tsl/hopscotch_set.h>
+
+namespace perspective::server {
+
+namespace {
+
+// Typed column copy for fixed-size types.
+template <typename T>
+void
+copy_column_typed(
+    t_column* dst,
+    const t_column* src,
+    const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
+    t_uindex num_matched,
+    bool use_first
+) {
+    for (t_uindex i = 0; i < num_matched; ++i) {
+        auto src_idx = use_first ? matched_rows[i].first : matched_rows[i].second;
+        if (src_idx == static_cast<t_uindex>(-1)) {
+            dst->clear(i);
+        } else {
+            const T* val = src->get_nth<T>(src_idx);
+            const t_status* st =
+                src->is_status_enabled() ? src->get_nth_status(src_idx)
+                                         : nullptr;
+            dst->set_nth<T>(i, *val, st ? *st : STATUS_VALID);
+        }
+    }
+}
+
+void
+copy_column_str(
+    t_column* dst,
+    const t_column* src,
+    const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
+    t_uindex num_matched,
+    bool use_first
+) {
+    for (t_uindex i = 0; i < num_matched; ++i) {
+        auto src_idx = use_first ? matched_rows[i].first : matched_rows[i].second;
+        if (src_idx == static_cast<t_uindex>(-1)) {
+            dst->clear(i);
+        } else {
+            dst->set_scalar(i, src->get_scalar(src_idx));
+        }
+    }
+}
+
+void
+copy_column_dispatch(
+    t_column* dst,
+    const t_column* src,
+    const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
+    t_uindex num_matched,
+    bool use_first
+) {
+    switch (dst->get_dtype()) {
+        case DTYPE_INT64:
+            copy_column_typed<std::int64_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_INT32:
+            copy_column_typed<std::int32_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_INT16:
+            copy_column_typed<std::int16_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_INT8:
+            copy_column_typed<std::int8_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_UINT64:
+            copy_column_typed<std::uint64_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_UINT32:
+            copy_column_typed<std::uint32_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_UINT16:
+            copy_column_typed<std::uint16_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_UINT8:
+            copy_column_typed<std::uint8_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_FLOAT64:
+            copy_column_typed<double>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_FLOAT32:
+            copy_column_typed<float>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_BOOL:
+            copy_column_typed<bool>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_TIME:
+            copy_column_typed<std::int64_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_DATE:
+            copy_column_typed<std::int32_t>(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        case DTYPE_STR:
+            copy_column_str(
+                dst, src, matched_rows, num_matched, use_first
+            );
+            break;
+        default:
+            // Fallback for any other types (OBJECT, F64PAIR, etc.)
+            for (t_uindex i = 0; i < num_matched; ++i) {
+                auto src_idx = use_first ? matched_rows[i].first
+                                         : matched_rows[i].second;
+                if (src_idx == static_cast<t_uindex>(-1)) {
+                    dst->clear(i);
+                } else {
+                    dst->set_scalar(i, src->get_scalar(src_idx));
+                }
+            }
+            break;
+    }
+}
+
+// Copy the join key column for OUTER join rows where the left side has no
+// match but the right side does. Uses the right key column as the source
+// for just those rows; all other rows are already filled from the left.
+void
+copy_join_key_fallback(
+    t_column* dst,
+    const t_column* right_key_col,
+    const std::vector<std::pair<t_uindex, t_uindex>>& matched_rows,
+    t_uindex num_matched
+) {
+    for (t_uindex i = 0; i < num_matched; ++i) {
+        if (matched_rows[i].first == static_cast<t_uindex>(-1)
+            && matched_rows[i].second != static_cast<t_uindex>(-1)) {
+            dst->set_scalar(
+                i, right_key_col->get_scalar(matched_rows[i].second)
+            );
+        }
+    }
+}
+
+} // anonymous namespace
+
+void
+JoinEngine::register_join(
+    const t_id& join_table_id,
+    const t_id& left_table_id,
+    const t_id& right_table_id,
+    const std::string& on_column,
+    const std::string& right_on_column,
+    proto::JoinType join_type
+) {
+    auto effective_right_on = right_on_column.empty() ? on_column : right_on_column;
+    JoinDef def{left_table_id, right_table_id, on_column, effective_right_on, join_type};
+    m_join_defs.emplace(join_table_id, def);
+    m_table_to_join_tables.emplace(left_table_id, join_table_id);
+    m_table_to_join_tables.emplace(right_table_id, join_table_id);
+}
+
+void
+JoinEngine::unregister_join(const t_id& join_table_id) {
+    auto it = m_join_defs.find(join_table_id);
+    if (it == m_join_defs.end()) {
+        return;
+    }
+
+    auto& def = it->second;
+
+    for (const auto& source_id : {def.left_table_id, def.right_table_id}) {
+        auto range = m_table_to_join_tables.equal_range(source_id);
+        for (auto jt = range.first; jt != range.second;) {
+            if (jt->second == join_table_id) {
+                jt = m_table_to_join_tables.erase(jt);
+            } else {
+                ++jt;
+            }
+        }
+    }
+
+    m_join_defs.erase(it);
+    m_caches.erase(join_table_id);
+}
+
+bool
+JoinEngine::is_join_table(const t_id& id) const {
+    return m_join_defs.contains(id);
+}
+
+std::vector<JoinEngine::t_id>
+JoinEngine::get_dependent_join_tables(const t_id& source_table_id) const {
+    std::vector<t_id> result;
+    auto range = m_table_to_join_tables.equal_range(source_table_id);
+    for (auto it = range.first; it != range.second; ++it) {
+        result.push_back(it->second);
+    }
+
+    return result;
+}
+
+const JoinDef&
+JoinEngine::get_join_def(const t_id& join_table_id) const {
+    return m_join_defs.at(join_table_id);
+}
+
+MakeJoinResult
+JoinEngine::make_join_table(
+    const std::string& on_column,
+    const std::string& right_on_column,
+    proto::JoinType join_type,
+    const std::shared_ptr<Table>& left_table,
+    const std::shared_ptr<Table>& right_table
+) {
+    auto effective_right_on = right_on_column.empty() ? on_column : right_on_column;
+    auto left_schema = left_table->get_schema();
+    auto right_schema = right_table->get_schema();
+    if (!left_schema.has_column(on_column)) {
+        std::stringstream ss;
+        ss << "Column \"" << on_column << "\" not found in left table";
+        return {nullptr, ss.str()};
+    }
+
+    if (!right_schema.has_column(effective_right_on)) {
+        std::stringstream ss;
+        ss << "Column \"" << effective_right_on << "\" not found in right table";
+        return {nullptr, ss.str()};
+    }
+
+    if (left_schema.get_dtype(on_column) != right_schema.get_dtype(effective_right_on)) {
+        return {nullptr, "Join column type mismatch"};
+    }
+
+    for (const auto& rcol : right_schema.columns()) {
+        if (rcol == effective_right_on) {
+            continue;
+        }
+
+        if (left_schema.has_column(rcol)) {
+            std::stringstream ss;
+            ss << "Column \"" << rcol << "\" exists in both tables";
+            return {nullptr, ss.str()};
+        }
+    }
+
+    std::vector<std::string> merged_columns;
+    std::vector<t_dtype> merged_types;
+    for (t_uindex i = 0; i < left_schema.columns().size(); ++i) {
+        merged_columns.push_back(left_schema.columns()[i]);
+        merged_types.push_back(left_schema.types()[i]);
+    }
+
+    for (t_uindex i = 0; i < right_schema.columns().size(); ++i) {
+        if (right_schema.columns()[i] == effective_right_on) {
+            continue;
+        }
+
+        merged_columns.push_back(right_schema.columns()[i]);
+        merged_types.push_back(right_schema.types()[i]);
+    }
+
+    t_schema merged_schema(merged_columns, merged_types);
+    auto join_table = Table::from_schema("", merged_schema);
+    return {join_table, ""};
+}
+
+void
+JoinEngine::build_right_index(
+    JoinCache& cache,
+    const std::shared_ptr<Table>& right_table,
+    const std::string& on_column
+) {
+    auto right_data = right_table->get_gnode()->get_table_sptr();
+    const auto& right_pkey_map = right_table->get_gnode()->get_pkey_map();
+    auto right_key_col = right_data->get_column(on_column);
+    cache.right_entries.assign(right_pkey_map.begin(), right_pkey_map.end());
+    std::sort(
+        cache.right_entries.begin(),
+        cache.right_entries.end(),
+        [](const auto& a, const auto& b) { return a.first < b.first; }
+    );
+
+    cache.right_index.clear();
+    cache.right_index.reserve(right_pkey_map.size());
+    for (const auto& [pkey, row_idx] : cache.right_entries) {
+        auto join_key = right_key_col->get_scalar(row_idx);
+        if (!join_key.is_none()) {
+            cache.right_index[join_key].push_back(row_idx);
+        }
+    }
+
+    cache.valid = true;
+}
+
+void
+JoinEngine::recompute(
+    const t_id& join_table_id,
+    const std::shared_ptr<Table>& left_table,
+    const std::shared_ptr<Table>& right_table,
+    const std::shared_ptr<Table>& join_table,
+    bool left_changed,
+    bool right_changed
+) {
+    const auto& def = m_join_defs.at(join_table_id);
+    auto& cache = m_caches[join_table_id];
+    auto left_data = left_table->get_gnode()->get_table_sptr();
+    auto right_data = right_table->get_gnode()->get_table_sptr();
+    const auto& left_pkey_map = left_table->get_gnode()->get_pkey_map();
+    auto left_key_col = left_data->get_column(def.on_column);
+    auto right_key_col = right_data->get_column(def.right_on_column);
+
+    // Rebuild right-side index only when the right table has changed,
+    // or on the first recompute when no cache exists yet.
+    if (right_changed || !cache.valid) {
+        build_right_index(cache, right_table, def.right_on_column);
+    }
+
+    const auto& right_join_key_to_rows = cache.right_index;
+    const auto& right_entries = cache.right_entries;
+
+    // Sort left pkey entries so the join result preserves left-table
+    // insertion order.
+    std::vector<std::pair<t_tscalar, t_uindex>> left_entries(
+        left_pkey_map.begin(), left_pkey_map.end()
+    );
+
+    std::sort(
+        left_entries.begin(),
+        left_entries.end(),
+        [](const auto& a, const auto& b) { return a.first < b.first; }
+    );
+
+    const t_uindex NO_MATCH = static_cast<t_uindex>(-1);
+    std::vector<std::pair<t_uindex, t_uindex>> matched_rows;
+    matched_rows.reserve(left_entries.size());
+    tsl::hopscotch_set<t_uindex> matched_right_rows;
+    for (const auto& [pkey, row_idx] : left_entries) {
+        auto join_key = left_key_col->get_scalar(row_idx);
+        if (join_key.is_none()) {
+            if (def.join_type == proto::LEFT
+                || def.join_type == proto::OUTER) {
+                matched_rows.emplace_back(row_idx, NO_MATCH);
+            }
+
+            continue;
+        }
+
+        auto it = right_join_key_to_rows.find(join_key);
+        if (it != right_join_key_to_rows.end()) {
+            for (auto right_row_idx : it->second) {
+                matched_rows.emplace_back(row_idx, right_row_idx);
+                if (def.join_type == proto::OUTER) {
+                    matched_right_rows.insert(right_row_idx);
+                }
+            }
+        } else if (def.join_type == proto::LEFT
+                   || def.join_type == proto::OUTER) {
+            matched_rows.emplace_back(row_idx, NO_MATCH);
+        }
+    }
+
+    if (def.join_type == proto::OUTER) {
+        for (const auto& [pkey, row_idx] : right_entries) {
+            if (matched_right_rows.find(row_idx)
+                == matched_right_rows.end()) {
+                matched_rows.emplace_back(NO_MATCH, row_idx);
+            }
+        }
+    }
+
+    t_uindex num_matched = matched_rows.size();
+    auto join_schema = join_table->get_schema();
+    t_data_table joined_data(join_schema);
+    joined_data.init();
+    joined_data.extend(num_matched);
+    auto left_schema = left_table->get_schema();
+    auto right_schema = right_table->get_schema();
+    for (const auto& col_name : join_schema.columns()) {
+        auto dst_col = joined_data.get_column(col_name);
+        bool is_join_col = (col_name == def.on_column);
+        if (left_schema.has_column(col_name)) {
+            auto left_src_col = left_data->get_column(col_name);
+            copy_column_dispatch(
+                dst_col.get(), left_src_col.get(), matched_rows, num_matched, true
+            );
+
+            if (is_join_col) {
+                copy_join_key_fallback(
+                    dst_col.get(), right_key_col.get(), matched_rows, num_matched
+                );
+            }
+        } else if (right_schema.has_column(col_name)) {
+            auto src_col = right_data->get_column(col_name);
+            copy_column_dispatch(
+                dst_col.get(), src_col.get(), matched_rows, num_matched, false
+            );
+        }
+    }
+
+    joined_data.set_size(num_matched);
+    auto* pkey_col = joined_data.add_column("psp_pkey", DTYPE_INT32, true);
+    auto* okey_col = joined_data.add_column("psp_okey", DTYPE_INT32, true);
+    for (t_uindex i = 0; i < num_matched; ++i) {
+        pkey_col->set_nth<std::int32_t>(i, static_cast<std::int32_t>(i), STATUS_VALID);
+        okey_col->set_nth<std::int32_t>(i, static_cast<std::int32_t>(i), STATUS_VALID);
+    }
+
+    join_table->clear();
+    join_table->init(joined_data, num_matched, t_op::OP_INSERT, 0);
+}
+
+} // namespace perspective::server

rust/perspective-server/cpp/perspective/src/cpp/kernel_engine.cpp

@@ -0,0 +1,24 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/kernel_engine.h>
+
+namespace perspective {
+t_kernel_evaluator::t_kernel_evaluator() {}
+
+t_kernel_evaluator*
+get_evaluator() {
+    static t_kernel_evaluator* evaluator = new t_kernel_evaluator();
+    return evaluator;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/main.cpp

@@ -0,0 +1,20 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+/**
+ * Main
+ */
+int
+main(int argc, char** argv) {
+    std::cout << "Perspective initialized successfully"
+              << "\n";
+}

rust/perspective-server/cpp/perspective/src/cpp/mask.cpp

@@ -0,0 +1,151 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/mask.h>
+#include <perspective/raii.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_mask::t_mask() { LOG_CONSTRUCTOR("t_mask"); }
+
+t_mask::t_mask(t_uindex size) : m_bitmap(t_msize(size)) {
+    LOG_CONSTRUCTOR("t_mask");
+}
+
+t_mask::t_mask(const t_simple_bitmask& m) {
+    m_bitmap = boost::dynamic_bitset<>(static_cast<size_t>(m.size()));
+
+    for (t_uindex idx = 0, loop_end = m.size(); idx < loop_end; ++idx) {
+        set(idx, m.is_set(idx));
+    }
+}
+
+t_mask::~t_mask() { LOG_DESTRUCTOR("t_mask"); }
+
+void
+t_mask::clear() {
+    m_bitmap.clear();
+}
+
+t_uindex
+t_mask::count() const {
+    return m_bitmap.count();
+}
+
+t_uindex
+t_mask::size() const {
+    return m_bitmap.size();
+}
+
+bool
+t_mask::get(t_uindex idx) const {
+    return m_bitmap.test(t_msize(idx));
+}
+
+void
+t_mask::set(t_uindex idx, bool v) {
+    m_bitmap.set(t_msize(idx), v);
+}
+
+void
+t_mask::set(t_uindex idx) {
+    m_bitmap.set(t_msize(idx), true);
+}
+
+t_mask&
+t_mask::operator&=(const t_mask& b) {
+    m_bitmap &= b.m_bitmap;
+    return *this;
+}
+
+t_mask&
+t_mask::operator|=(const t_mask& b) {
+    m_bitmap |= b.m_bitmap;
+    return *this;
+}
+
+t_mask&
+t_mask::operator^=(const t_mask& b) {
+    m_bitmap ^= b.m_bitmap;
+    return *this;
+}
+
+t_mask&
+t_mask::operator-=(const t_mask& b) {
+    m_bitmap -= b.m_bitmap;
+    return *this;
+}
+
+t_uindex
+t_mask::find_first() const {
+    return m_bitmap.find_first();
+}
+
+t_uindex
+t_mask::find_next(t_uindex pos) const {
+    return m_bitmap.find_next(t_msize(pos));
+}
+
+void
+t_mask::pprint() const {
+    std::cout << *this << '\n';
+}
+
+t_uindex
+t_mask_iterator::next() {
+    t_uindex rval = m_pos;
+    m_pos = m_mask->find_next(rval);
+    return rval;
+}
+
+bool
+t_mask_iterator::has_next() const {
+    return m_pos != m_end;
+}
+
+t_mask_iterator::t_mask_iterator(t_maskcsptr m) :
+    m_mask(std::move(std::move(m))),
+    m_pos(m_mask->find_first()) {
+    LOG_CONSTRUCTOR("t_mask_iterator");
+}
+
+t_mask_iterator::t_mask_iterator() { LOG_CONSTRUCTOR("t_mask_iterator"); }
+
+t_uindex
+t_mask_iterator::size() const {
+    return m_mask->size();
+}
+
+t_uindex
+t_mask_iterator::count() const {
+    return m_mask->count();
+}
+
+t_mask_iterator::~t_mask_iterator() { LOG_DESTRUCTOR("t_mask_iterator"); }
+
+} // end namespace perspective
+
+namespace std {
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_mask& mask) {
+    std::cout << "t_mask<\n";
+    for (perspective::t_uindex idx = 0, loop_end = mask.size(); idx < loop_end;
+         ++idx) {
+        std::cout << "\t" << idx << ". " << mask.get(idx) << '\n';
+    }
+    std::cout << ">\n";
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/multi_sort.cpp

@@ -0,0 +1,204 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <boost/math/special_functions/fpclassify.hpp>
+#include <perspective/base.h>
+#include <perspective/multi_sort.h>
+#include <perspective/scalar.h>
+#include <utility>
+#include <vector>
+
+namespace perspective {
+
+t_mselem::t_mselem() :
+    m_pkey(mknone()),
+    m_order(0),
+    m_deleted(false),
+    m_updated(false) {}
+
+t_mselem::t_mselem(const std::vector<t_tscalar>& row) :
+    m_row(row),
+    m_pkey(mknone()),
+    m_order(0),
+    m_deleted(false),
+    m_updated(false) {}
+
+t_mselem::t_mselem(const std::vector<t_tscalar>& row, t_uindex order) :
+    m_row(row),
+    m_pkey(mknone()),
+    m_order(order),
+    m_deleted(false),
+    m_updated(false) {}
+
+t_mselem::t_mselem(const t_tscalar& pkey, const std::vector<t_tscalar>& row) :
+    m_row(row),
+    m_pkey(pkey),
+    m_order(0),
+    m_deleted(false),
+    m_updated(false) {}
+
+t_minmax_idx::t_minmax_idx(t_index mn, t_index mx) : m_min(mn), m_max(mx) {}
+
+// Given a vector return the indices of the
+// minimum and maximum elements in it.
+t_minmax_idx
+get_minmax_idx(const std::vector<t_tscalar>& vec, t_sorttype stype) {
+    t_minmax_idx rval(-1, -1);
+
+    if (vec.empty()) {
+        return rval;
+    }
+
+    // min, max
+    std::pair<t_tscalar, t_tscalar> min_max;
+    min_max.first = *(vec.begin());
+    min_max.second = *(vec.begin());
+
+    switch (stype) {
+        case SORTTYPE_DESCENDING:
+        case SORTTYPE_ASCENDING: {
+            for (t_index idx = 0, loop_end = vec.size(); idx < loop_end;
+                 ++idx) {
+                const t_tscalar& val = vec[idx];
+                if (val <= min_max.first) {
+                    min_max.first = val;
+                    rval.m_min = idx;
+                }
+
+                if (val >= min_max.second) {
+                    min_max.second = val;
+                    rval.m_max = idx;
+                }
+            }
+            return rval;
+        } break;
+        case SORTTYPE_ASCENDING_ABS:
+        case SORTTYPE_DESCENDING_ABS: {
+            double mindbl = std::abs(vec[0].to_double());
+            double maxdbl = mindbl;
+            rval.m_min = 0;
+            rval.m_max = 0;
+            for (t_index idx = 1, loop_end = vec.size(); idx < loop_end;
+                 ++idx) {
+                double val = std::abs(vec[idx].to_double());
+
+                if (val <= mindbl) {
+                    mindbl = val;
+                    rval.m_min = idx;
+                }
+
+                if (val >= maxdbl) {
+                    maxdbl = val;
+                    rval.m_max = idx;
+                }
+            }
+            return rval;
+        } break;
+        case SORTTYPE_NONE: {
+            rval.m_min = 0;
+            rval.m_max = 0;
+            return rval;
+        } break;
+        default: {
+            return rval;
+        }
+    }
+
+    return rval;
+}
+
+double
+to_double(const t_tscalar& c) {
+    return c.to_double();
+}
+
+t_nancmp::t_nancmp() : m_active(false), m_cmpval(CMP_OP_EQ) {}
+
+t_nancmp
+nan_compare(t_sorttype order, const t_tscalar& a, const t_tscalar& b) {
+    t_nancmp rval;
+    bool a_fp = a.is_floating_point();
+    bool b_fp = b.is_floating_point();
+
+    if (!a_fp && !b_fp) {
+        return rval;
+    }
+
+    double a_dbl = a.to_double();
+    double b_dbl = b.to_double();
+
+    bool a_nan = std::isnan(a_dbl);
+    bool b_nan = std::isnan(b_dbl);
+
+    rval.m_active = a_nan || b_nan;
+
+    if (!rval.m_active) {
+        return rval;
+    }
+
+    if (a_nan && b_nan) {
+        rval.m_cmpval = CMP_OP_EQ;
+        return rval;
+    }
+
+    if (a_nan) {
+        switch (order) {
+            case SORTTYPE_NONE:
+            case SORTTYPE_ASCENDING:
+            case SORTTYPE_ASCENDING_ABS: {
+                rval.m_cmpval = CMP_OP_LT;
+            } break;
+            case SORTTYPE_DESCENDING_ABS:
+            case SORTTYPE_DESCENDING: {
+                rval.m_cmpval = CMP_OP_GT;
+            } break;
+        }
+        return rval;
+    }
+
+    switch (order) {
+        case SORTTYPE_NONE:
+        case SORTTYPE_ASCENDING:
+        case SORTTYPE_ASCENDING_ABS: {
+            rval.m_cmpval = CMP_OP_GT;
+        } break;
+        case SORTTYPE_DESCENDING_ABS:
+        case SORTTYPE_DESCENDING: {
+            rval.m_cmpval = CMP_OP_LT;
+        } break;
+    }
+
+    return rval;
+}
+
+t_multisorter::t_multisorter(const std::vector<t_sorttype>& order) :
+    m_sort_order(order) {}
+
+t_multisorter::t_multisorter(
+    std::shared_ptr<const std::vector<t_mselem>> elems,
+    const std::vector<t_sorttype>& order
+) :
+    m_sort_order(order),
+    m_elems(std::move(elems)) {}
+
+bool
+t_multisorter::operator()(const t_mselem& a, const t_mselem& b) const {
+    return cmp_mselem(a, b, m_sort_order);
+}
+
+bool
+t_multisorter::operator()(t_index a, t_index b) const {
+    return this->operator()((*m_elems)[a], (*m_elems)[b]);
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/none.cpp

@@ -0,0 +1,59 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/none.h>
+
+#include <boost/functional/hash.hpp>
+namespace perspective {
+
+t_none::t_none() = default;
+
+bool
+t_none::operator==(const t_none& /*unused*/) const {
+    return true;
+}
+
+bool
+t_none::operator<(const t_none& /*unused*/) const {
+    return false;
+}
+
+bool
+t_none::operator<=(const t_none& /*unused*/) const {
+    return true;
+}
+
+bool
+t_none::operator>(const t_none& /*unused*/) const {
+    return true;
+}
+
+bool
+t_none::operator>=(const t_none& /*unused*/) const {
+    return true;
+}
+
+size_t
+hash_value(const t_none& none) {
+    boost::hash<long> hasher;
+    return hasher(static_cast<long>(-1));
+}
+} // namespace perspective
+
+namespace std {
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_none& dt) {
+    os << "<t_none>";
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/path.cpp

@@ -0,0 +1,31 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/path.h>
+
+namespace perspective {
+t_path::t_path() = default;
+
+t_path::t_path(const std::vector<t_tscalar>& path) : m_path(path) {}
+
+const std::vector<t_tscalar>&
+t_path::path() const {
+    return m_path;
+}
+
+std::vector<t_tscalar>&
+t_path::path() {
+    return m_path;
+}
+
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/pivot.cpp

@@ -0,0 +1,59 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/pivot.h>
+#include <sstream>
+
+namespace perspective {
+
+t_pivot::t_pivot(const t_pivot_recipe& r) {
+    m_colname = r.m_colname;
+    m_name = r.m_name;
+    m_mode = r.m_mode;
+}
+
+t_pivot::t_pivot(const std::string& colname) :
+    m_colname(colname),
+    m_name(colname),
+    m_mode(PIVOT_MODE_NORMAL) {}
+
+t_pivot::t_pivot(const std::string& colname, t_pivot_mode mode) :
+    m_colname(colname),
+    m_name(colname),
+    m_mode(mode) {}
+
+const std::string&
+t_pivot::name() const {
+    return m_name;
+}
+
+const std::string&
+t_pivot::colname() const {
+    return m_colname;
+}
+
+t_pivot_mode
+t_pivot::mode() const {
+    return m_mode;
+}
+
+t_pivot_recipe
+t_pivot::get_recipe() const {
+    t_pivot_recipe rv;
+    rv.m_colname = m_colname;
+    rv.m_name = m_name;
+    rv.m_mode = m_mode;
+    return rv;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/pool.cpp

@@ -0,0 +1,360 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <cstdint>
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/pool.h>
+#include <perspective/update_task.h>
+#include <perspective/compat.h>
+#include <perspective/env_vars.h>
+#include <perspective/pyutils.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_updctx::t_updctx() = default;
+
+t_updctx::t_updctx(t_uindex gnode_id, std::string ctx) :
+    m_gnode_id(gnode_id),
+    m_ctx(std::move(ctx)) {}
+
+#if defined PSP_ENABLE_WASM && !defined PSP_ENABLE_PYTHON
+
+t_pool::t_pool() : m_sleep(0) { m_run.clear(); }
+
+#elif defined PSP_ENABLE_PYTHON
+
+// t_val
+// empty_callback() {
+//     return py::none();
+// }
+
+t_pool::t_pool() :
+// : m_update_delegate(empty_callback())
+#ifdef PSP_PARALLEL_FOR
+    m_lock(new std::shared_mutex()),
+#endif
+    m_sleep(0) {
+    m_run.clear();
+}
+
+#else
+
+t_pool::t_pool() : m_sleep(0) { m_run.clear(); }
+
+#endif
+
+t_pool::~t_pool() {
+#ifdef PSP_PARALLEL_FOR
+    delete m_lock;
+#endif
+}
+
+void
+t_pool::init() {
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.init " << std::endl;
+    }
+    m_run.test_and_set(std::memory_order_acquire);
+    m_data_remaining.store(false);
+}
+
+t_uindex
+t_pool::register_gnode(t_gnode* node) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_WRITE_LOCK(*m_lock)
+#endif
+
+    m_gnodes.push_back(node);
+    t_uindex id = m_gnodes.size() - 1;
+    node->set_id(id);
+    node->set_pool_cleanup([this, id]() { this->m_gnodes[id] = nullptr; });
+#ifdef PSP_PARALLEL_FOR
+    node->set_lock(m_lock);
+#endif
+
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.register_gnode node => " << node << " rv => " << id
+                  << std::endl;
+    }
+
+    return id;
+}
+
+void
+t_pool::unregister_gnode(t_uindex idx) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_WRITE_LOCK(*m_lock)
+#endif
+
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.unregister_gnode idx => " << idx << std::endl;
+    }
+
+    m_gnodes[idx] = nullptr;
+}
+
+void
+t_pool::send(t_uindex gnode_id, t_uindex port_id, const t_data_table& table) {
+    {
+#ifdef PSP_PARALLEL_FOR
+        PSP_WRITE_LOCK(*m_lock);
+#endif
+        // std::lock_guard<std::mutex> lg(m_mtx);
+        m_data_remaining.store(true);
+
+        if (m_gnodes[gnode_id] != nullptr) {
+            m_gnodes[gnode_id]->send(port_id, table);
+        }
+
+        if (t_env::log_progress()) {
+            std::cout << "t_pool.send gnode_id => " << gnode_id
+                      << " port_id => " << port_id << " tbl_size => "
+                      << table.size() << std::endl;
+        }
+
+        if (t_env::log_data_pool_send()) {
+            std::cout << "t_pool.send" << std::endl;
+            table.pprint();
+        }
+    }
+}
+
+#ifdef PSP_PARALLEL_FOR
+std::shared_mutex*
+t_pool::get_lock() const {
+    return m_lock;
+}
+#endif
+
+void
+t_pool::_process(std::optional<std::function<void(std::uint32_t)>> callback) {
+    auto work_to_do = m_data_remaining.load();
+    if (work_to_do) {
+        t_update_task task(*this);
+        task.run(callback);
+    }
+}
+
+void
+t_pool::stop() {
+    m_run.clear(std::memory_order_release);
+    _process();
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.stop" << std::endl;
+    }
+}
+
+void
+t_pool::set_sleep(t_uindex ms) {
+    m_sleep.store(ms);
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.set_sleep ms => " << ms << std::endl;
+    }
+}
+
+std::vector<t_stree*>
+t_pool::get_trees() {
+    std::vector<t_stree*> rval;
+    for (auto& g : m_gnodes) {
+        if (g == nullptr) {
+            continue;
+        }
+        auto trees = g->get_trees();
+        rval.insert(std::end(rval), std::begin(trees), std::end(trees));
+    }
+
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.get_trees: "
+                  << " rv => " << rval << std::endl;
+    }
+
+    return rval;
+}
+
+void
+t_pool::register_context(
+    t_uindex gnode_id,
+    const std::string& name,
+    t_ctx_type type,
+    std::uintptr_t ptr
+) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_WRITE_LOCK(*m_lock)
+#endif
+    if (!validate_gnode_id(gnode_id)) {
+        return;
+    }
+    m_gnodes[gnode_id]->_register_context(name, type, ptr);
+}
+
+void
+t_pool::notify_userspace(t_uindex port_id) {
+    // #if defined PSP_ENABLE_WASM && !defined PSP_ENABLE_PYTHON
+    //     m_update_delegate.call<void>("_update_callback", port_id);
+    // #elif PSP_ENABLE_PYTHON
+    //     if (!m_update_delegate.is_none()) {
+    //         m_update_delegate.attr("_update_callback")(port_id);
+    //     }
+    // #endif
+}
+
+void
+t_pool::unregister_context(t_uindex gnode_id, const std::string& name) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_WRITE_LOCK(*m_lock)
+#endif
+
+    if (t_env::log_progress()) {
+        std::cout << repr() << " << t_pool.unregister_context: "
+                  << " gnode_id => " << gnode_id << " name => " << name
+                  << std::endl;
+    }
+
+    if (!validate_gnode_id(gnode_id)) {
+        return;
+    }
+    m_gnodes[gnode_id]->_unregister_context(name);
+}
+
+bool
+t_pool::get_data_remaining() const {
+    auto data = m_data_remaining.load();
+    return data;
+}
+
+std::vector<t_tscalar>
+t_pool::get_row_data_pkeys(
+    t_uindex gnode_id, const std::vector<t_tscalar>& pkeys
+) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_READ_LOCK(*m_lock)
+#endif
+
+    if (!validate_gnode_id(gnode_id)) {
+        return {};
+    }
+
+    auto rv = m_gnodes[gnode_id]->get_row_data_pkeys(pkeys);
+
+    if (t_env::log_progress()) {
+        std::cout << "t_pool.get_row_data_pkeys: "
+                  << " gnode_id => " << gnode_id << " pkeys => " << pkeys
+                  << " rv => " << rv << std::endl;
+    }
+
+    return rv;
+}
+
+std::vector<t_updctx>
+t_pool::get_contexts_last_updated() {
+#ifdef PSP_PARALLEL_FOR
+    PSP_READ_LOCK(*m_lock)
+#endif
+
+    std::vector<t_updctx> rval;
+
+    for (auto& m_gnode : m_gnodes) {
+        if (m_gnode == nullptr) {
+            continue;
+        }
+
+        auto updated_contexts = m_gnode->get_contexts_last_updated();
+        auto gnode_id = m_gnode->get_id();
+
+        for (const auto& ctx_name : updated_contexts) {
+            if (t_env::log_progress()) {
+                std::cout << "t_pool.get_contexts_last_updated: "
+                          << " gnode_id => " << gnode_id << " ctx_name => "
+                          << ctx_name << std::endl;
+            }
+            rval.emplace_back(gnode_id, ctx_name);
+        }
+    }
+    return rval;
+}
+
+bool
+t_pool::validate_gnode_id(t_uindex gnode_id) const {
+    return (m_gnodes[gnode_id] != nullptr) && gnode_id < m_gnodes.size();
+}
+
+std::string
+t_pool::repr() const {
+    std::stringstream ss;
+    ss << "t_pool<" << this << ">";
+    return ss.str();
+}
+
+void
+t_pool::pprint_registered() const {
+    auto self = repr();
+
+    for (auto* m_gnode : m_gnodes) {
+        if (m_gnode == nullptr) {
+            continue;
+        }
+        auto gnode_id = m_gnode->get_id();
+        auto ctxnames = m_gnode->get_registered_contexts();
+
+        for (const auto& cname : ctxnames) {
+            std::cout << self << " gnode_id => " << gnode_id << " ctxname => "
+                      << cname << std::endl;
+        }
+    }
+}
+
+t_uindex
+t_pool::epoch() const {
+    return m_epoch.load();
+}
+
+void
+t_pool::inc_epoch() {
+    ++m_epoch;
+}
+
+std::vector<t_uindex>
+t_pool::get_gnodes_last_updated() {
+#ifdef PSP_PARALLEL_FOR
+    PSP_READ_LOCK(*m_lock)
+#endif
+
+    std::vector<t_uindex> rv;
+
+    for (t_uindex idx = 0, loop_end = m_gnodes.size(); idx < loop_end; ++idx) {
+        if ((m_gnodes[idx] == nullptr) || !m_gnodes[idx]->was_updated()) {
+            continue;
+        }
+
+        rv.push_back(idx);
+        m_gnodes[idx]->clear_updated();
+    }
+    return rv;
+}
+
+t_gnode*
+t_pool::get_gnode(t_uindex idx) {
+#ifdef PSP_PARALLEL_FOR
+    PSP_READ_LOCK(*m_lock)
+#endif
+
+    PSP_VERBOSE_ASSERT(
+        idx < m_gnodes.size() && m_gnodes[idx], "Bad gnode encountered"
+    );
+
+    return m_gnodes[idx];
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/port.cpp

@@ -0,0 +1,113 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/port.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_port::t_port(t_port_mode mode, t_schema schema) :
+    m_schema(std::move(schema)),
+    m_init(false),
+    m_table(nullptr),
+    m_prevsize(0) {
+    LOG_CONSTRUCTOR("t_port");
+}
+
+t_port::~t_port() { LOG_DESTRUCTOR("t_port"); }
+
+void
+t_port::init() {
+    m_table = nullptr;
+    m_table = std::make_shared<t_data_table>(
+        "", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_table->init();
+    m_init = true;
+}
+
+std::shared_ptr<t_data_table>
+t_port::get_table() {
+    return m_table;
+}
+
+void
+t_port::set_table(std::shared_ptr<t_data_table> table) {
+    m_table = nullptr;
+    m_table = std::move(table);
+}
+
+void
+t_port::send(const std::shared_ptr<const t_data_table>& table) {
+    m_table->append(*table);
+}
+
+void
+t_port::send(const t_data_table& table) {
+    m_table->append(table);
+}
+
+t_schema
+t_port::get_schema() const {
+    return m_schema;
+}
+
+void
+t_port::release()
+
+{
+    if (m_table == nullptr) {
+        return;
+    }
+
+    t_uindex size = m_table->size();
+
+    m_table = nullptr;
+    m_table = std::make_shared<t_data_table>(
+        "", "", m_schema, DEFAULT_EMPTY_CAPACITY, BACKING_STORE_MEMORY
+    );
+    m_table->init();
+
+    m_prevsize = size;
+}
+
+void
+t_port::release_or_clear()
+
+{
+    if (m_table == nullptr) {
+        return;
+    }
+
+    t_uindex size = m_table->size();
+
+    if (static_cast<double>(size) < 0.4 * double(m_prevsize)) {
+        m_table->clear();
+    } else {
+        release();
+    }
+
+    m_prevsize = size;
+}
+
+void
+t_port::clear() {
+    if (m_table == nullptr) {
+        return;
+    }
+
+    m_table->clear();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/process_state.cpp

@@ -0,0 +1,46 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/process_state.h>
+
+namespace perspective {
+
+t_process_state::t_process_state() = default;
+
+void
+t_process_state::clear_transitional_data_tables() const {
+    m_delta_data_table->clear();
+    m_prev_data_table->clear();
+    m_current_data_table->clear();
+    m_transitions_data_table->clear();
+    m_existed_data_table->clear();
+};
+
+void
+t_process_state::reserve_transitional_data_tables(t_uindex size) const {
+    m_delta_data_table->reserve(size);
+    m_prev_data_table->reserve(size);
+    m_current_data_table->reserve(size);
+    m_transitions_data_table->reserve(size);
+    m_existed_data_table->reserve(size);
+};
+
+void
+t_process_state::set_size_transitional_data_tables(t_uindex size) const {
+    m_delta_data_table->set_size(size);
+    m_prev_data_table->set_size(size);
+    m_current_data_table->set_size(size);
+    m_transitions_data_table->set_size(size);
+    m_existed_data_table->set_size(size);
+};
+
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/pyutils.cpp

@@ -0,0 +1,28 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/pyutils.h>
+
+namespace perspective {
+
+#ifdef PSP_ENABLE_PYTHON
+PerspectiveGILUnlock::PerspectiveGILUnlock() {}
+// m_thread_state(PyEval_SaveThread()) {}
+
+PerspectiveGILUnlock::~PerspectiveGILUnlock() {
+    // PyEval_RestoreThread(m_thread_state);
+}
+#endif
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/raii.cpp

@@ -0,0 +1,47 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raii.h>
+#ifndef WIN32
+#ifdef PSP_DEBUG
+#include <sys/mman.h>
+#endif
+#endif
+
+namespace perspective {
+
+#ifndef PSP_ENABLE_WASM
+t_file_handle::t_file_handle(t_handle value) : m_value(value) {}
+
+t_handle
+t_file_handle::value() const {
+    return m_value;
+}
+#endif
+
+t_mmap_handle::t_mmap_handle(void* value, t_uindex len) :
+    m_value(value),
+    m_len(len) {}
+
+void*
+t_mmap_handle::value() {
+    return m_value;
+}
+
+t_uindex
+t_mmap_handle::len() const {
+    return m_len;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/raii_impl_linux.cpp

@@ -0,0 +1,52 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __linux__
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raii.h>
+#include <sys/mman.h>
+
+namespace perspective {
+
+t_file_handle::~t_file_handle() {
+    if (valid()) {
+        t_index rcode = close(m_value);
+        PSP_VERBOSE_ASSERT(rcode == 0, "Error closing file.");
+    }
+}
+
+bool
+t_file_handle::valid() const {
+    return m_value >= 0;
+}
+
+void
+t_file_handle::release() {
+    m_value = -1;
+}
+
+t_mmap_handle::~t_mmap_handle() {
+    if (valid()) {
+        t_index rcode = munmap(m_value, m_len);
+        PSP_VERBOSE_ASSERT(rcode == 0, "munmap failed.");
+    }
+}
+
+bool
+t_mmap_handle::valid() {
+    return m_value != MAP_FAILED;
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/raii_impl_osx.cpp

@@ -0,0 +1,52 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __APPLE__
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raii.h>
+#include <sys/mman.h>
+
+namespace perspective {
+
+t_file_handle::~t_file_handle() {
+    if (valid()) {
+        t_index rcode = close(m_value);
+        PSP_VERBOSE_ASSERT(rcode, == 0, "Error closing file.");
+    }
+}
+
+bool
+t_file_handle::valid() const {
+    return m_value >= 0;
+}
+
+void
+t_file_handle::release() {
+    m_value = -1;
+}
+
+t_mmap_handle::~t_mmap_handle() {
+    if (valid()) {
+        t_index rcode = munmap(m_value, m_len);
+        PSP_VERBOSE_ASSERT(rcode, == 0, "munmap failed.");
+    }
+}
+
+bool
+t_mmap_handle::valid() {
+    return m_value != MAP_FAILED;
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/raii_impl_win.cpp

@@ -0,0 +1,51 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef WIN32
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raii.h>
+
+namespace perspective {
+
+t_file_handle::~t_file_handle() {
+    if (valid()) {
+        auto rb = CloseHandle(m_value);
+        PSP_VERBOSE_ASSERT(rb, "Error closing file");
+    }
+}
+
+bool
+t_file_handle::valid() const {
+    return m_value != INVALID_HANDLE_VALUE;
+}
+
+void
+t_file_handle::release() {
+    m_value = INVALID_HANDLE_VALUE;
+}
+
+t_mmap_handle::~t_mmap_handle() {
+    if (valid()) {
+        auto rc = UnmapViewOfFile(m_value);
+        PSP_VERBOSE_ASSERT(rc, "Error unmapping view");
+    }
+}
+
+bool
+t_mmap_handle::valid() {
+    return m_value != 0;
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/range.cpp

@@ -0,0 +1,75 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/range.h>
+
+namespace perspective {
+
+t_range::t_range(t_uindex bridx, t_uindex eridx) :
+    m_bridx(bridx),
+    m_eridx(eridx),
+    m_mode(RANGE_ROW) {}
+
+t_range::t_range(
+    t_uindex bridx, t_uindex eridx, t_uindex bcidx, t_uindex ecidx
+) :
+    m_bridx(bridx),
+    m_eridx(eridx),
+    m_bcidx(bcidx),
+    m_ecidx(ecidx),
+    m_mode(RANGE_ROW_COLUMN) {}
+
+t_range::t_range() : m_mode(RANGE_ALL) {}
+
+t_range::t_range(
+    const std::vector<t_tscalar>& brpath, const std::vector<t_tscalar>& erpath
+) :
+    m_brpath(brpath),
+    m_erpath(erpath),
+    m_mode(RANGE_ROW_PATH) {}
+
+t_range::t_range(
+    const std::vector<t_tscalar>& brpath,
+    const std::vector<t_tscalar>& erpath,
+    const std::vector<t_tscalar>& bcpath,
+    const std::vector<t_tscalar>& ecpath
+) {}
+
+t_range::t_range(const std::string& expr_name) {}
+
+t_uindex
+t_range::bridx() const {
+    return m_bridx;
+}
+
+t_uindex
+t_range::eridx() const {
+    return m_eridx;
+}
+
+t_uindex
+t_range::bcidx() const {
+    return m_bcidx;
+}
+
+t_uindex
+t_range::ecidx() const {
+    return m_ecidx;
+}
+
+t_range_mode
+t_range::get_mode() const {
+    return m_mode;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/regex.cpp

@@ -0,0 +1,40 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/regex.h>
+
+namespace perspective {
+
+RE2*
+t_regex_mapping::intern(const std::string& pattern) {
+    if (m_regex_map.count(pattern) == 1) {
+        return m_regex_map[pattern].get();
+    }
+
+    std::shared_ptr<RE2> compiled_pattern =
+        std::make_shared<RE2>(pattern, RE2::Quiet);
+
+    if (!compiled_pattern->ok()) {
+        // TODO: provide a better error message when the regex can't compile.
+        return nullptr;
+    }
+
+    m_regex_map[pattern] = compiled_pattern;
+    return m_regex_map[pattern].get();
+}
+
+void
+t_regex_mapping::clear() {
+    m_regex_map.clear();
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/residency.cpp

@@ -0,0 +1,176 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/residency.h>
+#include <perspective/storage.h>
+#include <algorithm>
+#include <cstdio>
+#include <cstdlib>
+#include <vector>
+
+namespace perspective {
+
+bool g_residency_active = false;
+std::uint64_t g_residency_tick = 0;
+
+t_residency_manager&
+t_residency_manager::inst() {
+    static t_residency_manager s_inst;
+    return s_inst;
+}
+
+void
+t_residency_manager::register_store(t_lstore* store) {
+    std::lock_guard<std::mutex> lk(m_mutex);
+    m_stores.insert(store);
+}
+
+void
+t_residency_manager::unregister_store(t_lstore* store) {
+    std::lock_guard<std::mutex> lk(m_mutex);
+    m_stores.erase(store);
+}
+
+// Hard-coded residency budget (bytes) for WASM. A browser has no environment,
+// so `PSP_MEMORY_BUDGET` is unreachable there — without a budget, residency is
+// inert, on-disk columns never evict to OPFS, and the heap can grow past the
+// 2GB signed-pointer ceiling. This caps resident disk-backed column buffers so
+// the cold set is flushed to OPFS. Tunable via `-DPSP_WASM_MEMORY_BUDGET=...`.
+#ifndef PSP_WASM_MEMORY_BUDGET
+#define PSP_WASM_MEMORY_BUDGET (1024ull * 1024ull * 1024ull) // 1 GiB
+#endif
+
+void
+t_residency_manager::refresh_config() {
+    std::size_t budget = 0;
+#ifdef PSP_ENABLE_WASM
+    budget = static_cast<std::size_t>(PSP_WASM_MEMORY_BUDGET);
+#else
+    const char* budget_env = std::getenv("PSP_MEMORY_BUDGET");
+    if (budget_env != nullptr) {
+        budget = static_cast<std::size_t>(std::strtoull(budget_env, nullptr, 10));
+    }
+#endif
+
+    bool was_active = g_residency_active;
+    m_budget = budget;
+    g_residency_active = budget > 0;
+
+    // If residency was just disabled, restore every evicted store so the lazy
+    // `ensure_resident()` hook (
+    // now a no-op) never sees a null `m_base`.
+    if (was_active && !g_residency_active) {
+        for (auto* store : m_stores) {
+            store->restore();
+        }
+    }
+}
+
+std::size_t
+t_residency_manager::resident_bytes() {
+    std::lock_guard<std::mutex> lk(m_mutex);
+    std::size_t total = 0;
+    for (auto* store : m_stores) {
+        if (store->is_resident()) {
+            total += store->capacity();
+        }
+    }
+
+    return total;
+}
+
+std::size_t
+t_residency_manager::prepare() {
+    refresh_config();
+    std::lock_guard<std::mutex> lk(m_mutex);
+    m_pending.clear();
+    m_pending_fnames.clear();
+
+    if (!g_residency_active) {
+        return 0;
+    }
+
+    g_residency_tick = ++m_tick;
+    std::size_t resident = 0;
+    std::vector<t_lstore*> candidates;
+    candidates.reserve(m_stores.size());
+    for (auto* store : m_stores) {
+        if (store->is_resident()) {
+            resident += store->capacity();
+            candidates.push_back(store);
+        }
+    }
+
+    if (resident <= m_budget) {
+        return 0;
+    }
+
+    std::sort(
+        candidates.begin(),
+        candidates.end(),
+        [](const t_lstore* a, const t_lstore* b) {
+            return a->residency_tick() < b->residency_tick();
+        }
+    );
+
+    // Select victims to bring under budget, but do NOT evict yet — the JS driver
+    // must open each victim's OPFS handle before `commit()` can flush it.
+    for (auto* store : candidates) {
+        if (resident <= m_budget) {
+            break;
+        }
+
+        resident -= store->capacity();
+        m_pending.push_back(store);
+        m_pending_fnames.push_back(store->get_fname());
+    }
+
+    return m_pending.size();
+}
+
+const char*
+t_residency_manager::victim_fname(std::size_t i) const {
+    if (i >= m_pending_fnames.size()) {
+        return "";
+    }
+    return m_pending_fnames[i].c_str();
+}
+
+void
+t_residency_manager::commit() {
+    std::size_t n = 0;
+    {
+        std::lock_guard<std::mutex> lk(m_mutex);
+        for (auto* store : m_pending) {
+            store->evict();
+            ++m_evictions;
+        }
+
+        n = m_pending.size();
+        m_pending.clear();
+        m_pending_fnames.clear();
+    }
+
+    if (n == 0) {
+        return;
+    }
+}
+
+void
+t_residency_manager::safepoint() {
+    std::lock_guard<std::mutex> lk(m_safepoint_mutex);
+    prepare();
+    commit();
+}
+
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/rlookup.cpp

@@ -0,0 +1,29 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/rlookup.h>
+
+namespace perspective {
+t_rlookup::t_rlookup(t_uindex idx, bool exists) : m_idx(idx), m_exists(exists) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_rlookup");
+}
+
+t_rlookup::t_rlookup() = default;
+
+t_rlookup::~t_rlookup() {
+    PSP_TRACE_SENTINEL();
+    LOG_DESTRUCTOR("t_rlookup");
+}
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/schema.cpp

@@ -0,0 +1,214 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/schema.h>
+
+namespace perspective {
+
+t_schema::t_schema() = default;
+
+t_schema::t_schema(
+    const std::vector<std::string>& columns, const std::vector<t_dtype>& types
+) :
+    m_columns(columns),
+    m_types(types),
+    m_status_enabled(columns.size()),
+    m_pkeyidx(0),
+    m_opidx(0) {
+    PSP_VERBOSE_ASSERT(columns.size() == types.size(), "Size mismatch");
+
+    bool pkey_found = false;
+    bool op_found = false;
+
+    std::string pkey_str("psp_pkey");
+    std::string op_str("psp_op");
+    for (std::vector<std::string>::size_type idx = 0, loop_end = types.size();
+         idx < loop_end;
+         ++idx) {
+        m_colidx_map[columns[idx]] = idx;
+        m_coldt_map[columns[idx]] = types[idx];
+        m_status_enabled[idx] = true;
+        if (columns[idx] == pkey_str) {
+            pkey_found = true;
+            m_pkeyidx = idx;
+        }
+
+        if (columns[idx] == op_str) {
+            op_found = true;
+            m_opidx = idx;
+        }
+    }
+
+    m_is_pkey = pkey_found && op_found;
+}
+
+t_uindex
+t_schema::get_num_columns() const {
+    return m_columns.size();
+}
+
+t_uindex
+t_schema::size() const {
+    return m_columns.size();
+}
+
+t_uindex
+t_schema::get_colidx(const std::string& colname) const {
+    auto iter = m_colidx_map.find(colname);
+    if (iter == m_colidx_map.end()) {
+        std::stringstream ss;
+        ss << "Could not find column index for `" << colname
+           << "` as it does not exist in the schema." << '\n';
+        PSP_COMPLAIN_AND_ABORT(ss.str());
+    }
+    return iter->second;
+}
+
+t_uindex
+t_schema::get_colidx_safe(const std::string& colname) const {
+    auto iter = m_colidx_map.find(colname);
+    if (iter == m_colidx_map.end()) {
+        return -1;
+    }
+    return iter->second;
+}
+
+t_dtype
+t_schema::get_dtype(const std::string& colname) const {
+    auto iter = m_coldt_map.find(colname);
+    if (iter == m_coldt_map.end()) {
+        std::stringstream ss;
+        ss << "Could not get dtype for column `" << colname
+           << "` as it does not exist in the schema." << '\n';
+        PSP_COMPLAIN_AND_ABORT(ss.str());
+    }
+    return iter->second;
+}
+
+bool
+t_schema::is_pkey() const {
+    return m_is_pkey;
+}
+
+bool
+t_schema::operator==(const t_schema& rhs) const {
+    return m_columns == rhs.m_columns && m_types == rhs.m_types
+        && m_status_enabled == rhs.m_status_enabled;
+}
+
+void
+t_schema::add_column(const std::string& colname, t_dtype dtype) {
+    t_uindex idx = m_columns.size();
+    m_columns.push_back(colname);
+    m_status_enabled.push_back(true);
+    m_types.push_back(dtype);
+    m_colidx_map[colname] = idx;
+    m_coldt_map[colname] = dtype;
+
+    if (colname == "psp_pkey") {
+        m_pkeyidx = idx;
+        m_is_pkey = true;
+    }
+
+    if (colname == "psp_op") {
+        m_opidx = idx;
+        m_is_pkey = true;
+    }
+}
+
+void
+t_schema::retype_column(const std::string& colname, t_dtype dtype) {
+    if (colname == "psp_pkey"
+        || colname == "psp_op") {
+        PSP_COMPLAIN_AND_ABORT("Cannot retype primary key or operation columns."
+        );
+    }
+    if (!has_column(colname)) {
+        std::stringstream ss;
+        ss << "Cannot retype column `" << colname << "` as it does not exist."
+           << '\n';
+        PSP_COMPLAIN_AND_ABORT(ss.str());
+    }
+
+    t_uindex idx = get_colidx(colname);
+    m_types[idx] = dtype;
+    m_colidx_map[colname] = idx;
+    m_coldt_map[colname] = dtype;
+}
+
+bool
+t_schema::has_column(std::string_view colname) const {
+    auto iter = m_colidx_map.find(colname.data());
+    return iter != m_colidx_map.end();
+}
+
+const std::vector<std::string>&
+t_schema::columns() const {
+    return m_columns;
+}
+
+std::vector<t_dtype>
+t_schema::types() const {
+    return m_types;
+}
+
+std::string
+t_schema::str() const {
+    std::stringstream ss;
+    ss << *this;
+    return ss.str();
+}
+
+t_schema
+t_schema::drop(const std::set<std::string>& columns) const {
+    std::vector<std::string> cols;
+    std::vector<t_dtype> types;
+
+    for (t_uindex idx = 0, loop_end = m_columns.size(); idx < loop_end; ++idx) {
+        if (columns.find(m_columns[idx]) == columns.end()) {
+            cols.push_back(m_columns[idx]);
+            types.push_back(m_types[idx]);
+        }
+    }
+    return t_schema(cols, types);
+}
+
+t_schema
+t_schema::operator+(const t_schema& o) const {
+    t_schema rv(m_columns, m_types);
+    for (t_uindex idx = 0, loop_end = o.m_columns.size(); idx < loop_end;
+         ++idx) {
+        rv.add_column(o.m_columns[idx], o.m_types[idx]);
+    }
+    return rv;
+}
+
+} // end namespace perspective
+
+namespace std {
+
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_schema& s) {
+    using namespace perspective;
+    const std::vector<std::string>& cols = s.columns();
+    const std::vector<t_dtype>& types = s.types();
+
+    os << "t_schema<\n";
+    for (size_t idx = 0, loop_end = cols.size(); idx < loop_end; ++idx) {
+        os << "\t" << idx << ". " << cols[idx] << ", "
+           << get_dtype_descr(types[idx]) << '\n';
+    }
+    os << ">\n";
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/schema_column.cpp

@@ -0,0 +1,26 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/schema_column.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_schema_column::t_schema_column(
+    std::string tblname, std::string name, std::string altname, t_dtype dtype
+) :
+    m_tblname(std::move(tblname)),
+    m_name(std::move(name)),
+    m_altname(std::move(altname)) {}
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/simple_bitmask.cpp

@@ -0,0 +1,20 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/simple_bitmask.h>
+
+namespace perspective {
+
+t_simple_bitmask::t_simple_bitmask(t_uindex idx)
+}

rust/perspective-server/cpp/perspective/src/cpp/slice.cpp

@@ -0,0 +1,132 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/slice.h>
+
+namespace perspective {
+
+const t_range&
+t_slice::range() const {
+    return m_range;
+}
+
+const std::vector<t_path>&
+t_slice::row_paths() const {
+    return m_row_paths;
+}
+
+const std::vector<t_path>&
+t_slice::column_paths() const {
+    return m_column_paths;
+}
+
+const std::vector<t_index>&
+t_slice::row_indices() const {
+    return m_row_indices;
+}
+
+const std::vector<t_index>&
+t_slice::column_indices() const {
+    return m_column_indices;
+}
+
+const std::vector<t_data>&
+t_slice::row_data() const {
+    return m_row_data;
+}
+
+const std::vector<t_data>&
+t_slice::column_data() const {
+    return m_column_data;
+}
+
+const std::vector<t_uindex>&
+t_slice::row_depth() const {
+
+    return m_row_depth;
+}
+
+const std::vector<t_uindex>&
+t_slice::column_depth() const {
+    return m_column_depth;
+}
+
+const std::vector<t_uindex>&
+t_slice::is_row_expanded() const {
+    return m_is_row_expanded;
+}
+
+const std::vector<t_uindex>&
+t_slice::is_column_expanded() const {
+    return m_is_column_expanded;
+}
+
+t_range&
+t_slice::range() {
+    return m_range;
+}
+
+std::vector<t_path>&
+t_slice::row_paths() {
+    return m_row_paths;
+}
+
+std::vector<t_path>&
+t_slice::column_paths() {
+    return m_column_paths;
+}
+
+std::vector<t_index>&
+t_slice::row_indices() {
+    return m_row_indices;
+}
+
+std::vector<t_index>&
+t_slice::column_indices() {
+    return m_column_indices;
+}
+
+std::vector<t_data>&
+t_slice::row_data() {
+    return m_row_data;
+}
+
+std::vector<t_data>&
+t_slice::column_data() {
+    return m_column_data;
+}
+
+std::vector<t_uindex>&
+t_slice::row_depth() {
+
+    return m_row_depth;
+}
+
+std::vector<t_uindex>&
+t_slice::column_depth() {
+    return m_column_depth;
+}
+
+std::vector<t_uindex>&
+t_slice::is_row_expanded() {
+    return m_is_row_expanded;
+}
+
+std::vector<t_uindex>&
+t_slice::is_column_expanded() {
+    return m_is_column_expanded;
+}
+
+} // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/sort_specification.cpp

@@ -0,0 +1,82 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/sort_specification.h>
+
+#include <utility>
+
+namespace perspective {
+
+t_sortspec::t_sortspec(
+    std::string column_name, t_index agg_index, t_sorttype sort_type
+) :
+    m_colname(std::move(column_name)),
+    m_agg_index(agg_index),
+    m_sort_type(sort_type),
+    m_sortspec_type(SORTSPEC_TYPE_IDX) {}
+
+t_sortspec::t_sortspec(t_index agg_index, t_sorttype sort_type) :
+    m_agg_index(agg_index),
+    m_sort_type(sort_type),
+    m_sortspec_type(SORTSPEC_TYPE_IDX) {}
+
+t_sortspec::t_sortspec(
+    const std::vector<t_tscalar>& path, t_index agg_index, t_sorttype sort_type
+) :
+    m_agg_index(agg_index),
+    m_sort_type(sort_type),
+    m_sortspec_type(SORTSPEC_TYPE_PATH),
+    m_path(path) {}
+
+t_sortspec::t_sortspec() :
+    m_agg_index(INVALID_INDEX),
+    m_sort_type(SORTTYPE_NONE),
+    m_sortspec_type(SORTSPEC_TYPE_IDX) {}
+
+bool
+t_sortspec::operator==(const t_sortspec& s2) const {
+    return (m_agg_index == s2.m_agg_index) && (m_sort_type == s2.m_sort_type);
+}
+
+bool
+t_sortspec::operator!=(const t_sortspec& s2) const {
+    return !(*this == s2);
+}
+
+std::vector<t_sorttype>
+get_sort_orders(const std::vector<t_sortspec>& vec) {
+    if (vec.empty()) {
+        return {};
+    }
+    auto num = vec.size();
+    std::vector<t_sorttype> sort_orders(num);
+
+    for (std::vector<t_sortspec>::size_type idx = 0; idx < num; ++idx) {
+        sort_orders[idx] = vec[idx].m_sort_type;
+    }
+    return sort_orders;
+}
+
+} // end namespace perspective
+
+namespace std {
+PERSPECTIVE_EXPORT std::ostream&
+operator<<(std::ostream& os, const perspective::t_sortspec& t) {
+    using namespace perspective;
+    os << "t_sortspec<idx: " << t.m_agg_index << " stype: " << t.m_sort_type
+       << ">";
+    return os;
+}
+
+} // end namespace std

rust/perspective-server/cpp/perspective/src/cpp/sparse_tree_node.cpp

@@ -0,0 +1,92 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/sparse_tree_node.h>
+
+namespace perspective {
+
+t_stnode::t_stnode(
+    t_uindex idx,
+    t_uindex pidx,
+    const t_tscalar& value,
+    std::uint8_t depth,
+    const t_tscalar& sort_value,
+    t_uindex nstrands,
+    t_uindex aggidx
+) :
+    m_idx(idx),
+    m_pidx(pidx),
+    m_depth(depth),
+    m_nstrands(nstrands),
+    m_aggidx(aggidx) {
+    m_value.set(value);
+    m_sort_value.set(sort_value);
+}
+
+t_stnode::t_stnode() = default;
+
+void
+t_stnode::set_nstrands(t_index nstrands) {
+    m_nstrands = nstrands;
+}
+
+void
+t_stnode::set_sort_value(t_tscalar sv) {
+    m_sort_value.set(sv);
+}
+
+t_stpkey::t_stpkey(t_uindex idx, t_tscalar pkey) : m_idx(idx), m_pkey(pkey) {}
+
+t_stpkey::t_stpkey() = default;
+
+t_stleaves::t_stleaves(t_uindex idx, t_uindex lfidx) :
+    m_idx(idx),
+    m_lfidx(lfidx) {}
+
+t_stleaves::t_stleaves() = default;
+
+t_cellinfo::t_cellinfo() = default;
+
+t_cellinfo::t_cellinfo(
+    t_index idx,
+    t_depth treenum,
+    t_index agg_index,
+    t_uindex ridx,
+    t_uindex cidx
+) :
+    m_idx(idx),
+    m_treenum(treenum),
+    m_agg_index(agg_index),
+    m_ridx(ridx),
+    m_cidx(cidx) {}
+
+} // end namespace perspective
+
+namespace std {
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_stnode& node) {
+    os << "t_stnode<"
+       << "idx: " << node.m_idx << " pidx: " << node.m_pidx
+       << " value: " << node.m_value << " sort_value: " << node.m_sort_value
+       << " aggidx: " << node.m_aggidx << " nstrands: " << node.m_nstrands
+       << " depth: " << static_cast<perspective::t_uindex>(node.m_depth) << ">";
+    return os;
+}
+
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_cellinfo& node) {
+    os << "t_cellinfo<idx: " << node.m_idx << " treenum: " << node.m_treenum
+       << " aggidx: " << node.m_agg_index << ">";
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/step_delta.cpp

@@ -0,0 +1,85 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/step_delta.h>
+
+namespace perspective {
+
+// Deltas for various contexts
+
+t_zcdelta::t_zcdelta(
+    t_tscalar pkey, t_index colidx, t_tscalar old_value, t_tscalar new_value
+) :
+    m_pkey(pkey),
+    m_colidx(colidx),
+    m_old_value(old_value),
+    m_new_value(new_value) {}
+
+t_tcdelta::t_tcdelta(
+    t_uindex nidx, t_uindex aggidx, t_tscalar old_value, t_tscalar new_value
+) :
+    m_nidx(nidx),
+    m_aggidx(aggidx),
+    m_old_value(old_value),
+    m_new_value(new_value) {}
+
+t_cellupd::t_cellupd(
+    t_index row,
+    t_index column,
+    const t_tscalar& old_value,
+    const t_tscalar& new_value
+) :
+    row(row),
+    column(column),
+    old_value(old_value),
+    new_value(new_value) {}
+
+t_cellupd::t_cellupd() = default;
+
+// t_stepdelta contains a vector of t_cellupd objects showing the cells that
+// have been changed
+t_stepdelta::t_stepdelta() = default;
+
+t_stepdelta::t_stepdelta(
+    bool rows_changed, bool columns_changed, const std::vector<t_cellupd>& cells
+) :
+    rows_changed(rows_changed),
+    columns_changed(columns_changed),
+    cells(cells) {}
+
+// t_rowdelta contains a vector of row indices that have been changed
+t_rowdelta::t_rowdelta() = default;
+
+t_rowdelta::t_rowdelta(
+    bool rows_changed,
+    t_uindex num_rows_changed,
+    const std::vector<t_tscalar>& data
+) :
+    rows_changed(rows_changed),
+    num_rows_changed(num_rows_changed),
+    data(data) {}
+} // end namespace perspective
+
+namespace std {
+
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_cellupd& cell) {
+    os << "t_cellupd \n{"
+       << "\n\trow => " << cell.row << "\n\tcolumn => " << cell.column
+       << "\n\told_value => " << cell.old_value << "\n\tnew_value => "
+       << cell.new_value << "\n}" << '\n';
+
+    return os;
+}
+
+} // end namespace std

rust/perspective-server/cpp/perspective/src/cpp/storage.cpp

@@ -0,0 +1,726 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/portable.h>
+SUPPRESS_WARNINGS_VC(4505)
+#include <cstdlib>
+#include <cassert>
+#include <csignal>
+#include <iostream>
+#include <map>
+#include <perspective/base.h>
+#include <perspective/compat.h>
+#include <perspective/defaults.h>
+#include <perspective/raii.h>
+#include <perspective/raw_types.h>
+#include <perspective/storage.h>
+#include <perspective/opfs.h>
+#include <perspective/tracing.h>
+#include <perspective/utils.h>
+#include <perspective/env_vars.h>
+#include <sstream>
+#include <utility>
+#include <vector>
+#include <fstream>
+#ifndef WIN32
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#endif
+
+namespace perspective {
+
+t_lstore_recipe::t_lstore_recipe() : m_alignment(0), m_from_recipe(false) {}
+
+t_lstore_recipe::t_lstore_recipe(t_uindex capacity) :
+    m_capacity(capacity),
+    m_size(0),
+    m_alignment(0),
+    m_fflags(PSP_DEFAULT_FFLAGS),
+    m_fmode(PSP_DEFAULT_FMODE),
+    m_creation_disposition(PSP_DEFAULT_CREATION_DISPOSITION),
+    m_mprot(PSP_DEFAULT_MPROT),
+    m_mflags(PSP_DEFAULT_MFLAGS),
+    m_backing_store(BACKING_STORE_MEMORY),
+    m_from_recipe(false)
+
+{
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore_recipe");
+}
+
+t_lstore_recipe::t_lstore_recipe(
+    std::string dirname,
+    std::string colname,
+    t_uindex capacity,
+    t_backing_store backing_store
+) :
+    m_dirname(std::move(dirname)),
+    m_colname(std::move(colname)),
+    m_capacity(capacity),
+    m_size(0),
+    m_alignment(0),
+    m_fflags(PSP_DEFAULT_FFLAGS),
+    m_fmode(PSP_DEFAULT_FMODE),
+    m_creation_disposition(PSP_DEFAULT_CREATION_DISPOSITION),
+    m_mprot(PSP_DEFAULT_MPROT),
+    m_mflags(PSP_DEFAULT_MFLAGS),
+    m_backing_store(backing_store),
+    m_from_recipe(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore_recipe");
+}
+
+t_lstore_recipe::t_lstore_recipe(
+    std::string dirname,
+    std::string colname,
+    t_uindex capacity,
+    t_fflag fflags,
+    t_fflag fmode,
+    t_fflag creation_disposition,
+    t_fflag mprot,
+    t_fflag mflags,
+    t_backing_store backing_store
+) :
+    m_dirname(std::move(dirname)),
+    m_colname(std::move(colname)),
+    m_capacity(capacity),
+    m_size(0),
+    m_alignment(0),
+    m_fflags(fflags),
+    m_fmode(fmode),
+    m_creation_disposition(creation_disposition),
+    m_mprot(mprot),
+    m_mflags(mflags),
+    m_backing_store(backing_store),
+    m_from_recipe(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore_recipe");
+}
+
+t_lstore_recipe::t_lstore_recipe(
+    std::string colname,
+    t_uindex capacity,
+    t_fflag mprot,
+    t_fflag mflags,
+    t_backing_store backing_store
+) :
+    m_colname(std::move(colname)),
+    m_capacity(capacity),
+    m_size(0),
+    m_alignment(0),
+    m_fflags(PSP_DEFAULT_FFLAGS),
+    m_fmode(PSP_DEFAULT_FMODE),
+    m_creation_disposition(PSP_DEFAULT_CREATION_DISPOSITION),
+    m_mprot(mprot),
+    m_mflags(mflags),
+    m_backing_store(backing_store),
+    m_from_recipe(false) {
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore_recipe");
+}
+
+t_lstore::t_lstore() :
+    m_base(nullptr),
+    m_fd(0),
+    m_capacity(0),
+    m_size(0),
+    m_alignment(0),
+    m_backing_store(BACKING_STORE_MEMORY),
+    m_init(false),
+    m_resize_factor(1.2),
+    m_version(0) {
+
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore");
+#ifdef PSP_MPROTECT
+    PSP_VERBOSE_ASSERT(
+        sizeof(t_lstore) == get_page_size(), "Bad lstore sizeof"
+    );
+#endif
+}
+
+t_lstore::t_lstore(const t_lstore& s, t_lstore_tmp_init_tag t) {
+    PSP_VERBOSE_ASSERT(this != &s, "Initializing from self");
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore");
+    copy_helper(s);
+    m_version = 0;
+    m_base = nullptr;
+    m_size = 0;
+    m_alignment = 0;
+    m_fd = 0;
+    m_init = false;
+    if (s.m_backing_store == BACKING_STORE_DISK) {
+        m_fname = s.get_desc_fname();
+    }
+    init();
+    set_size(s.size());
+#ifdef PSP_MPROTECT
+    PSP_VERBOSE_ASSERT(
+        sizeof(t_lstore) == get_page_size(), "Bad lstore sizeof"
+    );
+    freeze_impl();
+#endif
+}
+
+std::string
+t_lstore::repr() const {
+    std::stringstream ss;
+    ss << "t_lstore<" << this << ">";
+    return ss.str();
+}
+
+void
+t_lstore::copy_helper(t_lstore& other) {
+    PSP_TRACE_SENTINEL();
+    copy_helper_(other);
+}
+
+void
+t_lstore::copy_helper(const t_lstore& other) {
+    PSP_TRACE_SENTINEL();
+    copy_helper_(other);
+}
+
+void
+t_lstore::copy_helper_(const t_lstore& other) {
+    PSP_TRACE_SENTINEL();
+    m_dirname = other.m_dirname;
+    m_fname = other.m_fname;
+    m_colname = other.m_colname;
+    m_base = nullptr;
+    m_fd = other.m_fd;
+    m_capacity = other.m_capacity;
+    m_size = other.m_size;
+    m_alignment = other.m_alignment;
+    m_fflags = other.m_fflags;
+    m_fmode = other.m_fmode;
+    m_creation_disposition = other.m_creation_disposition;
+    m_mprot = other.m_mprot;
+    m_mflags = other.m_mflags;
+    m_backing_store = other.m_backing_store;
+    m_init = false;
+    m_resize_factor = other.m_resize_factor;
+    m_version = other.m_version;
+    m_from_recipe = other.m_from_recipe;
+    PSP_CHECK_CAPACITY();
+}
+
+t_lstore::t_lstore(t_lstore&& other) noexcept {
+    PSP_VERBOSE_ASSERT(this != &other, "Constructing from self");
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore");
+    copy_helper(other);
+    m_init = false;
+}
+
+t_lstore&
+t_lstore::operator=(t_lstore&& other) noexcept {
+    PSP_VERBOSE_ASSERT(this != &other, "Assigning self");
+    PSP_TRACE_SENTINEL();
+    LOG_CONSTRUCTOR("t_lstore");
+    copy_helper(other);
+    m_init = false;
+    return *this;
+}
+
+t_lstore::~t_lstore() {
+    PSP_TRACE_SENTINEL();
+    if (!m_init) {
+        return;
+    }
+
+    switch (m_backing_store) {
+        case BACKING_STORE_DISK: {
+            t_residency_manager::inst().unregister_store(this);
+            destroy_mapping();
+            close_file(m_fd);
+
+            bool dont_delete =
+                std::getenv("PSP_DO_NOT_DELETE_TABLES") != nullptr;
+
+            if (!dont_delete) {
+                rmfile(m_fname);
+            }
+        } break;
+        case BACKING_STORE_MEMORY: {
+#ifdef _MSC_VER
+            if (m_alignment >= 2) {
+                _aligned_free(m_base); // seriously
+            } else
+#endif // _MSC_VER
+            {
+                free(m_base);
+            }
+
+#ifdef PSP_MPROTECT
+            unfreeze_impl();
+#endif
+        } break;
+        default: {
+            PSP_VERBOSE_ASSERT(false, "Unknown backing store");
+        } break;
+    }
+}
+
+void
+t_lstore::set_size(t_uindex idx) // in bytes
+{
+    PSP_TRACE_SENTINEL();
+    t_unlock_store tmp(this);
+#ifdef PSP_VERIFY
+    PSP_VERBOSE_ASSERT(m_size <= m_capacity, "Setting bad size");
+#endif
+    m_size = idx;
+}
+
+void
+t_lstore::init() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(!m_init, "Already inited column");
+    LOG_INIT("t_lstore");
+
+    t_unlock_store tmp(this);
+    switch (m_backing_store) {
+        case BACKING_STORE_DISK: {
+            PSP_VERBOSE_ASSERT(
+                m_alignment < 2,
+                "nontrivial alignments currently "
+                "unsupported for BACKING_STORE_DISK"
+            );
+            m_fd = create_file();
+            m_base = create_mapping();
+        } break;
+        case BACKING_STORE_MEMORY: {
+            size_t const alloc_size = std::max(
+                std::max(size_t(m_alignment), size_t(8U)), size_t(capacity())
+            );
+
+            if (m_alignment < 2) {
+                m_base = calloc(alloc_size, 1);
+            } else {
+                // nontrivial alignment
+                PSP_VERBOSE_ASSERT(
+                    !(m_alignment & (m_alignment - 1)),
+                    "store alignment must be a power of two!"
+                );
+
+#ifdef _MSC_VER
+                m_base = _aligned_malloc(alloc_size, size_t(m_alignment));
+#else
+                int result = posix_memalign(
+                    &m_base,
+                    std::max(sizeof(void*), size_t(m_alignment)),
+                    alloc_size
+                );
+                if (result != 0) {
+                    m_base = nullptr;
+                }
+#endif
+
+                PSP_VERBOSE_ASSERT(
+                    m_base,
+                    "MALLOC_FAILED"
+                ); // ensure we check before trying to
+                   // memset() in this case
+                memset(m_base, 0, alloc_size);
+            }
+
+            PSP_VERBOSE_ASSERT(m_base, "MALLOC_FAILED");
+        } break;
+        default: {
+            PSP_VERBOSE_ASSERT(false, "Unknown backing store");
+        } break;
+    }
+
+    m_init = true;
+
+    if (m_backing_store == BACKING_STORE_DISK) {
+        m_residency_tick = g_residency_tick;
+        t_residency_manager::inst().register_store(this);
+    }
+}
+
+void
+t_lstore::reserve(t_uindex capacity) {
+    reserve_impl(capacity, false);
+}
+
+void
+t_lstore::shrink(t_uindex capacity) {
+    reserve_impl(capacity, true);
+}
+
+void
+t_lstore::reserve_impl(t_uindex capacity, bool allow_shrink) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // ensure_resident();
+    if ((capacity < m_capacity) && !allow_shrink) {
+        return;
+    }
+
+    PSP_VERBOSE_ASSERT(
+        capacity >= m_size, "reduce size before reducing capacity!"
+    );
+    capacity = std::max(capacity, m_size);
+
+    capacity = 4 * std::uint64_t(ceil(double(capacity * m_resize_factor) / 4));
+    capacity = std::max(capacity, static_cast<t_uindex>(8));
+    if (m_alignment > 1) {
+        capacity = (capacity + m_alignment - 1) & ~(m_alignment - 1);
+    }
+    t_uindex ocapacity = m_capacity;
+
+    if (t_env::log_storage_resize()) {
+        std::cout << repr() << " ocap => " << ocapacity << " ncap => "
+                  << capacity << std::endl;
+    }
+
+    switch (m_backing_store) {
+        case BACKING_STORE_MEMORY: {
+            void* base = nullptr;
+
+            if (m_alignment < 2) {
+                base = realloc(m_base, size_t(capacity));
+            } else {
+// nontrivial alignment
+#if _MSC_VER
+                base = _aligned_realloc(
+                    m_base, size_t(capacity), size_t(m_alignment)
+                );
+#else
+                base = realloc(m_base, size_t(capacity));
+
+                if ((uintptr_t(base) & (m_alignment - 1)) != 0) {
+                    // realloc() hasn't given us the correct alignment
+                    // so we need to fix it up
+                    PSP_VERBOSE_ASSERT(
+                        !(m_alignment & (m_alignment - 1)),
+                        "store alignment must be a power of two!"
+                    );
+
+                    void* aligned_base = nullptr;
+                    int result = posix_memalign(
+                        &aligned_base,
+                        std::max(sizeof(void*), size_t(m_alignment)),
+                        size_t(capacity)
+                    );
+                    PSP_VERBOSE_ASSERT(result, == 0, "posix_memalign failed");
+
+                    memcpy(aligned_base, base, ocapacity);
+                    free(base);
+                    base = aligned_base;
+                }
+#endif
+            }
+
+            PSP_VERBOSE_ASSERT(base != nullptr, "realloc failed");
+            {
+                t_unlock_store tmp(this);
+                m_base = base;
+                m_capacity = capacity;
+                ++m_version;
+            }
+        } break;
+        case BACKING_STORE_DISK: {
+            PSP_VERBOSE_ASSERT(
+                m_alignment < 2,
+                "nontrivial alignments currently "
+                "unsupported for BACKING_STORE_DISK"
+            );
+            resize_mapping(capacity);
+            ++m_version;
+        } break;
+        default: {
+            PSP_COMPLAIN_AND_ABORT("unknown backing medium");
+        }
+    }
+
+    if (capacity > ocapacity) {
+        memset(
+            static_cast<unsigned char*>(m_base) + ocapacity,
+            0,
+            size_t(capacity - ocapacity)
+        );
+    }
+}
+
+// Assumes store has been initted
+void
+t_lstore::load(const std::string& fname) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+
+    t_rfmapping imap;
+    map_file_read(fname, imap);
+    reserve(imap.m_size);
+    memcpy(m_base, imap.m_base, size_t(imap.m_size));
+    m_size = imap.m_size;
+    PSP_CHECK_CAPACITY();
+}
+
+void
+t_lstore::save(const std::string& fname) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    PSP_VERBOSE_ASSERT(m_init, "Store not inited.");
+
+    t_rfmapping omap;
+    map_file_write(fname, capacity(), omap);
+    memcpy(omap.m_base, m_base, size_t(capacity()));
+}
+
+void
+t_lstore::copy(t_lstore& out) const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    PSP_COMPLAIN_AND_ABORT("copy is unimplemented!");
+}
+
+void
+t_lstore::warmup() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+}
+
+t_uindex
+t_lstore::size() const {
+    PSP_TRACE_SENTINEL();
+    return m_size;
+}
+
+t_uindex
+t_lstore::capacity() const {
+    PSP_TRACE_SENTINEL();
+    return m_capacity;
+}
+
+std::pair<std::uint32_t, std::uint32_t>
+t_lstore::capacity_pair() const {
+    PSP_TRACE_SENTINEL();
+    t_uindex cap = capacity();
+    return std::pair<std::uint32_t, std::uint32_t>(upper32(cap), lower32(cap));
+}
+
+std::string
+t_lstore::get_fname() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_fname;
+}
+
+void
+t_lstore::push_back(const void* ptr, t_uindex len) {
+    PSP_TRACE_SENTINEL();
+    // ensure_resident();
+    if (m_size + len >= m_capacity) {
+        reserve(static_cast<t_uindex>(m_size + len)
+        ); // reserve() will multiply by m_resize_factor internally
+    }
+
+    PSP_VERBOSE_ASSERT(m_size + len < m_capacity, "Insufficient capacity.");
+
+    memcpy(static_cast<unsigned char*>(m_base) + m_size, ptr, size_t(len));
+
+    {
+        t_unlock_store tmp(this);
+        m_size += len;
+    }
+    PSP_CHECK_CAPACITY();
+}
+
+void*
+t_lstore::get_ptr(t_uindex offset) {
+    return static_cast<void*>(static_cast<unsigned char*>(m_base) + offset);
+}
+
+const void*
+t_lstore::get_ptr(t_uindex offset) const {
+    return static_cast<void*>(static_cast<unsigned char*>(m_base) + offset);
+}
+
+std::string
+t_lstore::get_desc_fname() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return unique_path(m_fname);
+}
+
+t_uindex
+t_lstore::get_version() const {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    return m_version;
+}
+
+void
+t_lstore::append(const t_lstore& other) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // const_cast<t_lstore&>(other).ensure_resident();
+    push_back(other.m_base, other.size());
+}
+
+void
+t_lstore::clear() {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+#ifndef PSP_ENABLE_WASM
+    ensure_resident();
+    memset(m_base, 0, size_t(capacity()));
+#endif
+    {
+        t_unlock_store tmp(this);
+        m_size = 0;
+    }
+}
+
+t_lstore_recipe
+t_lstore::get_recipe() const {
+    t_lstore_recipe rval(
+        m_dirname,
+        m_colname,
+        m_capacity,
+        PSP_DEFAULT_SHARED_RO_FFLAGS,
+        PSP_DEFAULT_SHARED_RO_FMODE,
+        PSP_DEFAULT_SHARED_RO_CREATION_DISPOSITION,
+        PSP_DEFAULT_SHARED_RO_MPROT,
+        PSP_DEFAULT_SHARED_RO_MFLAGS,
+        m_backing_store
+    );
+    rval.m_fname = m_fname;
+    rval.m_from_recipe = true;
+    rval.m_size = m_size;
+    rval.m_alignment = m_alignment;
+    return rval;
+}
+
+t_lstore_recipe
+t_lstore::get_clone_recipe() const {
+    t_lstore_recipe rval = get_recipe();
+    if (m_backing_store == BACKING_STORE_DISK) {
+        rval.m_from_recipe = false;
+    }
+    return rval;
+}
+
+void
+t_lstore::fill(const t_lstore& other) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    // const_cast<t_lstore&>(other).ensure_resident();
+    reserve(other.size());
+    memcpy(m_base, const_cast<void*>(other.m_base), size_t(other.size()));
+    set_size(other.size());
+}
+
+void
+t_lstore::fill(const t_lstore& other, const t_mask& mask, t_uindex elem_size) {
+    PSP_TRACE_SENTINEL();
+    PSP_VERBOSE_ASSERT(m_init, "touching uninited object");
+    reserve(mask.size() * elem_size);
+
+    PSP_VERBOSE_ASSERT(
+        mask.size() * elem_size <= m_size, "Not enough space to fill"
+    );
+
+    t_uindex offset = 0;
+
+    const auto* src_base = reinterpret_cast<const char*>(other.get_ptr(0));
+    auto* dst_base = reinterpret_cast<char*>(m_base);
+
+    for (t_uindex idx = 0, loop_end = mask.size(); idx < loop_end; ++idx) {
+        if (mask.get(idx)) {
+            memcpy(
+                dst_base + offset, src_base + idx * elem_size, size_t(elem_size)
+            );
+            offset += elem_size;
+        }
+    }
+
+    set_size(mask.count() * elem_size);
+}
+
+void
+t_lstore::pprint() const {
+    std::cout << repr() << std::endl;
+    t_uindex nelems = size() / sizeof(std::uint8_t);
+    for (t_uindex idx = 0; idx < size() / nelems; ++idx) {
+
+        std::cout << idx << " => "
+                  << static_cast<t_uindex>(*(get_nth<std::uint8_t>(idx)))
+                  << std::endl;
+    }
+}
+
+std::shared_ptr<t_lstore>
+t_lstore::clone() const {
+    auto recipe = get_clone_recipe();
+    std::shared_ptr<t_lstore> rval(new t_lstore(recipe));
+    rval->init();
+    rval->set_size(m_size);
+    rval->fill(*this);
+    return rval;
+}
+
+void
+t_lstore::evict() {
+    if (m_backing_store != BACKING_STORE_DISK || !m_init || m_base == nullptr) {
+        return;
+    }
+#ifdef PSP_HAS_OPFS_BRIDGE
+    // Flush the resident buffer to OPFS (suspending), then free it. Only valid
+    // from a `promising` safepoint — never mid-request.
+    psp_opfs_store(
+        m_fname.c_str(), m_base, static_cast<int>(capacity())
+    );
+    free(m_base);
+    m_base = nullptr;
+#else
+    // Native: `destroy_mapping()` (`munmap`) writes back the `MAP_SHARED` pages;
+    // the file `m_fd` stays open for a later `restore()`.
+    destroy_mapping();
+    m_base = nullptr;
+#endif
+    ++m_version;
+}
+
+void
+t_lstore::restore() {
+    if (m_backing_store != BACKING_STORE_DISK || !m_init || m_base != nullptr) {
+        return;
+    }
+    // May be called from parallel-for workers within a request (native); the
+    // OPFS path is only entered from the single-threaded promising safepoint.
+    std::lock_guard<std::mutex> lk(t_residency_manager::inst().mutex());
+    if (m_base != nullptr) {
+        return;
+    }
+#ifdef PSP_HAS_OPFS_BRIDGE
+    auto cap = static_cast<size_t>(capacity());
+    m_base = calloc(std::max(cap, static_cast<size_t>(1)), 1);
+    psp_opfs_load(m_fname.c_str(), m_base, static_cast<int>(capacity()));
+#else
+    m_base = create_mapping();
+#endif
+    // Stamp LRU recency at bring-in time. Moved here from `ensure_resident()` so
+    // the per-access hot path reads no residency global (see `storage.h`).
+    m_residency_tick = g_residency_tick;
+    t_residency_manager::inst().note_restore();
+    ++m_version;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/storage_impl_linux.cpp

@@ -0,0 +1,132 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+
+// NB: `first.h` force-defines `__linux__` on any non-mac/non-win platform,
+// *including emscripten*. Exclude WASM here so `storage_impl_wasm.cpp` (and not
+// this `open`/`mmap` implementation) provides the storage backend on WASM.
+#if defined(__linux__) && !defined(PSP_ENABLE_WASM)
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/storage.h>
+#include <perspective/raii.h>
+#include <perspective/defaults.h>
+#include <perspective/compat.h>
+#include <perspective/utils.h>
+#include <iostream>
+#include <assert.h>
+#include <csignal>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <csignal>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+namespace perspective {
+
+t_lstore::t_lstore(const t_lstore_recipe& a) :
+    m_base(0),
+    m_dirname(a.m_dirname),
+    m_colname(a.m_colname),
+    m_fd(-1),
+    m_capacity(a.m_capacity),
+    m_size(0),
+    m_alignment(a.m_alignment),
+    m_fflags(a.m_fflags),
+    m_fmode(a.m_fmode),
+    m_creation_disposition(a.m_creation_disposition),
+    m_mprot(a.m_mprot),
+    m_mflags(a.m_mflags),
+    m_backing_store(a.m_backing_store),
+    m_init(false),
+    m_resize_factor(1.3),
+    m_version(0),
+    m_from_recipe(a.m_from_recipe) {
+    if (m_from_recipe) {
+        m_fname = a.m_fname;
+        return;
+    }
+
+    if (m_backing_store == BACKING_STORE_DISK) {
+        std::stringstream ss;
+        ss << a.m_dirname << "/"
+           << "_col_" << a.m_colname << "_" << this;
+        m_fname = unique_path(ss.str());
+    }
+}
+
+t_handle
+t_lstore::create_file() {
+    t_handle fd = open(m_fname.c_str(), m_fflags, m_fmode);
+    PSP_VERBOSE_ASSERT(fd != -1, "Error opening file");
+
+    if (m_from_recipe) {
+        return fd;
+    }
+
+    t_index truncate_bytes = static_cast<t_index>(capacity());
+
+    t_index rcode = ftruncate(fd, truncate_bytes);
+
+    PSP_VERBOSE_ASSERT(rcode >= 0, "Ftruncate failed");
+    return fd;
+}
+
+void*
+// NOLINTNEXTLINE
+t_lstore::create_mapping() {
+    void* rval = mmap(0, capacity(), m_mprot, m_mflags, m_fd, 0);
+    PSP_VERBOSE_ASSERT(rval != MAP_FAILED, "mmap failed");
+    return rval;
+}
+
+void
+t_lstore::resize_mapping(t_uindex cap_new) {
+    t_index rcode = ftruncate(m_fd, cap_new);
+    PSP_VERBOSE_ASSERT(rcode == 0, "ftruncate failed");
+    void* base = mremap(m_base, capacity(), cap_new, MREMAP_MAYMOVE);
+
+    if (base == MAP_FAILED) {
+        PSP_COMPLAIN_AND_ABORT("mremap failed!");
+    }
+
+    m_base = base;
+    m_capacity = cap_new;
+}
+
+void
+t_lstore::destroy_mapping() {
+    if (m_base == nullptr) {
+        return; // already evicted
+    }
+    t_index rc = munmap(m_base, capacity());
+    PSP_VERBOSE_ASSERT(!rc, "Failed to destroy mapping");
+}
+
+void
+t_lstore::freeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+t_lstore::unfreeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/storage_impl_osx.cpp

@@ -0,0 +1,135 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __APPLE__
+
+#include <perspective/first.h>
+
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/storage.h>
+#include <perspective/raii.h>
+
+#include <perspective/defaults.h>
+#include <perspective/compat.h>
+#include <perspective/utils.h>
+#include <iostream>
+#include <cassert>
+#include <csignal>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+namespace perspective {
+
+t_lstore::t_lstore(const t_lstore_recipe& a) :
+    m_base(nullptr),
+    m_dirname(a.m_dirname),
+    m_colname(a.m_colname),
+    m_fd(-1),
+    m_capacity(a.m_capacity),
+    m_size(0),
+    m_alignment(a.m_alignment),
+    m_fflags(a.m_fflags),
+    m_fmode(a.m_fmode),
+    m_creation_disposition(a.m_creation_disposition),
+    m_mprot(a.m_mprot),
+    m_mflags(a.m_mflags),
+    m_backing_store(a.m_backing_store),
+    m_init(false),
+    m_resize_factor(1.3),
+    m_version(0),
+    m_from_recipe(a.m_from_recipe) {
+    if (m_from_recipe) {
+        m_fname = a.m_fname;
+        return;
+    }
+
+    if (m_backing_store == BACKING_STORE_DISK) {
+        std::stringstream ss;
+        ss << a.m_dirname << "/"
+           << "_col_" << a.m_colname << "_" << this;
+        m_fname = unique_path(ss.str());
+    }
+}
+
+t_handle
+t_lstore::create_file() {
+    t_handle fd = open(m_fname.c_str(), m_fflags, m_fmode);
+    PSP_VERBOSE_ASSERT(fd != -1, "Error opening file");
+
+    if (m_from_recipe) {
+        return fd;
+    }
+
+    auto truncate_bytes = static_cast<t_index>(capacity());
+
+    t_index rcode = ftruncate(fd, truncate_bytes);
+
+    PSP_VERBOSE_ASSERT(rcode, >= 0, "Ftruncate failed");
+    return fd;
+}
+
+void*
+// NOLINTNEXTLINE
+t_lstore::create_mapping() {
+    void* rval = mmap(nullptr, capacity(), m_mprot, m_mflags, m_fd, 0);
+    PSP_VERBOSE_ASSERT(rval, != MAP_FAILED, "mmap failed");
+    return rval;
+}
+
+void
+t_lstore::resize_mapping(t_uindex cap_new) {
+    t_index rcode = ftruncate(m_fd, cap_new);
+    PSP_VERBOSE_ASSERT(rcode, == 0, "ftruncate failed");
+
+    if (munmap(m_base, capacity()) == -1) {
+        throw;
+    }
+
+    void* base =
+        mmap(nullptr, cap_new, PROT_READ | PROT_WRITE, MAP_SHARED, m_fd, 0);
+
+    if (base == MAP_FAILED) {
+        PSP_COMPLAIN_AND_ABORT("mremap failed!");
+    }
+
+    m_base = base;
+    m_capacity = cap_new;
+}
+
+void
+t_lstore::destroy_mapping() {
+    if (m_base == nullptr) {
+        return; // already evicted
+    }
+    t_index rc = munmap(m_base, capacity());
+    PSP_VERBOSE_ASSERT(rc, == 0, "Failed to destroy mapping");
+}
+
+void
+t_lstore::freeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+t_lstore::unfreeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/storage_impl_wasm.cpp

@@ -0,0 +1,127 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+
+#ifdef PSP_ENABLE_WASM
+
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/storage.h>
+#include <perspective/raii.h>
+#include <perspective/defaults.h>
+#include <perspective/compat.h>
+#include <perspective/utils.h>
+#include <cstdlib>
+#include <cstring>
+#include <sstream>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+namespace perspective {
+
+// ┌─────────────────────────────────────────────────────────────────────────┐
+// │ Emulated mmap over WasmFS/OPFS                                            │
+// │                                                                           │
+// │ WasmFS has no usable file-backed `mmap` (and even where one exists it     │
+// │ copies the whole file into linear memory), so `BACKING_STORE_DISK` on     │
+// │ WASM keeps a resident `malloc` buffer (`m_base`) that mirrors the backing │
+// │ file. Reads/writes go through `m_base`; the file is the durable copy that │
+// │ is (re)read via `pread` on mapping and written via `pwrite` on flush.     │
+// │                                                                           │
+// │ Within a session `m_base` is the source of truth — the file is sized to   │
+// │ match (so a later read/restore is correct) but is only written when the   │
+// │ buffer is flushed. This is the substrate the residency manager (Phase 4)  │
+// │ evicts: `pwrite` the buffer, `free` it, keep the file; restore by         │
+// │ re-`pread`ing into a fresh buffer.                                        │
+// └─────────────────────────────────────────────────────────────────────────┘
+
+t_lstore::t_lstore(const t_lstore_recipe& a) :
+    m_base(nullptr),
+    m_dirname(a.m_dirname),
+    m_colname(a.m_colname),
+    m_fd(-1),
+    m_capacity(a.m_capacity),
+    m_size(0),
+    m_alignment(a.m_alignment),
+    m_fflags(a.m_fflags),
+    m_fmode(a.m_fmode),
+    m_creation_disposition(a.m_creation_disposition),
+    m_mprot(a.m_mprot),
+    m_mflags(a.m_mflags),
+    m_backing_store(a.m_backing_store),
+    m_init(false),
+    m_resize_factor(1.3),
+    m_version(0),
+    m_from_recipe(a.m_from_recipe) {
+    if (m_from_recipe) {
+        m_fname = a.m_fname;
+        return;
+    }
+
+    if (m_backing_store == BACKING_STORE_DISK) {
+        std::stringstream ss;
+        ss << a.m_dirname << "/"
+           << "_col_" << a.m_colname << "_" << this;
+        m_fname = unique_path(ss.str());
+    }
+}
+
+// A disk-backed column on WASM is just a resident `malloc` buffer; there is no
+// OS file (the build is `NO_FILESYSTEM`). Persistence to OPFS is performed by
+// the residency manager (`t_lstore::evict`/`restore` in storage.cpp) via the
+// `psp_opfs_*` bridge, at promising safepoints. `m_fname` is the OPFS key.
+
+t_handle
+t_lstore::create_file() {
+    // No OS file; return a sentinel handle.
+    return 1;
+}
+
+void*
+t_lstore::create_mapping() {
+    auto cap = static_cast<size_t>(capacity());
+    void* base = calloc(std::max(cap, static_cast<size_t>(1)), 1);
+    PSP_VERBOSE_ASSERT(base != nullptr, "calloc failed");
+    return base;
+}
+
+void
+t_lstore::resize_mapping(t_uindex cap_new) {
+    void* base = realloc(m_base, static_cast<size_t>(cap_new));
+    PSP_VERBOSE_ASSERT(base != nullptr, "realloc failed");
+    m_base = base;
+    m_capacity = cap_new;
+}
+
+void
+t_lstore::destroy_mapping() {
+    if (m_base != nullptr) {
+        free(m_base);
+        m_base = nullptr;
+    }
+}
+
+void
+t_lstore::freeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+void
+t_lstore::unfreeze_impl() {
+    PSP_COMPLAIN_AND_ABORT("Not implemented");
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/storage_impl_win.cpp

@@ -0,0 +1,185 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef WIN32
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/raw_types.h>
+#include <perspective/storage.h>
+#include <perspective/raii.h>
+#include <perspective/defaults.h>
+#include <perspective/compat.h>
+#include <perspective/utils.h>
+#include <iostream>
+#include <assert.h>
+#include <csignal>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <csignal>
+
+namespace perspective {
+
+t_lstore::t_lstore(const t_lstore_recipe& a) :
+    m_dirname(a.m_dirname),
+    m_colname(a.m_colname),
+    m_base(0),
+    m_fd(0),
+    m_capacity(a.m_capacity),
+    m_size(0),
+    m_alignment(a.m_alignment),
+    m_fflags(a.m_fflags),
+    m_fmode(a.m_fmode),
+    m_creation_disposition(a.m_creation_disposition),
+    m_mprot(a.m_mprot),
+    m_mflags(a.m_mflags),
+    m_backing_store(a.m_backing_store),
+    m_init(false),
+    m_resize_factor(1.3),
+    m_version(0),
+    m_from_recipe(a.m_from_recipe) {
+    if (m_from_recipe) {
+        m_fname = a.m_fname;
+        return;
+    }
+
+    if (m_backing_store == BACKING_STORE_DISK) {
+        std::stringstream ss;
+        ss << a.m_dirname << "\\"
+           << "_col_" << a.m_colname << "_" << this;
+        m_fname = unique_path(ss.str());
+    }
+}
+
+t_handle
+t_lstore::create_file() {
+    t_handle rval = CreateFile(
+        m_fname.c_str(),
+        m_fflags,
+        m_fmode,
+        0, // security
+        m_creation_disposition,
+        FILE_ATTRIBUTE_NORMAL,
+        0 // template file
+    );
+
+    PSP_VERBOSE_ASSERT(rval != INVALID_HANDLE_VALUE, "Error opening file");
+
+    if (m_from_recipe) {
+        return rval;
+    }
+
+    LARGE_INTEGER sz;
+    sz.QuadPart = capacity();
+
+    auto rb = SetFilePointerEx(rval, sz, 0, FILE_BEGIN);
+    PSP_VERBOSE_ASSERT(rb, "Error setting fpointer");
+
+    rb = SetEndOfFile(rval);
+    PSP_VERBOSE_ASSERT(rb, "Error setting eof");
+
+    return rval;
+}
+
+void*
+t_lstore::create_mapping() {
+    std::pair<std::uint32_t, std::uint32_t> capacity = capacity_pair();
+
+    m_winmapping_handle = CreateFileMapping(
+        m_fd,
+        0, // default security
+        m_mprot,
+        capacity.first,
+        capacity.second,
+        0 // anonymous mapping
+    );
+
+    PSP_VERBOSE_ASSERT(m_winmapping_handle != 0, "Error creating filemapping");
+
+    void* ptr = MapViewOfFile(
+        m_winmapping_handle,
+        m_mflags,
+        0, // 0 offset
+        0, // 0 offset
+        0  // entire file
+    );
+
+    PSP_VERBOSE_ASSERT(ptr != 0, "Error mapping file");
+
+    return ptr;
+}
+
+void
+t_lstore::resize_mapping(t_uindex cap_new) {
+    flush_mapping(m_base, capacity());
+    destroy_mapping();
+
+    // TODO error handling
+    CloseHandle(m_winmapping_handle);
+
+    LARGE_INTEGER sz;
+    sz.QuadPart = cap_new;
+
+    auto rb = SetFilePointerEx(m_fd, sz, 0, FILE_BEGIN);
+    PSP_VERBOSE_ASSERT(rb, "Error setting fpointer");
+
+    rb = SetEndOfFile(m_fd);
+    PSP_VERBOSE_ASSERT(rb, "Error setting eof");
+
+    m_capacity = cap_new;
+    m_base = create_mapping();
+}
+
+void
+t_lstore::destroy_mapping() {
+    auto rc = UnmapViewOfFile(m_base);
+    PSP_VERBOSE_ASSERT(rc, "Error unmapping view");
+    m_base = 0;
+}
+
+void
+t_lstore::freeze_impl() {
+    DWORD dwOld;
+    if (VirtualProtect(
+            (LPVOID)this,
+            static_cast<size_t>(get_page_size()),
+            PAGE_READONLY,
+            &dwOld
+        )
+        == 0) {
+        std::cout << "Virtual protect failed addr => " << this
+                  << " error code => " << GetLastError() << "\n";
+        PSP_VERBOSE_ASSERT(false, "virtual protect failed");
+    }
+}
+
+void
+t_lstore::unfreeze_impl() {
+    DWORD dwOld;
+    if (VirtualProtect(
+            (LPVOID)this,
+            static_cast<size_t>(get_page_size()),
+            PAGE_READWRITE,
+            &dwOld
+        )
+        == 0) {
+        std::cout << "Virtual protect failed addr => " << this
+                  << " error code => " << GetLastError() << "\n";
+        PSP_VERBOSE_ASSERT(false, "virtual protect failed");
+    }
+}
+
+} // end namespace perspective
+
+#endif

rust/perspective-server/cpp/perspective/src/cpp/sym_table.cpp

@@ -0,0 +1,128 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/sym_table.h>
+#include <perspective/column.h>
+#include <tsl/hopscotch_map.h>
+#include <mutex>
+
+namespace perspective {
+
+std::mutex sym_table_mutex;
+
+t_symtable::t_symtable() = default;
+
+t_symtable::~t_symtable() {
+    for (const auto& kv : m_mapping) {
+        free(const_cast<char*>(kv.second));
+    }
+}
+
+const char*
+t_symtable::get_interned_cstr(const char* s) {
+    auto iter = m_mapping.find(s);
+
+    if (iter != m_mapping.end()) {
+        return iter->second;
+    }
+
+    auto* scopy = strdup(s);
+    m_mapping[scopy] = scopy;
+    return scopy;
+}
+
+t_tscalar
+t_symtable::get_interned_tscalar(const char* s) {
+#ifdef PSP_SSO_SCALAR
+    if (t_tscalar::can_store_inplace(s)) {
+        t_tscalar rval;
+        rval.set(s);
+        return rval;
+    }
+#endif
+    t_tscalar rval;
+    rval.set(get_interned_cstr(s));
+    return rval;
+}
+
+t_tscalar
+t_symtable::get_interned_tscalar(const t_tscalar& s) {
+#ifdef PSP_SSO_SCALAR
+    if (!s.is_str() || s.is_inplace()) {
+        return s;
+    }
+#else
+    if (!s.is_str()) {
+        return s;
+    }
+#endif
+    t_tscalar rval;
+    rval.set(get_interned_cstr(s.get_char_ptr()));
+    rval.m_status = s.m_status;
+    return rval;
+}
+
+t_uindex
+t_symtable::size() const {
+    return m_mapping.size();
+}
+
+static t_symtable*
+get_symtable() {
+    static t_symtable* sym = nullptr;
+
+    if (sym == nullptr) {
+        sym = new t_symtable;
+    }
+    return sym;
+}
+
+const char*
+get_interned_cstr(const char* s) {
+    std::lock_guard<std::mutex> guard(sym_table_mutex);
+    auto* sym = get_symtable();
+    return sym->get_interned_cstr(s);
+}
+
+t_tscalar
+get_interned_tscalar(const char* s) {
+#ifdef PSP_SSO_SCALAR
+    if (t_tscalar::can_store_inplace(s)) {
+        t_tscalar rval;
+        rval.set(s);
+        return rval;
+    }
+#endif
+    t_tscalar rval;
+    rval.set(get_interned_cstr(s));
+    return rval;
+}
+
+t_tscalar
+get_interned_tscalar(const t_tscalar& s) {
+#ifdef PSP_SSO_SCALAR
+    if (!s.is_str() || s.is_inplace()) {
+        return s;
+    }
+#else
+    if (!s.is_str()) {
+        return s;
+    }
+#endif
+    t_tscalar rval;
+    rval.set(get_interned_cstr(s.get_char_ptr()));
+    return rval;
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/time.cpp

@@ -0,0 +1,289 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/time.h>
+#include <perspective/utils.h>
+
+namespace perspective {
+
+t_tdelta::t_tdelta() : v(0) {}
+
+t_tdelta::t_tdelta(std::int64_t v) : v(v) {}
+
+t_tdelta&
+t_tdelta::operator*=(std::int64_t multiplier) {
+    v *= multiplier;
+    return *this;
+}
+
+t_time::t_time() : m_storage(0) {}
+
+t_time::t_time(std::int64_t raw_val) : m_storage(raw_val) {}
+
+t_time::t_time(int year, int month, int day, int hour, int min, int sec) :
+    m_storage(static_cast<std::int64_t>(
+        to_gmtime(year, month, day, hour, min, sec) * 1000
+    )) {}
+
+std::int64_t
+t_time::raw_value() const {
+    return m_storage;
+}
+
+bool
+operator<(const t_time& a, const t_time& b) {
+    return a.m_storage < b.m_storage;
+}
+
+bool
+operator<=(const t_time& a, const t_time& b) {
+    return a.m_storage <= b.m_storage;
+}
+
+bool
+operator>(const t_time& a, const t_time& b) {
+    return a.m_storage > b.m_storage;
+}
+
+bool
+operator>=(const t_time& a, const t_time& b) {
+    return a.m_storage >= b.m_storage;
+}
+
+bool
+operator==(const t_time& a, const t_time& b) {
+    return a.m_storage == b.m_storage;
+}
+
+bool
+operator!=(const t_time& a, const t_time& b) {
+    return a.m_storage != b.m_storage;
+}
+
+bool
+t_time::as_tm(struct tm& out) const {
+    return gmtime(out, as_seconds(), 0) == 1;
+}
+
+std::tm
+t_time::get_tm() const {
+    std::tm rval;
+    gmtime(rval, microseconds(), 0);
+    return rval;
+}
+
+std::int32_t
+isleap(long int year) {
+    return (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) ? 1 : 0;
+}
+
+std::int32_t
+t_time::gmtime(struct tm& out, std::int64_t secs, std::int32_t offset) {
+    std::int64_t days;
+    std::int64_t rem;
+    std::int64_t y;
+    const unsigned short int* ip;
+
+    days = secs / SECS_PER_DAY;
+    rem = secs % SECS_PER_DAY;
+    rem += offset;
+    while (rem < 0) {
+        rem += SECS_PER_DAY;
+        --days;
+    }
+    while (rem >= SECS_PER_DAY) {
+        rem -= SECS_PER_DAY;
+        ++days;
+    }
+    out.tm_hour = rem / SECS_PER_HOUR;
+    rem %= SECS_PER_HOUR;
+    out.tm_min = rem / 60;
+    out.tm_sec = rem % 60;
+    /* January 1, 1970 was a Thursday.  */
+    out.tm_wday = (4 + days) % 7;
+    if (out.tm_wday < 0) {
+        out.tm_wday += 7;
+    }
+    y = 1970;
+
+#define DIV(a, b) ((a) / (b) - ((a) % (b) < 0))
+#define LEAPS_THRU_END_OF(y) (DIV(y, 4) - DIV(y, 100) + DIV(y, 400))
+
+    while (days < 0 || days >= (isleap(y) != 0 ? 366 : 365)) {
+        /* Guess a corrected year, assuming 365 days per
+         * year.  */
+        long int yg =
+            y + days / 365 - static_cast<std::int64_t>(days % 365 < 0);
+
+        /* Adjust DAYS and Y to match the guessed year.
+         */
+        days -=
+            ((yg - y) * 365 + LEAPS_THRU_END_OF(yg - 1)
+             - LEAPS_THRU_END_OF(y - 1));
+        y = yg;
+    }
+    out.tm_year = y - 1900;
+    if (out.tm_year != y - 1900) {
+        /* The year cannot be represented due to
+         * overflow.  */
+        return 0;
+    }
+    out.tm_yday = days;
+    ip = __mon_yday[isleap(y)];
+    for (y = 11; days < (long int)ip[y]; --y) {}
+    days -= ip[y];
+    out.tm_mon = y;
+    out.tm_mday = days + 1;
+    return 1;
+}
+
+std::int32_t
+t_time::year(const struct tm& t) const {
+    return t.tm_year + 1900;
+}
+std::int32_t
+t_time::month(const struct tm& t) const {
+    return t.tm_mon + 1;
+}
+std::int32_t
+t_time::day(const struct tm& t) const {
+    return t.tm_mday;
+}
+std::int32_t
+t_time::hours(const struct tm& t) const {
+    return t.tm_hour;
+}
+std::int32_t
+t_time::minutes(const struct tm& t) const {
+    return t.tm_min;
+}
+std::int32_t
+t_time::seconds(const struct tm& t) const {
+    return t.tm_sec;
+}
+std::int32_t
+t_time::microseconds() const // component
+{
+    std::int32_t micros = m_storage % 1000000;
+    return micros + ((micros >> 31) & 1000000);
+}
+
+std::int64_t
+t_time::as_seconds() const {
+    if (m_storage < 0 && ((m_storage % 1000000) != 0)) {
+        return m_storage / 1000000 - 1;
+    }
+    return m_storage / 1000000;
+}
+
+std::string
+t_time::str(const struct tm& t) const {
+    std::stringstream ss;
+
+    double s = seconds(t) + microseconds() / 1000000.0;
+
+    ss << year(t) << "-" << str_(month(t)) << "-" << str_(day(t)) << " "
+       << str_(hours(t)) << ":" << str_(minutes(t)) << ":" << std::setfill('0')
+       << std::setw(6) << std::fixed << std::setprecision(3) << s;
+
+    return ss.str();
+}
+
+std::int32_t
+days_before_year(std::int32_t year) {
+    std::int32_t y = year - 1;
+    if (y >= 0) {
+        return y * 365 + y / 4 - y / 100 + y / 400;
+    }
+    return -366;
+}
+
+std::int32_t
+days_before_month(std::int32_t year, std::int32_t month) {
+    if (month < 1 || month > 12) {
+        return 0;
+    }
+    return __mon_yday[isleap(year)][month - 1];
+}
+
+std::int32_t
+ymd_to_ord(std::int32_t year, std::int32_t month, std::int32_t day) {
+    return days_before_year(year) + days_before_month(year, month) + day;
+}
+
+std::int64_t
+to_gmtime(
+    std::int32_t year,
+    std::int32_t month,
+    std::int32_t day,
+    std::int32_t hour,
+    std::int32_t min,
+    std::int32_t sec
+) {
+    static std::int32_t EPOCH_ORD = ymd_to_ord(1970, 1, 1);
+    std::int64_t days = ymd_to_ord(year, month, day) - EPOCH_ORD;
+    std::int64_t res = ((days * 24 + hour) * 60 + min) * 60 + sec;
+    return static_cast<std::int64_t>(res);
+}
+
+std::tm
+gmtime_from_epoch_ms(std::int64_t ms) {
+    std::tm out{};
+    std::int64_t secs = ms / 1000;
+    if (ms % 1000 < 0) {
+        secs -= 1;
+    }
+
+    t_time::gmtime(out, secs, 0);
+    return out;
+}
+
+t_time&
+t_time::operator+=(const t_tdelta& d) {
+    m_storage += d.v;
+    return *this;
+}
+
+t_time&
+t_time::operator-=(const t_tdelta& d) {
+    m_storage -= d.v;
+    return *this;
+}
+
+t_tdelta
+operator-(const t_time& a, const t_time& b) {
+    return {a.m_storage - b.m_storage};
+}
+
+} // end namespace perspective
+
+namespace std {
+std::ostream&
+operator<<(std::ostream& s, const perspective::t_tdelta& td) {
+    s << "t_tdelta(" << td.v << ")";
+    return s;
+}
+
+std::ostream&
+operator<<(std::ostream& os, const perspective::t_time& t) {
+    struct tm tstruct;
+    bool rcode = t.as_tm(tstruct);
+    if (rcode) {
+        os << "t_time<" << t.str(tstruct) << ">" << std::endl;
+    } else {
+        os << "t_time<" << t.raw_value() << ">" << std::endl;
+    }
+
+    return os;
+}
+} // namespace std

rust/perspective-server/cpp/perspective/src/cpp/tracing.cpp

@@ -0,0 +1,64 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/tracing.h>
+#include <perspective/compat.h>
+#include <sys/types.h>
+#include <sys/syscall.h>
+#include <sys/resource.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <sstream>
+#include <cxxabi.h>
+#include <fstream>
+#include <tsl/hopscotch_set.h>
+#include <execinfo.h>
+#include <cstring>
+
+#define THR_BUFFER_NELEMS 1000000
+#define THR_MAX_FUNCNAME_LEN 16000;
+
+static thread_local perspective::t_instrec th_trace_buffer[THR_BUFFER_NELEMS];
+static thread_local perspective::std::int32_t th_traceidx;
+static thread_local FILE* th_file;
+
+namespace perspective {
+t_trace::t_trace() { write_record(TRACE_TYPE_FNTRACE_BEGIN); }
+
+t_trace::~t_trace() { write_record(TRACE_TYPE_FNTRACE_END); }
+
+void
+t_trace::write_record(t_trace_type ttype) const {
+    if (th_traceidx == THR_BUFFER_NELEMS) {
+        flush_thbuffer(THR_BUFFER_NELEMS);
+        th_traceidx = 0;
+    }
+    t_instrec* ptr = th_trace_buffer + th_traceidx;
+    ptr = th_trace_buffer + th_traceidx;
+    ptr->m_trace_type = ttype;
+    ptr->m_time = psp_curtime();
+    ptr->t_fntrace.m_fn =
+        __builtin_extract_return_addr(__builtin_return_address(0));
+    ++th_traceidx;
+}
+
+t_uindex
+get_instrec_size() {
+    return sizeof(t_instrec);
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/tracing_impl_linux.cpp

@@ -0,0 +1,139 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __linux__
+#include <perspective/first.h>
+#include <perspective/tracing.h>
+#include <perspective/tracing_impl_linux.h>
+
+extern "C" {
+
+__attribute__((__constructor__)) void
+th_trace_init() {
+    using namespace std;
+    using namespace perspective;
+    if (!th_file) {
+        pid_t pid = getpid();
+        pid_t tid = syscall(__NR_gettid);
+        std::stringstream ss;
+        ss << "psptrace_" << pid << "_" << tid << ".trace";
+        th_file = fopen(ss.str().c_str(), "wb");
+        PSP_VERBOSE_ASSERT(th_file != 0, "Unable to open trace file");
+    } else {
+        PSP_COMPLAIN_AND_ABORT("Thread file unexpectedly non null");
+    }
+}
+
+__attribute__((__destructor__)) void
+th_trace_fini() {
+    using namespace std;
+    using namespace perspective;
+    flush_thbuffer(th_traceidx);
+    fclose(th_file);
+
+    pid_t pid = getpid();
+    pid_t tid = syscall(__NR_gettid);
+
+    std::stringstream sso;
+    sso << "psptrace_" << pid << "_" << tid << ".syms";
+    std::ofstream of(sso.str());
+
+    std::stringstream ssi;
+    ssi << "psptrace_" << pid << "_" << tid << ".trace";
+
+    int ifd = open(ssi.str().c_str(), O_RDONLY);
+    PSP_VERBOSE_ASSERT(ifd != -1, "Error opening file");
+
+    struct stat fb;
+    fstat(ifd, &fb);
+
+    uint64_t ifsize = fb.st_size;
+
+    if (ifsize == 0) {
+        return;
+    }
+
+    void* iptr = mmap(0, ifsize, PROT_READ, MAP_SHARED, ifd, 0);
+
+    PSP_VERBOSE_ASSERT(iptr != MAP_FAILED, "Error in mmap");
+    PSP_VERBOSE_ASSERT(
+        ifsize % sizeof(t_instrec) == 0, "Partial record encountered"
+    );
+
+    std::int64_t ndrecs = ifsize / sizeof(t_instrec);
+
+    t_instrec* irecs = static_cast<t_instrec*>(iptr);
+
+    tsl::hopscotch_set<void*> fptrs;
+
+    for (t_index idx = 0; idx < ndrecs; ++idx) {
+        t_instrec* irec = irecs + idx;
+        fptrs.emplace(irec->t_fntrace.m_fn);
+    }
+
+    for (tsl::hopscotch_set<void*>::const_iterator iter = fptrs.begin();
+         iter != fptrs.end();
+         ++iter) {
+
+        of << *iter << " ";
+        char** mangled;
+        void* code_arr[1];
+        code_arr[0] = *iter;
+        char** stack_strings = backtrace_symbols(code_arr, 1);
+
+        size_t sz = THR_MAX_FUNCNAME_LEN;
+
+        char* function = static_cast<char*>(malloc(sz));
+        char *begin = 0, *end = 0;
+        for (char* j = stack_strings[0]; *j; ++j) {
+            if (*j == '(') {
+                begin = j;
+            } else if (*j == '+') {
+                end = j;
+            }
+        }
+        if (begin && end) {
+
+            *begin++ = ' ';
+            *end = '\0';
+            int status;
+            char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
+            if (ret) {
+                function = ret;
+            } else {
+                std::strncpy(function, begin, sz);
+                std::strncat(function, "()", sz);
+                function[sz - 1] = ' ';
+            }
+
+            of << function << "\n";
+        } else {
+            of << stack_strings[0] << "\n";
+        }
+        free(function);
+        free(stack_strings);
+    }
+
+    close(ifd);
+    munmap(iptr, ifsize);
+}
+
+void
+flush_thbuffer(perspective::std::int32_t elemidx) {
+    using namespace std;
+    using namespace perspective;
+    t_instrec* mbuf = th_trace_buffer;
+    fwrite(mbuf, sizeof(t_instrec), elemidx, th_file);
+}
+} // end extern c
+
+#endif // end _WIN32

rust/perspective-server/cpp/perspective/src/cpp/tracing_impl_osx.cpp

@@ -0,0 +1,139 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#ifdef __APPLE__
+#include <perspective/first.h>
+#include <perspective/tracing.h>
+#include <perspective/tracing_impl_linux.h>
+
+extern "C" {
+
+__attribute__((__constructor__)) void
+th_trace_init() {
+    using namespace std;
+    using namespace perspective;
+    if (!th_file) {
+        pid_t pid = getpid();
+        pid_t tid = syscall(__NR_gettid);
+        std::stringstream ss;
+        ss << "psptrace_" << pid << "_" << tid << ".trace";
+        th_file = fopen(ss.str().c_str(), "wb");
+        PSP_VERBOSE_ASSERT(th_file != 0, "Unable to open trace file");
+    } else {
+        PSP_COMPLAIN_AND_ABORT("Thread file unexpectedly non null");
+    }
+}
+
+__attribute__((__destructor__)) void
+th_trace_fini() {
+    using namespace std;
+    using namespace perspective;
+    flush_thbuffer(th_traceidx);
+    fclose(th_file);
+
+    pid_t pid = getpid();
+    pid_t tid = syscall(__NR_gettid);
+
+    std::stringstream sso;
+    sso << "psptrace_" << pid << "_" << tid << ".syms";
+    std::ofstream of(sso.str());
+
+    std::stringstream ssi;
+    ssi << "psptrace_" << pid << "_" << tid << ".trace";
+
+    int ifd = open(ssi.str().c_str(), O_RDONLY);
+    PSP_VERBOSE_ASSERT(ifd != -1, "Error opening file");
+
+    struct stat fb;
+    fstat(ifd, &fb);
+
+    uint64_t ifsize = fb.st_size;
+
+    if (ifsize == 0) {
+        return;
+    }
+
+    void* iptr = mmap(0, ifsize, PROT_READ, MAP_SHARED, ifd, 0);
+
+    PSP_VERBOSE_ASSERT(iptr != MAP_FAILED, "Error in mmap");
+    PSP_VERBOSE_ASSERT(
+        ifsize % sizeof(t_instrec) == 0, "Partial record encountered"
+    );
+
+    std::int64_t ndrecs = ifsize / sizeof(t_instrec);
+
+    t_instrec* irecs = static_cast<t_instrec*>(iptr);
+
+    tsl::hopscotch_set<void*> fptrs;
+
+    for (t_index idx = 0; idx < ndrecs; ++idx) {
+        t_instrec* irec = irecs + idx;
+        fptrs.emplace(irec->t_fntrace.m_fn);
+    }
+
+    for (tsl::hopscotch_set<void*>::const_iterator iter = fptrs.begin();
+         iter != fptrs.end();
+         ++iter) {
+
+        of << *iter << " ";
+        char** mangled;
+        void* code_arr[1];
+        code_arr[0] = *iter;
+        char** stack_strings = backtrace_symbols(code_arr, 1);
+
+        size_t sz = THR_MAX_FUNCNAME_LEN;
+
+        char* function = static_cast<char*>(malloc(sz));
+        char *begin = 0, *end = 0;
+        for (char* j = stack_strings[0]; *j; ++j) {
+            if (*j == '(') {
+                begin = j;
+            } else if (*j == '+') {
+                end = j;
+            }
+        }
+        if (begin && end) {
+
+            *begin++ = ' ';
+            *end = '\0';
+            int status;
+            char* ret = abi::__cxa_demangle(begin, function, &sz, &status);
+            if (ret) {
+                function = ret;
+            } else {
+                std::strncpy(function, begin, sz);
+                std::strncat(function, "()", sz);
+                function[sz - 1] = ' ';
+            }
+
+            of << function << "\n";
+        } else {
+            of << stack_strings[0] << "\n";
+        }
+        free(function);
+        free(stack_strings);
+    }
+
+    close(ifd);
+    munmap(iptr, ifsize);
+}
+
+void
+flush_thbuffer(perspective::std::int32_t elemidx) {
+    using namespace std;
+    using namespace perspective;
+    t_instrec* mbuf = th_trace_buffer;
+    fwrite(mbuf, sizeof(t_instrec), elemidx, th_file);
+}
+} // end extern c
+
+#endif // end _WIN32

rust/perspective-server/cpp/perspective/src/cpp/traversal.cpp

@@ -0,0 +1,911 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/traversal.h>
+#include <perspective/sparse_tree.h>
+#include <perspective/arg_sort.h>
+#include <perspective/sort_specification.h>
+#include <tsl/hopscotch_set.h>
+
+namespace perspective {
+
+t_vdnode::t_vdnode() : m_expanded(0), m_depth(INVALID_INDEX) {}
+
+t_vdnode::t_vdnode(bool expanded, t_depth depth) :
+    m_expanded(static_cast<t_index>(expanded)),
+    m_depth(depth) {}
+
+t_vdnode::t_vdnode(t_index expanded, t_depth depth) :
+    m_expanded(expanded),
+    m_depth(depth) {}
+
+t_vdnode::t_vdnode(bool expanded, bool has_children) :
+    m_expanded(static_cast<t_index>(expanded)),
+    m_has_children(has_children) {}
+
+t_traversal::t_traversal(const std::shared_ptr<const t_stree>& tree) :
+    m_tree(tree) {
+    t_stnode_vec rchildren;
+    tree->get_child_nodes(0, rchildren);
+    populate_root_children(rchildren);
+}
+
+void
+t_traversal::populate_root_children(const t_stnode_vec& rchildren) {
+    m_nodes = std::make_shared<std::vector<t_tvnode>>(rchildren.size() + 1);
+
+    // Initialize root
+    (*m_nodes)[0].m_expanded = true;
+    (*m_nodes)[0].m_depth = 0;
+    (*m_nodes)[0].m_rel_pidx = INVALID_INDEX;
+    (*m_nodes)[0].m_tnid = 0;
+    (*m_nodes)[0].m_ndesc = rchildren.size();
+    (*m_nodes)[0].m_nchild = rchildren.size();
+
+    t_index count = 1;
+
+    for (const auto& iter : rchildren) {
+        t_tvnode& cnode = (*m_nodes)[count];
+        cnode.m_expanded = false;
+        cnode.m_depth = 1;
+        cnode.m_rel_pidx = count;
+        cnode.m_tnid = iter.m_idx;
+        cnode.m_ndesc = 0;
+        cnode.m_nchild = 0;
+        count += 1;
+    }
+}
+
+void
+t_traversal::populate_root_children(const std::shared_ptr<const t_stree>& tree
+) {
+    t_stnode_vec rchildren;
+    tree->get_child_nodes(0, rchildren);
+    populate_root_children(rchildren);
+}
+
+t_index
+t_traversal::expand_node(t_index exp_idx) {
+    if (m_leaves_only) {
+        return 0;
+    }
+
+    t_tvnode& exp_tvnode = (*m_nodes)[exp_idx];
+
+    if (exp_tvnode.m_expanded) {
+        return 0;
+    }
+
+    t_stnode_vec tchildren;
+    m_tree->get_child_nodes(exp_tvnode.m_tnid, tchildren);
+    t_index n_changed = tchildren.size();
+    std::vector<t_tvnode> children = std::vector<t_tvnode>(n_changed);
+
+    t_index count = 0;
+    for (const auto& iter : tchildren) {
+        t_tvnode& tv_node = children[count];
+        tv_node.m_expanded = false;
+        tv_node.m_depth = exp_tvnode.m_depth + 1;
+        tv_node.m_rel_pidx = count + 1;
+        tv_node.m_tnid = iter.m_idx;
+        tv_node.m_ndesc = 0;
+        tv_node.m_nchild = 0;
+        count += 1;
+    }
+
+    // Update node being expanded
+    exp_tvnode.m_expanded = !tchildren.empty();
+    ;
+    exp_tvnode.m_ndesc += n_changed;
+    exp_tvnode.m_nchild = n_changed;
+
+    // insert children of node into the traversal
+    m_nodes->insert(
+        m_nodes->begin() + exp_idx + 1, children.begin(), children.end()
+    );
+
+    // update ancestors about their new descendents
+    update_ancestors(exp_idx, n_changed);
+    update_sucessors(exp_idx, n_changed);
+
+    return n_changed;
+}
+
+t_index
+t_traversal::expand_node(
+    const std::vector<t_sortspec>& sortby, t_index exp_idx, t_ctx2* ctx2
+) {
+    if (m_leaves_only) {
+        return 0;
+    }
+
+    t_tvnode& exp_tvnode = (*m_nodes)[exp_idx];
+
+    if (exp_tvnode.m_expanded) {
+        return 0;
+    }
+
+    t_stnode_vec tchildren;
+    m_tree->get_child_nodes(exp_tvnode.m_tnid, tchildren);
+    t_index n_changed = tchildren.size();
+    t_index count = 0;
+    std::vector<t_index> sorted_idx(n_changed);
+
+    std::vector<t_index> sortby_agg_indices(sortby.size());
+    t_uindex scount = 0;
+    for (const auto& s : sortby) {
+        sortby_agg_indices[scount] = s.m_agg_index;
+        ++scount;
+    }
+
+    if (!sortby.empty()) {
+        auto sortelems = std::make_shared<std::vector<t_mselem>>(
+            static_cast<size_t>(n_changed)
+        );
+        t_index num_aggs = sortby.size();
+        std::vector<t_tscalar> aggregates(num_aggs);
+
+        t_uindex child_idx = 0;
+        for (const auto& iter : tchildren) {
+            m_tree->get_aggregates_for_sorting(
+                iter.m_idx, sortby_agg_indices, aggregates, ctx2
+            );
+            (*sortelems)[count] = t_mselem(aggregates, child_idx);
+            ++count;
+            ++child_idx;
+        }
+
+        std::vector<t_sorttype> sort_orders = get_sort_orders(sortby);
+        t_multisorter sorter(sortelems, sort_orders);
+        argsort(sorted_idx, sorter);
+    } else {
+        for (t_index i = 0, loop_end = sorted_idx.size(); i != loop_end; ++i) {
+            sorted_idx[i] = i;
+        }
+    }
+
+    std::vector<t_tvnode> children = std::vector<t_tvnode>(n_changed);
+    count = 0;
+    for (long long idx : sorted_idx) {
+        t_tvnode& tv_node = children[count];
+        tv_node.m_expanded = false;
+        tv_node.m_depth = exp_tvnode.m_depth + 1;
+        tv_node.m_rel_pidx = count + 1;
+        tv_node.m_tnid = tchildren[idx].m_idx;
+        tv_node.m_ndesc = 0;
+        tv_node.m_nchild = 0;
+        count += 1;
+    }
+
+    // Update node being expanded
+    exp_tvnode.m_expanded = !sorted_idx.empty();
+    exp_tvnode.m_ndesc += n_changed;
+    exp_tvnode.m_nchild = n_changed;
+
+    // insert children of node into the traversal
+    m_nodes->insert(
+        m_nodes->begin() + exp_idx + 1, children.begin(), children.end()
+    );
+
+    // update ancestors about their new descendents
+    update_ancestors(exp_idx, n_changed);
+    update_sucessors(exp_idx, n_changed);
+
+    return n_changed;
+}
+
+t_index
+t_traversal::collapse_node(t_index idx) {
+    if (m_leaves_only) {
+        return 0;
+    }
+
+    t_tvnode& node = (*m_nodes)[idx];
+
+    if (!node.m_expanded) {
+        return 0;
+    }
+
+    // Calculate span of descendents
+    t_index n_changed = node.m_ndesc;
+
+    t_index bidx = idx + 1;
+    t_index eidx = bidx + n_changed;
+
+    // remove entries from traversal
+    m_nodes->erase(m_nodes->begin() + bidx, m_nodes->begin() + eidx);
+
+    // Update node being collapsed
+    node.m_expanded = false;
+    node.m_ndesc -= n_changed;
+    node.m_nchild = 0;
+
+    // update ancestors about removal of their
+    // descendents
+    update_ancestors(idx, -n_changed);
+    update_sucessors(idx, -n_changed);
+
+    return n_changed;
+}
+
+void
+t_traversal::add_node(
+    const std::vector<t_sortspec>& sortby,
+    const std::vector<t_uindex>& indices,
+    t_index insert_level_idx,
+    t_ctx2* ctx2
+) {
+    std::vector<t_index> tv_indices;
+    t_index collapsed_ancestor = INVALID_INDEX;
+
+    get_expanded_span(
+        indices, tv_indices, collapsed_ancestor, insert_level_idx
+    );
+
+    if (static_cast<t_index>(tv_indices.size()) == insert_level_idx) {
+        t_index p_tvidx = tv_indices.back();
+        const t_tvnode& p_tvnode = (*m_nodes)[p_tvidx];
+        t_index p_ptidx = p_tvnode.m_tnid;
+        t_index p_nchild = p_tvnode.m_nchild + 1;
+        t_index c_ptidx = indices[insert_level_idx];
+        t_uindex cidx = m_tree->get_sibling_idx(p_ptidx, p_nchild, c_ptidx);
+        cidx = std::min(p_tvnode.m_nchild, cidx);
+        t_index cur_cidx = p_tvidx + 1;
+        for (t_uindex idx = 0; idx < cidx; ++idx) {
+            cur_cidx += (1 + (*m_nodes)[cur_cidx].m_ndesc);
+        }
+
+        (*m_nodes)[p_tvidx].m_nchild += 1;
+
+        t_depth depth = get_depth(p_tvidx) + 1;
+        t_tvnode new_node;
+        fill_travnode(&new_node, false, depth, cur_cidx - p_tvidx, 0, c_ptidx);
+        auto insert_iter = m_nodes->begin() + cur_cidx;
+        m_nodes->insert(insert_iter, new_node);
+        update_ancestors(cur_cidx, 1);
+        update_sucessors(cur_cidx, 1);
+    }
+}
+
+t_index
+t_traversal::update_ancestors(t_index nidx, t_index n_changed) {
+    if (nidx == 0) {
+        return 0;
+    }
+
+    t_index pidx = nidx - (*m_nodes)[nidx].m_rel_pidx;
+    while (pidx > INVALID_INDEX) {
+        t_tvnode& node = (*m_nodes)[pidx];
+        node.m_ndesc += n_changed;
+        if (pidx == 0) {
+            pidx = INVALID_INDEX;
+        } else {
+            pidx = pidx - node.m_rel_pidx;
+        }
+    }
+    return 0;
+}
+
+t_index
+t_traversal::update_sucessors(t_index nidx, t_index n_changed) {
+    t_tvnode* c_node = &((*m_nodes)[nidx]);
+
+    while (c_node->m_depth > 0) {
+        t_index pidx = nidx - c_node->m_rel_pidx;
+
+        const t_tvnode& p_node = (*m_nodes)[pidx];
+
+        t_index p_nchild = p_node.m_nchild;
+        t_index coffset = 1;
+
+        for (int i = 0; i < p_nchild; i++) {
+            t_index curr_cidx = pidx + coffset;
+            t_tvnode& child_node = (*m_nodes)[curr_cidx];
+            if (curr_cidx > nidx) {
+                child_node.m_rel_pidx += n_changed;
+            }
+            if (child_node.m_expanded) {
+                coffset = coffset + child_node.m_ndesc + 1;
+            } else {
+                coffset += 1;
+            }
+        }
+
+        nidx = pidx;
+        c_node = &((*m_nodes)[nidx]);
+    }
+    return 0;
+}
+
+t_index
+t_traversal::get_tree_index(t_index idx) const {
+    return (*m_nodes)[idx].m_tnid;
+}
+
+t_uindex
+t_traversal::size() const {
+    return m_nodes->size();
+}
+
+t_depth
+t_traversal::get_depth(t_index idx) const {
+    return (*m_nodes)[idx].m_depth;
+}
+
+t_index
+t_traversal::get_traversal_index(t_index idx) {
+    t_index rval = INVALID_INDEX;
+
+    for (t_index i = 0, loop_end = m_nodes->size(); i < loop_end; ++i) {
+        if ((*m_nodes)[i].m_tnid == idx) {
+            rval = i;
+            break;
+        }
+    }
+    return rval;
+}
+
+std::vector<t_vdnode>
+t_traversal::get_view_nodes(t_index bidx, t_index eidx) const {
+    std::vector<t_vdnode> vec(eidx - bidx);
+    for (t_index i = bidx; i < eidx; i++) {
+        t_index idx = i - bidx;
+        const t_tvnode& tv_node = (*m_nodes)[i];
+        vec[idx].m_expanded = static_cast<t_index>(tv_node.m_expanded);
+        vec[idx].m_depth = tv_node.m_depth;
+        t_index tree_idx = get_tree_index(i);
+        vec[idx].m_has_children = m_tree->get_num_children(tree_idx) > 0;
+    }
+    return vec;
+}
+
+void
+t_traversal::get_expanded_span(
+    const std::vector<t_uindex>& in_ptidxes,
+    std::vector<t_index>& out_indexes,
+    t_index& out_collpsed_ancestor,
+    t_index insert_level_idx
+) {
+    t_index pidx = 0;
+    t_index coffset = 1;
+
+    out_indexes.push_back(0);
+
+    for (t_index counter = 1, loop_end = in_ptidxes.size(); counter < loop_end;
+         counter++) {
+        bool level_node_found = false;
+        t_index level_idx = INVALID_INDEX;
+        t_index p_nchild = (*m_nodes)[pidx].m_nchild;
+
+        if (counter >= insert_level_idx) {
+            p_nchild = p_nchild - 1;
+        }
+
+        for (t_index cidx = 0; cidx < p_nchild; ++cidx) {
+            t_tvnode& cnode = (*m_nodes)[pidx + coffset];
+
+            if (static_cast<t_uindex>(cnode.m_tnid) == in_ptidxes[counter]) {
+                level_node_found = true;
+                level_idx = pidx + coffset;
+
+                if (cnode.m_expanded) {
+                    pidx = pidx + coffset;
+                    coffset = 1;
+                    p_nchild = (*m_nodes)[pidx].m_nchild;
+                    out_indexes.push_back(pidx);
+                    break;
+                }
+            } else {
+                // TODO: next line
+                coffset = coffset + cnode.m_ndesc + 1;
+            }
+        }
+
+        if (level_node_found && (!((*m_nodes)[level_idx].m_expanded))) {
+            out_collpsed_ancestor = level_idx;
+            break;
+        }
+
+        if (!level_node_found) {
+            break;
+        }
+    }
+}
+
+bool
+t_traversal::validate_cells(
+    const std::vector<std::pair<t_uindex, t_uindex>>& cells
+) const {
+    t_uindex trav_size = size();
+
+    for (const auto& cell : cells) {
+        auto r_tvidx = cell.first;
+        if (r_tvidx >= trav_size) {
+            return false;
+        }
+    }
+    return true;
+}
+
+t_index
+t_traversal::remove_subtree(t_index idx) {
+    t_tvnode& node = (*m_nodes)[idx];
+
+    // Calculate span of descendents
+    t_index n_changed = node.m_ndesc + 1;
+
+    t_index bidx = idx;
+    t_index eidx = bidx + n_changed;
+
+    update_sucessors(idx, -n_changed);
+
+    // update ancestors about removal of their
+    // descendents
+    update_ancestors(idx, -n_changed);
+
+    t_index pidx = idx - node.m_rel_pidx;
+    (*m_nodes)[pidx].m_nchild -= 1;
+
+    // remove entries from traversal
+    m_nodes->erase(m_nodes->begin() + bidx, m_nodes->begin() + eidx);
+
+    return n_changed;
+}
+
+void
+t_traversal::pprint() const {
+    for (t_index idx = 0, loop_end = m_nodes->size(); idx < loop_end; ++idx) {
+        const t_tvnode& node = (*m_nodes)[idx];
+        const t_stnode tnode = m_tree->get_node(node.m_tnid);
+        for (t_uindex didx = 0; didx < node.m_depth; didx++) {
+            std::cout << "\t";
+        }
+        std::cout << "tvidx: " << idx << " value: " << tnode.m_value
+                  << " depth: " << node.m_depth
+                  << " m_rel_pidx: " << node.m_rel_pidx
+                  << " ndesc: " << node.m_ndesc << " tnid: " << node.m_tnid
+                  << " nchild: " << node.m_nchild << '\n';
+    }
+}
+
+t_tvnode
+t_traversal::get_node(t_index idx) const {
+    return (*m_nodes)[idx];
+}
+
+void
+t_traversal::get_leaves(std::vector<t_index>& out_data) const {
+    for (t_index curidx = 0, loop_end = m_nodes->size(); curidx < loop_end;
+         ++curidx) {
+        if (!(*m_nodes)[curidx].m_expanded) {
+            out_data.push_back(curidx);
+        }
+    }
+}
+
+void
+t_traversal::get_child_indices(
+    t_index nidx, std::vector<std::pair<t_index, t_index>>& out_data
+) const {
+    const t_tvnode& tvnode = (*m_nodes)[nidx];
+    t_index nchild = tvnode.m_nchild;
+    t_index coffset = 1;
+
+    for (int i = 0; i < nchild; i++) {
+        t_index curr_cidx = nidx + coffset;
+        const t_tvnode& child_node = (*m_nodes)[curr_cidx];
+        out_data.emplace_back(curr_cidx, child_node.m_tnid);
+        coffset = coffset + child_node.m_ndesc + 1;
+    }
+}
+
+void
+t_traversal::print_stats() {
+    std::cout << "Traversal size => " << m_nodes->size() << '\n';
+}
+
+t_index
+t_traversal::get_num_tree_leaves(t_index idx) const {
+    const t_tvnode& node = (*m_nodes)[idx];
+
+    t_index rval = 0;
+
+    for (t_index curidx = idx + 1, loop_end = idx + node.m_ndesc + 1;
+         curidx < loop_end;
+         ++curidx) {
+        if (!(*m_nodes)[curidx].m_expanded) {
+            ++rval;
+        }
+    }
+
+    return rval;
+}
+
+void
+t_traversal::post_order(t_index nidx, std::vector<t_index>& out_vec) {
+    std::vector<std::pair<t_index, t_index>> children;
+    get_child_indices(nidx, children);
+
+    for (auto& idx : children) {
+        post_order(idx.first, out_vec);
+    }
+
+    out_vec.push_back(nidx);
+}
+
+// Traversal
+t_index
+t_traversal::set_depth(
+    const std::vector<t_sortspec>& sortby, t_depth depth, t_ctx2* ctx2
+) {
+    std::vector<t_index> pending;
+    depth = depth + 1;
+    pending.push_back(0);
+    t_index n_changed = 0;
+    while (!pending.empty()) {
+        t_index curidx = pending.back();
+        pending.pop_back();
+        n_changed += expand_node(sortby, curidx, ctx2);
+        std::vector<std::pair<t_index, t_index>> children;
+        get_child_indices(curidx, children);
+        std::vector<t_index> collapse;
+        for (const auto& child : children) {
+            const t_tvnode& tv_node = (*m_nodes)[child.first];
+
+            if (tv_node.m_depth < depth) {
+                pending.push_back(child.first);
+            } else if (tv_node.m_depth == depth && tv_node.m_expanded) {
+                collapse.push_back(child.first);
+            }
+        }
+        // Now collapse any children
+        for (auto rit = collapse.rbegin(); rit != collapse.rend(); ++rit) {
+            t_index curidx = *rit;
+            n_changed += collapse_node(curidx);
+        }
+    }
+
+    return n_changed;
+}
+
+std::vector<t_ftreenode>
+t_traversal::get_flattened_tree(t_index idx, t_depth stop_depth) const {
+    std::queue<t_index> queue;
+    queue.push(idx);
+    std::vector<t_ftreenode> rvec;
+    t_index rvec_idx = 1;
+    while (!queue.empty()) {
+        t_index hidx = queue.front();
+        queue.pop();
+        const t_tvnode& c_node = (*m_nodes)[hidx];
+        t_depth curdepth = c_node.m_depth;
+        t_ftreenode rnode;
+        rnode.m_idx = c_node.m_tnid;
+        if (curdepth < stop_depth) {
+            t_index nchild = m_tree->get_num_children(rnode.m_idx);
+            rnode.m_fcidx = rvec_idx;
+            rnode.m_nchild = nchild;
+            rnode.m_depth = c_node.m_depth;
+            t_index curr_cidx = hidx + 1;
+            std::vector<t_index> children(nchild);
+            for (int cidx = 0; cidx < nchild; cidx++) {
+                const t_tvnode& child_node = (*m_nodes)[curr_cidx];
+                children[cidx] = curr_cidx;
+                if (child_node.m_expanded) {
+                    curr_cidx = curr_cidx + child_node.m_ndesc + 1;
+                } else {
+                    curr_cidx += 1;
+                }
+                rvec_idx++;
+            }
+            for (long long cidx : children) {
+                queue.push(cidx);
+            }
+        } else {
+            rnode.m_fcidx = 0;
+            rnode.m_nchild = 0;
+            rnode.m_depth = INVALID_INDEX;
+        }
+        rvec.push_back(rnode);
+    }
+    return rvec;
+}
+
+t_index
+t_traversal::tree_index_lookup(t_index idx, t_index bidx) const {
+    t_index tvidx = INVALID_INDEX;
+    for (t_index i = bidx, loop_end = m_nodes->size(); i < loop_end; ++i) {
+        if ((*m_nodes)[i].m_tnid == idx) {
+            tvidx = i;
+            break;
+        }
+    }
+    return tvidx;
+}
+
+void
+t_traversal::get_node_ancestors(t_index nidx, std::vector<t_index>& ancestors)
+    const {
+    if (nidx == 0) {
+        return;
+    }
+
+    t_index pidx = nidx - (*m_nodes)[nidx].m_rel_pidx;
+    while (pidx > INVALID_INDEX) {
+        ancestors.push_back(pidx);
+        const t_tvnode& node = (*m_nodes)[pidx];
+        if (pidx == 0) {
+            pidx = INVALID_INDEX;
+        } else {
+            pidx = pidx - node.m_rel_pidx;
+        }
+    }
+}
+
+void
+t_traversal::get_expanded(std::vector<t_index>& expanded_tidx) const {
+    // Ancestors of expanded nodes
+    std::set<t_index> ancestors;
+    std::vector<t_index> expanded;
+
+    if (m_nodes->empty()) {
+        return;
+    }
+
+    for (t_index i = m_nodes->size() - 1; i > -1; i--) {
+        const t_tvnode& node = (*m_nodes)[i];
+
+        if (node.m_expanded && ancestors.find(i) == ancestors.end()) {
+            expanded.push_back(i);
+            std::vector<t_index> node_ancestors;
+            get_node_ancestors(i, node_ancestors);
+            ancestors.insert(node_ancestors.begin(), node_ancestors.end());
+        }
+    }
+
+    std::vector<t_index> rval(expanded.size());
+
+    for (t_index i = 0, loop_end = rval.size(); i < loop_end; i++) {
+        const t_tvnode& node = (*m_nodes)[expanded[i]];
+        rval[i] = node.m_tnid;
+    }
+
+    std::swap(rval, expanded_tidx);
+}
+
+void
+t_traversal::drop_tree_indices(const std::vector<t_uindex>& indices) {
+    if (indices.empty() || m_nodes->empty()) {
+        return;
+    }
+
+    auto& nodes = *m_nodes;
+    const t_index n = static_cast<t_index>(nodes.size());
+
+    // The previous implementation removed each dropped strand individually: a
+    // linear `tree_index_lookup` (O(N)) plus a mid-vector `erase` (O(N)) and a
+    // per-op `update_sucessors`/`update_ancestors` fix-up, i.e. O(D*N) for D
+    // drops. Instead, mark every dropped subtree and rebuild the pre-order node
+    // vector in a single O(N + D) pass, recomputing the relative-parent
+    // (m_rel_pidx), descendant (m_ndesc) and child (m_nchild) encoding in bulk.
+    // This mirrors the one-pass rebuild `sort_by` already uses.
+    tsl::hopscotch_set<t_index> drop_set;
+    drop_set.reserve(indices.size());
+    for (auto idx : indices) {
+        drop_set.insert(static_cast<t_index>(idx));
+    }
+
+    // `remap[i]` = number of survivors in [0, i), which is also the new index of
+    // the survivor at old index `i`. A node is dead iff it lies within the
+    // subtree span of any dropped node; because the traversal is pre-order and
+    // dropped subtrees are nested-or-disjoint, a single non-decreasing "dead
+    // watermark" (the last index known to be inside a dropped subtree)
+    // classifies every node in one linear scan. `m_tnid` is unique per traversal
+    // node, so set membership matches what the old per-strand lookup found.
+    std::vector<t_index> remap(n + 1);
+    t_index surv = 0;
+    t_index dead_until = INVALID_INDEX;
+    for (t_index i = 0; i < n; ++i) {
+        if (drop_set.find(nodes[i].m_tnid) != drop_set.end()) {
+            const t_index end = i + static_cast<t_index>(nodes[i].m_ndesc);
+            if (end > dead_until) {
+                dead_until = end;
+            }
+        }
+        remap[i] = surv;
+        if (i > dead_until) {
+            ++surv;
+        }
+    }
+    remap[n] = surv;
+
+    // The root (tnid 0, depth 0) is never a strand and so is never dropped, so
+    // at least it must survive. A future caller dropping the root would empty
+    // the traversal and OOB downstream consumers (e.g. get_expanded_span reads
+    // node 0), so assert the invariant here.
+    PSP_VERBOSE_ASSERT(surv >= 1, "drop_tree_indices dropped the root node");
+
+    if (surv == n) {
+        // No dropped strand was actually present in this traversal.
+        return;
+    }
+
+    // Compact survivors into a fresh pre-order vector, recomputing the relative
+    // encoding from the survivor prefix-sum.
+    std::vector<t_tvnode> new_nodes(surv);
+    for (t_index old_idx = 0; old_idx < n; ++old_idx) {
+        // Alive iff a survivor was counted at this index.
+        if ((remap[old_idx + 1] - remap[old_idx]) != 1) {
+            continue;
+        }
+
+        const t_index new_idx = remap[old_idx];
+        t_tvnode node = nodes[old_idx];
+
+        // A surviving node's descendants stay contiguous immediately after it,
+        // so count the survivors over its original subtree span
+        // [old_idx, old_end] and subtract the node itself.
+        const t_index old_end = old_idx + static_cast<t_index>(node.m_ndesc);
+        node.m_ndesc =
+            static_cast<t_uindex>(remap[old_end + 1] - remap[old_idx] - 1);
+
+        // A survivor's parent always survives (only whole subtrees are dropped),
+        // so its new parent index is well-defined.
+        if (node.m_depth > 0) {
+            const t_index old_parent = old_idx - node.m_rel_pidx;
+            node.m_rel_pidx = new_idx - remap[old_parent];
+        }
+
+        node.m_nchild = 0; // recomputed in the next pass
+        new_nodes[new_idx] = node;
+    }
+
+    // Recompute direct-child counts: each surviving non-root node bumps its
+    // (surviving) parent's count.
+    for (t_index new_idx = 0; new_idx < surv; ++new_idx) {
+        const t_tvnode& node = new_nodes[new_idx];
+        if (node.m_depth > 0) {
+            new_nodes[new_idx - node.m_rel_pidx].m_nchild += 1;
+        }
+    }
+
+    std::swap(nodes, new_nodes);
+}
+
+bool
+t_traversal::is_valid_idx(t_index idx) const {
+    return idx >= 0 && idx < t_index(size());
+}
+
+const t_stree*
+t_traversal::get_tree() const {
+    return m_tree.get();
+}
+
+bool
+t_traversal::get_node_expanded(t_index idx) const {
+    if (idx < 0 || static_cast<t_uindex>(idx) > m_nodes->size()) {
+        return false;
+    }
+    return m_nodes->at(idx).m_expanded;
+}
+
+void
+t_traversal::set_leaves_only(bool enabled, t_uindex leaf_depth) {
+    m_leaves_only = enabled;
+    m_leaf_depth = leaf_depth;
+    if (m_leaves_only) {
+        rebuild_from_leaves({});
+    }
+}
+
+bool
+t_traversal::is_leaves_only() const {
+    return m_leaves_only;
+}
+
+void
+t_traversal::collect_leaves(
+    t_uindex tnid,
+    t_uindex current_depth,
+    const std::vector<t_sortspec>& sortby,
+    const std::vector<t_index>& sortby_agg_indices,
+    const std::vector<t_sorttype>& sort_orders
+) {
+    if (current_depth >= m_leaf_depth) {
+        t_tvnode node;
+        node.m_expanded = false;
+        node.m_depth = m_leaf_depth;
+        node.m_rel_pidx = 0;
+        node.m_ndesc = 0;
+        node.m_nchild = 0;
+        node.m_tnid = tnid;
+        m_nodes->push_back(node);
+        return;
+    }
+
+    t_stnode_vec children;
+    m_tree->get_child_nodes(tnid, children);
+
+    if (!sortby.empty() && children.size() > 1) {
+        auto n_children = children.size();
+        auto sortelems =
+            std::make_shared<std::vector<t_mselem>>(n_children);
+        std::vector<t_tscalar> aggregates(sortby.size());
+
+        for (t_uindex i = 0; i < n_children; ++i) {
+            m_tree->get_aggregates_for_sorting(
+                children[i].m_idx, sortby_agg_indices, aggregates, nullptr
+            );
+            (*sortelems)[i] = t_mselem(aggregates, i);
+        }
+
+        std::vector<t_index> sorted_idx(n_children);
+        t_multisorter sorter(sortelems, sort_orders);
+        argsort(sorted_idx, sorter);
+
+        t_stnode_vec sorted_children(n_children);
+        for (t_uindex i = 0; i < n_children; ++i) {
+            sorted_children[i] = children[sorted_idx[i]];
+        }
+        children = std::move(sorted_children);
+    }
+
+    for (const auto& child : children) {
+        collect_leaves(
+            child.m_idx, current_depth + 1, sortby, sortby_agg_indices,
+            sort_orders
+        );
+    }
+}
+
+void
+t_traversal::rebuild_from_leaves(const std::vector<t_sortspec>& sortby) {
+    m_nodes = std::make_shared<std::vector<t_tvnode>>();
+
+    std::vector<t_index> sortby_agg_indices(sortby.size());
+    for (t_uindex i = 0; i < sortby.size(); ++i) {
+        sortby_agg_indices[i] = sortby[i].m_agg_index;
+    }
+
+    std::vector<t_sorttype> sort_orders = get_sort_orders(sortby);
+    collect_leaves(0, 0, sortby, sortby_agg_indices, sort_orders);
+}
+
+void
+t_traversal::set_total_only(bool enabled) {
+    m_total_only = enabled;
+    if (m_total_only) {
+        rebuild_for_total();
+    }
+}
+
+bool
+t_traversal::is_total_only() const {
+    return m_total_only;
+}
+
+void
+t_traversal::rebuild_for_total() {
+    m_nodes = std::make_shared<std::vector<t_tvnode>>();
+    t_tvnode node;
+    node.m_expanded = false;
+    node.m_depth = 0;
+    node.m_rel_pidx = 0;
+    node.m_ndesc = 0;
+    node.m_nchild = 0;
+    node.m_tnid = 0;
+    m_nodes->push_back(node);
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/traversal_nodes.cpp

@@ -0,0 +1,34 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/traversal_nodes.h>
+
+namespace perspective {
+
+void
+fill_travnode(
+    t_tvnode* node,
+    bool expanded,
+    t_uindex depth,
+    t_uindex rel_pidx,
+    t_uindex ndesc,
+    t_uindex tnid
+) {
+    node->m_expanded = expanded;
+    node->m_depth = depth;
+    node->m_rel_pidx = rel_pidx;
+    node->m_ndesc = ndesc;
+    node->m_tnid = tnid;
+    node->m_nchild = 0;
+}
+}; // namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/tree_context_common.cpp

@@ -0,0 +1,287 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/base.h>
+#include <perspective/filter.h>
+#include <perspective/path.h>
+#include <perspective/sparse_tree.h>
+#include <perspective/data_table.h>
+#include <perspective/traversal.h>
+#include <perspective/env_vars.h>
+#include <perspective/dense_tree.h>
+#include <perspective/dense_tree_context.h>
+#include <tsl/hopscotch_set.h>
+
+#include <utility>
+
+namespace perspective {
+
+void
+notify_sparse_tree_common(
+    const std::shared_ptr<t_data_table>& strands,
+    const std::shared_ptr<t_data_table>& strand_deltas,
+    const std::shared_ptr<t_stree>& tree,
+    const std::shared_ptr<t_traversal>& traversal,
+    bool process_traversal,
+    const std::vector<t_aggspec>& aggregates,
+    const std::vector<std::pair<std::string, std::string>>& tree_sortby,
+    const std::vector<t_sortspec>& ctx_sortby,
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table
+) {
+    t_filter fltr;
+    if (t_env::log_data_nsparse_strands()) {
+        std::cout << "nsparse_strands" << '\n';
+        strands->pprint();
+    }
+
+    if (t_env::log_data_nsparse_strand_deltas()) {
+        std::cout << "nsparse_strand_deltas" << '\n';
+        strand_deltas->pprint();
+    }
+
+    const auto& pivots = tree->get_pivots();
+
+    t_dtree dtree(strands, pivots, tree_sortby);
+    dtree.init();
+
+    dtree.check_pivot(fltr, pivots.size() + 1);
+
+    if (t_env::log_data_nsparse_dtree()) {
+        std::cout << "nsparse_dtree" << '\n';
+        dtree.pprint(fltr);
+    }
+
+    t_dtree_ctx dctx(strands, strand_deltas, dtree, aggregates);
+
+    dctx.init();
+
+    tree->update_shape_from_static(dctx);
+
+    auto zero_strands = tree->zero_strands();
+
+    bool is_leaves_only =
+        process_traversal && traversal != nullptr && traversal->is_leaves_only();
+    bool is_total_only =
+        process_traversal && traversal != nullptr && traversal->is_total_only();
+
+    if (process_traversal && !is_leaves_only && !is_total_only) {
+        t_uindex t_osize = traversal->size();
+        traversal->drop_tree_indices(zero_strands);
+        t_uindex t_nsize = traversal->size();
+        if (t_osize != t_nsize) {
+            tree->set_has_deltas(true);
+        }
+    }
+
+    auto non_zero_ids = tree->non_zero_ids(zero_strands);
+    auto non_zero_leaves = tree->non_zero_leaves(zero_strands);
+
+    tree->drop_zero_strands();
+
+    tree->populate_leaf_index(non_zero_leaves);
+
+    tree->update_aggs_from_static(dctx, gstate, expression_master_table);
+
+    if (is_total_only) {
+        traversal->rebuild_for_total();
+    } else if (is_leaves_only) {
+        traversal->rebuild_from_leaves(ctx_sortby);
+    } else {
+        std::set<t_uindex> visited;
+
+        struct t_leaf_path {
+            std::vector<t_tscalar> m_path;
+            t_uindex m_lfidx;
+        };
+
+        std::vector<t_leaf_path> leaf_paths(non_zero_leaves.size());
+
+        t_uindex count = 0;
+
+        for (auto lfidx : non_zero_leaves) {
+            leaf_paths[count].m_lfidx = lfidx;
+            tree->get_sortby_path(lfidx, leaf_paths[count].m_path);
+            std::reverse(
+                leaf_paths[count].m_path.begin(),
+                leaf_paths[count].m_path.end()
+            );
+            ++count;
+        }
+
+        std::sort(
+            leaf_paths.begin(),
+            leaf_paths.end(),
+            [](const t_leaf_path& a, const t_leaf_path& b) {
+                return a.m_path < b.m_path;
+            }
+        );
+
+        if (!leaf_paths.empty() && (traversal != nullptr)
+            && traversal->size() == 1) {
+            if (traversal->get_node(0).m_expanded) {
+                traversal->populate_root_children(tree);
+            }
+        } else {
+            for (const auto& lpath : leaf_paths) {
+                t_uindex lfidx = lpath.m_lfidx;
+                auto ancestry = tree->get_ancestry(lfidx);
+
+                t_uindex num_tnodes_existed = 0;
+
+                for (auto nidx : ancestry) {
+                    if (non_zero_ids.find(nidx) == non_zero_ids.end()
+                        || visited.find(nidx) != visited.end()) {
+                        ++num_tnodes_existed;
+                    } else {
+                        break;
+                    }
+                }
+
+                if (process_traversal) {
+                    traversal->add_node(
+                        ctx_sortby, ancestry, num_tnodes_existed
+                    );
+                }
+
+                for (auto nidx : ancestry) {
+                    visited.insert(nidx);
+                }
+            }
+        }
+    }
+}
+
+void
+notify_sparse_tree(
+    const std::shared_ptr<t_stree>& tree,
+    const std::shared_ptr<t_traversal>& traversal,
+    bool process_traversal,
+    const std::vector<t_aggspec>& aggregates,
+    const std::vector<std::pair<std::string, std::string>>& tree_sortby,
+    const std::vector<t_sortspec>& ctx_sortby,
+    const t_data_table& flattened,
+    const t_data_table& delta,
+    const t_data_table& prev,
+    const t_data_table& current,
+    const t_data_table& transitions,
+    const t_data_table& existed,
+    const t_config& config,
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table
+) {
+
+    auto strand_values = tree->build_strand_table(
+        flattened, delta, prev, current, transitions, aggregates, config
+    );
+
+    auto strands = strand_values.first;
+    auto strand_deltas = strand_values.second;
+    notify_sparse_tree_common(
+        strands,
+        strand_deltas,
+        tree,
+        traversal,
+        process_traversal,
+        aggregates,
+        tree_sortby,
+        ctx_sortby,
+        gstate,
+        expression_master_table
+    );
+}
+
+void
+notify_sparse_tree(
+    const std::shared_ptr<t_stree>& tree,
+    const std::shared_ptr<t_traversal>& traversal,
+    bool process_traversal,
+    const std::vector<t_aggspec>& aggregates,
+    const std::vector<std::pair<std::string, std::string>>& tree_sortby,
+    const std::vector<t_sortspec>& ctx_sortby,
+    const t_data_table& flattened,
+    const t_config& config,
+    const t_gstate& gstate,
+    const t_data_table& expression_master_table
+) {
+    auto strand_values =
+        tree->build_strand_table(flattened, aggregates, config);
+
+    auto strands = strand_values.first;
+    auto strand_deltas = strand_values.second;
+    notify_sparse_tree_common(
+        strands,
+        strand_deltas,
+        tree,
+        traversal,
+        process_traversal,
+        aggregates,
+        tree_sortby,
+        ctx_sortby,
+        gstate,
+        expression_master_table
+    );
+}
+
+std::vector<t_path>
+ctx_get_expansion_state(
+    const std::shared_ptr<const t_stree>& tree,
+    const std::shared_ptr<const t_traversal>& traversal
+) {
+    std::vector<t_path> paths;
+    std::vector<t_index> expanded;
+    traversal->get_expanded(expanded);
+
+    for (long long i : expanded) {
+        std::vector<t_tscalar> path;
+        tree->get_path(i, path);
+        paths.emplace_back(path);
+    }
+    return paths;
+}
+
+std::vector<t_tscalar>
+ctx_get_path(
+    const std::shared_ptr<const t_stree>& tree,
+    const std::shared_ptr<const t_traversal>& traversal,
+    t_index idx
+) {
+    if (idx < 0 || idx >= t_index(traversal->size())) {
+        std::vector<t_tscalar> rval;
+        return rval;
+    }
+
+    auto tree_index = traversal->get_tree_index(idx);
+    std::vector<t_tscalar> rval;
+    tree->get_path(tree_index, rval);
+    return rval;
+}
+
+std::vector<t_ftreenode>
+ctx_get_flattened_tree(
+    t_index idx,
+    t_depth stop_depth,
+    t_traversal& trav,
+    const t_config& config,
+    const std::vector<t_sortspec>& sortby
+) {
+    t_index ptidx = trav.get_tree_index(idx);
+    trav.set_depth(sortby, stop_depth);
+    if (!sortby.empty()) {
+        trav.sort_by(config, sortby, *(trav.get_tree()));
+    }
+    t_index new_tvidx = trav.tree_index_lookup(ptidx, idx);
+    return trav.get_flattened_tree(new_tvidx, stop_depth);
+}
+
+} // end namespace perspective

rust/perspective-server/cpp/perspective/src/cpp/update_task.cpp

@@ -0,0 +1,49 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+#include <perspective/first.h>
+#include <perspective/pool.h>
+#include <perspective/update_task.h>
+
+namespace perspective {
+t_update_task::t_update_task(t_pool& pool) : m_pool(pool) {}
+
+void
+t_update_task::run(std::optional<std::function<void(std::uint32_t)>> callback) {
+    auto work_to_do = m_pool.m_data_remaining.load();
+    m_pool.m_data_remaining.store(false);
+
+    if (work_to_do) {
+        for (auto* g : m_pool.m_gnodes) {
+            if (g != nullptr) {
+                t_uindex num_input_ports = g->num_input_ports();
+
+                // Call process for each port, and notify the updates from
+                // each port individually.
+                for (t_uindex port_id = 0; port_id < num_input_ports;
+                     ++port_id) {
+                    bool did_notify_context = g->process(port_id);
+                    if (did_notify_context) {
+                        if (callback) {
+                            (*callback)(port_id);
+                        }
+                        m_pool.notify_userspace(port_id);
+                    }
+                    g->clear_output_ports();
+                }
+            }
+        }
+    }
+
+    m_pool.inc_epoch();
+}
+} // end namespace perspective