dolthub--dolt/go/performance/utils/benchmark_runner/run_test.go

// Copyright 2022 Dolthub, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package benchmark_runner

import (
	"context"
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"testing"
)

var runTests = os.Getenv("RUN_BENCHMARK_RUNNER_TESTS")
var doltExec = os.Getenv("BENCHMARK_RUNNER_DOLT_EXEC")
var doltVersion = os.Getenv("BENCHMARK_RUNNER_DOLT_VERSION")
var mysqlExec = os.Getenv("BENCHMARK_RUNNER_MYSQL_EXEC")
var mysqlProtocol = os.Getenv("BENCHMARK_RUNNER_MYSQL_PROTOCOL")
var mysqlSocket = os.Getenv("BENCHMARK_RUNNER_MYSQL_SOCKET")
var mysqlVersion = os.Getenv("BENCHMARK_RUNNER_MYSQL_VERSION")
var doltgresExec = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_EXEC")
var doltgresVersion = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_VERSION")
var doltgresConfigFilePath = os.Getenv("BENCHMARK_RUNNER_DOLTGRES_CONFIG_FILE_PATH")
var postgresExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_EXEC")
var postgresInitExec = os.Getenv("BENCHMARK_RUNNER_POSTGRES_INIT_EXEC")
var postgresVersion = os.Getenv("BENCHMARK_RUNNER_POSTGRES_VERSION")
var sysbenchLuaScripts = os.Getenv("BENCHMARK_RUNNER_SYSBENCH_LUA_SCRIPTS")
var tpccLuaScripts = os.Getenv("BENCHMARK_RUNNER_TPCC_LUA_SCRIPTS")

func TestRunner(t *testing.T) {
	if runTests == "" {
		t.Skip()
	}
	dir := t.TempDir()
	log.Println(dir)
	err := os.Chdir(dir)
	if err != nil {
		log.Fatal(err)
	}

	conf := &sysbenchRunnerConfigImpl{
		Tests: []TestConfig{
			NewTestConfig("oltp_read_only", nil, false),
			//NewTestConfig("oltp_update_index", nil, false),
			//NewTestConfig("oltp_delete_insert", nil, true),
		},
		//Tests: selectTests("oltp_read_write", "oltp_update_index", "oltp_update_non_index", "oltp_insert", "bulk_insert", "oltp_write_only", "oltp_delete"),
		Servers: []ServerConfig{
			&doltServerConfigImpl{
				Id:            "test",
				Version:       doltVersion,
				ResultsFormat: CsvFormat,
				ServerExec:    doltExec,
			},
		},
		ScriptDir: sysbenchLuaScripts,
		TestOptions: []string{
			"--rand-seed=1",
			"--table-size=30",
			"--rand-type=uniform",
			"--time=30",
			"--percentile=50",
		},
		InitBigRepo: true,
	}

	err = Run(context.Background(), conf)
	if err != nil {
		log.Fatal(err)
	}
}

func selectTests(names ...string) []TestConfig {
	tests := make([]TestConfig, len(names))
	for i := range names {
		tests[i] = &testConfigImpl{Name: names[i], FromScript: false}
	}
	return tests
}

func TestDoltMysqlSysbenchRunner(t *testing.T) {
	if runTests == "" {
		t.Skip()
	}
	dir := t.TempDir()
	log.Println(dir)
	err := os.Chdir(dir)
	if err != nil {
		log.Fatal(err)
	}

	conf := &sysbenchRunnerConfigImpl{
		Tests: []TestConfig{
			NewTestConfig("oltp_read_only", nil, false),
			//NewTestConfig("oltp_update_index", nil, false),
			//NewTestConfig("oltp_delete_insert", nil, true),
		},
		Servers: []ServerConfig{
			&doltServerConfigImpl{
				Id:            "test-dolt",
				Version:       doltVersion,
				ResultsFormat: CsvFormat,
				ServerExec:    doltExec,
			},
			&mysqlServerConfigImpl{
				Id:                 "test-mysql",
				Port:               3606,
				Version:            mysqlVersion,
				ResultsFormat:      CsvFormat,
				ServerExec:         mysqlExec,
				ServerUser:         "root",
				SkipLogBin:         true,
				ConnectionProtocol: mysqlProtocol,
				Socket:             mysqlSocket,
			},
		},
		ScriptDir: sysbenchLuaScripts,
		TestOptions: []string{
			"--rand-seed=1",
			"--table-size=30",
			"--rand-type=uniform",
			"--time=30",
			"--percentile=50",
		},
		InitBigRepo: true,
	}

	err = Run(context.Background(), conf)
	if err != nil {
		log.Fatal(err)
	}
}

func TestDoltgresPostgresSysbenchRunner(t *testing.T) {
	if runTests == "" {
		t.Skip()
	}
	if doltgresConfigFilePath == "" {
		t.Skip("skipping doltgres/postgres benchmark runner tests, no config file specified")
	}

	dir := t.TempDir()
	log.Println(dir)
	err := os.Chdir(dir)
	if err != nil {
		log.Fatal(err)
	}

	conf := &sysbenchRunnerConfigImpl{
		Tests: []TestConfig{
			NewTestConfig("oltp_read_only", nil, false),
			//NewTestConfig("oltp_update_index", nil, false),
		},
		Servers: []ServerConfig{
			&postgresServerConfigImpl{
				Id:            "test-postgres",
				Host:          "127.0.0.1",
				Version:       postgresVersion,
				ResultsFormat: CsvFormat,
				ServerExec:    postgresExec,
				InitExec:      postgresInitExec,
				ServerUser:    "root",
			},
			&doltgresServerConfigImpl{
				Id:             "test-doltgres",
				Port:           4433,
				Host:           "127.0.0.1",
				ConfigFilePath: doltgresConfigFilePath,
				Version:        doltgresVersion,
				ResultsFormat:  CsvFormat,
				ServerExec:     doltgresExec,
			},
		},
		TestOptions: []string{
			"--rand-seed=1",
			"--table-size=30",
			"--rand-type=uniform",
			"--time=30",
			"--percentile=50",
		},
	}

	err = Run(context.Background(), conf)
	if err != nil {
		log.Fatal(err)
	}
}

func TestDoltProfiler(t *testing.T) {
	if runTests == "" {
		t.Skip()
	}
	dir := t.TempDir()
	log.Println(dir)
	err := os.Chdir(dir)
	if err != nil {
		log.Fatal(err)
	}

	id := "test-dolt-profile"
	conf := &sysbenchRunnerConfigImpl{
		Tests: []TestConfig{
			NewTestConfig("oltp_read_only", nil, false),
		},
		Servers: []ServerConfig{
			&doltServerConfigImpl{
				Id:            id,
				Version:       doltVersion,
				ResultsFormat: CsvFormat,
				ServerExec:    doltExec,
				ServerProfile: CpuServerProfile,
				ProfilePath:   dir,
			},
		},
		TestOptions: []string{
			"--rand-seed=1",
			"--table-size=30",
			"--rand-type=uniform",
			"--time=30",
			"--percentile=50",
		},
	}

	err = Run(context.Background(), conf)
	if err != nil {
		log.Fatal(err)
	}

	expectedProfile := filepath.Join(dir, fmt.Sprintf("%s_%s", id, cpuProfileFilename))
	if _, err := os.Stat(expectedProfile); errors.Is(err, os.ErrNotExist) {
		log.Fatal("failed to create dolt cpu profile")
	}
}

func TestDoltMysqlTpccRunner(t *testing.T) {
	t.Skip() // skip for now since this is kinda slow for pr ci
	dir := t.TempDir()
	log.Println(dir)
	err := os.Chdir(dir)
	if err != nil {
		log.Fatal(err)
	}

	conf := &tpccConfigImpl{
		Servers: []ServerConfig{
			&doltServerConfigImpl{
				Id:            "test-dolt-tpcc",
				Version:       doltVersion,
				ResultsFormat: CsvFormat,
				ServerExec:    doltExec,
			},
			&mysqlServerConfigImpl{
				Id:                 "test-mysql-tpcc",
				Port:               3606,
				Version:            mysqlVersion,
				ResultsFormat:      CsvFormat,
				ServerExec:         mysqlExec,
				ServerUser:         "root",
				SkipLogBin:         true,
				ConnectionProtocol: mysqlProtocol,
				Socket:             mysqlSocket,
			},
		},
		ScriptDir: tpccLuaScripts,
	}

	err = RunTpcc(context.Background(), conf)
	if err != nil {
		log.Fatal(err)
	}
}