wehub-resource-sync
423 lines
11 KiB
Go
423 lines
11 KiB
Go
// Copyright 2020 Dolthub, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package testcommands
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"github.com/dolthub/go-mysql-server/sql"
|
|
|
|
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
|
cmd "github.com/dolthub/dolt/go/cmd/dolt/commands"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/dbfactory"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb/durable"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/dtestutils"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/env/actions"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/dprocedures"
|
|
"github.com/dolthub/dolt/go/libraries/utils/config"
|
|
"github.com/dolthub/dolt/go/libraries/utils/filesys"
|
|
"github.com/dolthub/dolt/go/store/datas"
|
|
"github.com/dolthub/dolt/go/store/datas/pull"
|
|
"github.com/dolthub/dolt/go/store/types"
|
|
)
|
|
|
|
const (
|
|
homePrefix = "home"
|
|
)
|
|
|
|
type MultiRepoTestSetup struct {
|
|
envs map[string]*env.DoltEnv
|
|
Remote string
|
|
DoltDBs map[string]*doltdb.DoltDB
|
|
DbNames []string
|
|
Root string
|
|
DbPaths map[string]string
|
|
Home string
|
|
Remotes map[string]env.Remote
|
|
Errhand func(args ...interface{})
|
|
}
|
|
|
|
const (
|
|
name = "billy bob"
|
|
email = "bigbillieb@fake.horse"
|
|
defaultBranch = "main"
|
|
)
|
|
|
|
func NewMultiRepoTestSetup(errhand func(args ...interface{})) *MultiRepoTestSetup {
|
|
dir, err := os.MkdirTemp("", "")
|
|
if err != nil {
|
|
errhand(err)
|
|
}
|
|
|
|
homeDir, err := os.MkdirTemp(dir, homePrefix)
|
|
if err != nil {
|
|
errhand(err)
|
|
}
|
|
|
|
return &MultiRepoTestSetup{
|
|
envs: make(map[string]*env.DoltEnv),
|
|
Remotes: make(map[string]env.Remote),
|
|
DoltDBs: make(map[string]*doltdb.DoltDB, 0),
|
|
DbNames: make([]string, 0),
|
|
Root: dir,
|
|
Home: homeDir,
|
|
DbPaths: make(map[string]string, 0),
|
|
Errhand: errhand,
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) GetEnv(dbName string) *env.DoltEnv {
|
|
return mr.envs[dbName]
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) homeProv() (string, error) {
|
|
return mr.Home, nil
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) Close() {
|
|
for _, db := range mr.DoltDBs {
|
|
err := db.Close()
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) Cleanup(dbName string) {
|
|
os.RemoveAll(mr.Root)
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) NewDB(dbName string) {
|
|
ctx := context.Background()
|
|
|
|
repo := filepath.Join(mr.Root, dbName)
|
|
os.Mkdir(repo, os.ModePerm)
|
|
|
|
err := os.Chdir(repo)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
// TODO sometimes tempfiles scrubber is racy with tempfolder deleter
|
|
dEnv := env.LoadWithoutDB(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
|
if err != nil {
|
|
mr.Errhand("Failed to initialize environment:" + err.Error())
|
|
}
|
|
cfg, _ := dEnv.Config.GetConfig(env.GlobalConfig)
|
|
cfg.SetStrings(map[string]string{
|
|
config.UserNameKey: name,
|
|
config.UserEmailKey: email,
|
|
})
|
|
err = dEnv.InitRepo(context.Background(), types.Format_DOLT, name, email, defaultBranch)
|
|
if err != nil {
|
|
mr.Errhand("Failed to initialize environment:" + err.Error())
|
|
}
|
|
|
|
ddb, err := doltdb.LoadDoltDB(ctx, types.Format_DOLT, doltdb.LocalDirDoltDB, filesys.LocalFS)
|
|
if err != nil {
|
|
mr.Errhand("Failed to initialize environment:" + err.Error())
|
|
}
|
|
|
|
dEnv = env.Load(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
|
|
|
mr.envs[dbName] = dEnv
|
|
mr.DoltDBs[dbName] = ddb
|
|
mr.DbNames = append(mr.DbNames, dbName)
|
|
mr.DbPaths[dbName] = repo
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) NewRemote(remoteName string) {
|
|
remote := filepath.Join(mr.Root, remoteName)
|
|
os.Mkdir(remote, os.ModePerm)
|
|
remotePath := fmt.Sprintf("file:///%s", remote)
|
|
|
|
rem := env.NewRemote(remoteName, remotePath, nil)
|
|
|
|
for _, dEnv := range mr.envs {
|
|
dEnv.RepoState.AddRemote(rem)
|
|
dEnv.RepoState.Save(filesys.LocalFS)
|
|
}
|
|
|
|
mr.Remotes[remoteName] = rem
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) NewBranch(ctx context.Context, dbName, branchName string) {
|
|
dEnv := mr.envs[dbName]
|
|
err := actions.CreateBranchWithStartPt(context.Background(), dEnv.DbData(ctx), branchName, "head", false, nil)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) CheckoutBranch(dbName, branchName string) {
|
|
dEnv := mr.envs[dbName]
|
|
var err error
|
|
cliCtx, err := cmd.NewArgFreeCliContext(context.Background(), dEnv, dEnv.FS)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
defer cliCtx.Close()
|
|
queryist, err := cliCtx.QueryEngine(context.Background())
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
err = dprocedures.MoveWorkingSetToBranch(queryist.Context, branchName, false, false, true)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) CloneDB(fromRemote, dbName string) {
|
|
ctx := context.Background()
|
|
cloneDir := filepath.Join(mr.Root, dbName)
|
|
|
|
r := mr.GetRemote(fromRemote)
|
|
srcDB, err := r.GetRemoteDB(ctx, types.Format_DOLT, mr.envs[dbName])
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
dEnv := env.LoadWithoutDB(context.Background(), mr.homeProv, filesys.LocalFS, doltdb.LocalDirDoltDB, "test")
|
|
dEnv, err = actions.EnvForClone(ctx, srcDB.Format(), r, cloneDir, dEnv.FS, dEnv.Version, mr.homeProv)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
pull.WithDiscardingStatsCh(func(statsCh chan pull.Stats) {
|
|
err = actions.CloneRemote(ctx, srcDB, r.Name, "", false, -1, dEnv, statsCh)
|
|
})
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
err = dbfactory.ClearDatabaseInProgress(dEnv.FS)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
wd, err := os.Getwd()
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
err = os.Chdir(cloneDir)
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
defer os.Chdir(wd)
|
|
|
|
ddb := dEnv.DoltDB(ctx)
|
|
|
|
mr.envs[dbName] = dEnv
|
|
mr.DoltDBs[dbName] = ddb
|
|
mr.DbNames = append(mr.DbNames, dbName)
|
|
mr.DbPaths[dbName] = cloneDir
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) GetRemote(remoteName string) env.Remote {
|
|
rem, ok := mr.Remotes[remoteName]
|
|
if !ok {
|
|
mr.Errhand("remote not found")
|
|
}
|
|
return rem
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) GetDB(dbName string) *doltdb.DoltDB {
|
|
db, ok := mr.DoltDBs[dbName]
|
|
if !ok {
|
|
mr.Errhand("db not found")
|
|
}
|
|
return db
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) CommitWithWorkingSet(dbName string) *doltdb.Commit {
|
|
ctx := sql.NewEmptyContext()
|
|
dEnv := mr.envs[dbName]
|
|
ws, err := dEnv.WorkingSet(ctx)
|
|
if err != nil {
|
|
panic("couldn't get working set: " + err.Error())
|
|
}
|
|
|
|
prevHash, err := ws.HashOf()
|
|
if err != nil {
|
|
panic("couldn't get working set: " + err.Error())
|
|
}
|
|
|
|
var mergeParentCommits []*doltdb.Commit
|
|
if ws.MergeActive() {
|
|
mergeParentCommits = []*doltdb.Commit{ws.MergeState().Commit()}
|
|
}
|
|
|
|
roots, err := dEnv.Roots(ctx)
|
|
if err != nil {
|
|
panic("couldn't get roots: " + err.Error())
|
|
}
|
|
ident := datas.CommitIdent{Name: name, Email: email}
|
|
commitStagedProps := actions.CommitStagedProps{
|
|
Message: "auto commit",
|
|
AllowEmpty: true,
|
|
Author: ident,
|
|
Committer: ident,
|
|
}
|
|
pendingCommit, err := actions.GetCommitStaged(ctx, doltdb.SimpleTableResolver{}, roots, ws, mergeParentCommits, dEnv.DbData(ctx).Ddb, commitStagedProps)
|
|
if err != nil {
|
|
panic("pending commit error: " + err.Error())
|
|
}
|
|
|
|
headRef, err := dEnv.RepoStateReader().CWBHeadRef(ctx)
|
|
if err != nil {
|
|
panic("couldn't get working set: " + err.Error())
|
|
}
|
|
|
|
commit, err := dEnv.DoltDB(ctx).CommitWithWorkingSet(
|
|
ctx,
|
|
headRef,
|
|
ws.Ref(),
|
|
pendingCommit,
|
|
ws.WithStagedRoot(pendingCommit.Roots.Staged).WithWorkingRoot(pendingCommit.Roots.Working).ClearMerge(),
|
|
prevHash,
|
|
doltdb.TodoWorkingSetMeta(),
|
|
nil,
|
|
)
|
|
if err != nil {
|
|
panic("couldn't commit: " + err.Error())
|
|
}
|
|
return commit
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) CreateTable(ctx context.Context, dbName, tblName string) {
|
|
dEnv := mr.envs[dbName]
|
|
|
|
sch, err := dtestutils.Schema()
|
|
if err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
|
|
if err := createTestTable(dEnv, tblName, sch); err != nil {
|
|
mr.Errhand(err)
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) StageAll(dbName string) {
|
|
dEnv := mr.envs[dbName]
|
|
|
|
ctx := context.Background()
|
|
roots, err := dEnv.Roots(ctx)
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to get roots: %s", dbName))
|
|
}
|
|
|
|
roots, err = actions.StageAllTables(ctx, roots, true)
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to stage tables: %s", dbName))
|
|
}
|
|
err = dEnv.UpdateRoots(ctx, roots)
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to update roots: %s", dbName))
|
|
}
|
|
}
|
|
|
|
func (mr *MultiRepoTestSetup) PushToRemote(dbName, remoteName, branchName string) {
|
|
ctx := context.Background()
|
|
dEnv := mr.envs[dbName]
|
|
|
|
ap := cli.CreatePushArgParser()
|
|
apr, err := ap.Parse([]string{remoteName, branchName})
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to push remote: %s", err.Error()))
|
|
}
|
|
targets, remote, err := env.NewPushOpts(ctx, apr, dEnv.RepoStateReader(), dEnv.DoltDB(ctx), false, false, false, false)
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to push remote: %s", err.Error()))
|
|
}
|
|
|
|
remoteDB, err := remote.GetRemoteDB(ctx, dEnv.DoltDB(ctx).ValueReadWriter().Format(), mr.envs[dbName])
|
|
if err != nil {
|
|
mr.Errhand(actions.HandleInitRemoteStorageClientErr(remote.Name, remote.Url, err))
|
|
}
|
|
|
|
tmpDir, err := dEnv.TempTableFilesDir()
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to access .dolt directory: %s", err.Error()))
|
|
}
|
|
|
|
pushOptions := &env.PushOptions[context.Context]{
|
|
Targets: targets,
|
|
Remote: remote,
|
|
Rsr: dEnv.RepoStateReader(),
|
|
Rsw: dEnv.RepoStateWriter(),
|
|
SrcDb: dEnv.DoltDB(ctx),
|
|
DestDb: remoteDB,
|
|
TmpDir: tmpDir,
|
|
}
|
|
pull.WithDiscardingStatsCh(func(statsCh chan pull.Stats) {
|
|
_, err = actions.DoPush(ctx, pushOptions, statsCh)
|
|
})
|
|
if err != nil {
|
|
mr.Errhand(fmt.Sprintf("Failed to push remote: %s", err.Error()))
|
|
}
|
|
}
|
|
|
|
// createTestTable creates a new test table with the name, schema, and rows given.
|
|
func createTestTable(dEnv *env.DoltEnv, tableName string, sch schema.Schema) error {
|
|
ctx := context.Background()
|
|
vrw := dEnv.DoltDB(ctx).ValueReadWriter()
|
|
ns := dEnv.DoltDB(ctx).NodeStore()
|
|
|
|
idx, err := durable.NewEmptyPrimaryIndex(ctx, vrw, ns, sch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
tbl, err := doltdb.NewTable(ctx, vrw, ns, sch, idx, nil, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
sch, err = tbl.GetSchema(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
root, err := dEnv.WorkingRoot(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("%w: %v", doltdb.ErrNomsIO, err)
|
|
}
|
|
|
|
newRoot, err := root.PutTable(ctx, doltdb.TableName{Name: tableName}, tbl)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
rootHash, err := root.HashOf()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
newRootHash, err := newRoot.HashOf()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if rootHash == newRootHash {
|
|
return nil
|
|
}
|
|
return dEnv.UpdateWorkingRoot(ctx, newRoot)
|
|
}
|