wehub-resource-sync
2007 lines
64 KiB
Go
2007 lines
64 KiB
Go
// Copyright 2019 Dolthub, Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package commands
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/dolthub/go-mysql-server/sql"
|
|
"github.com/dolthub/go-mysql-server/sql/types"
|
|
"github.com/dolthub/vitess/go/sqltypes"
|
|
ast "github.com/dolthub/vitess/go/vt/sqlparser"
|
|
"github.com/gocraft/dbr/v2"
|
|
"github.com/gocraft/dbr/v2/dialect"
|
|
|
|
"github.com/dolthub/dolt/go/cmd/dolt/cli"
|
|
"github.com/dolthub/dolt/go/cmd/dolt/errhand"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/diff"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/doltdb"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/env"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/schema"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typecompatibility"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/schema/typeinfo"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/sqle/sqlutil"
|
|
"github.com/dolthub/dolt/go/libraries/doltcore/table/untyped/tabular"
|
|
"github.com/dolthub/dolt/go/libraries/utils/argparser"
|
|
"github.com/dolthub/dolt/go/libraries/utils/iohelp"
|
|
"github.com/dolthub/dolt/go/libraries/utils/set"
|
|
eventsapi "github.com/dolthub/eventsapi_schema/dolt/services/eventsapi/v1alpha1"
|
|
)
|
|
|
|
type diffOutput int
|
|
type diffPart int
|
|
|
|
const (
|
|
SchemaOnlyDiff diffPart = 1 // 0b0000 0001
|
|
DataOnlyDiff diffPart = 2 // 0b0000 0010
|
|
NameOnlyDiff diffPart = 4 // 0b0000 0100
|
|
Stat diffPart = 8 // 0b0000 1000
|
|
Summary diffPart = 16 // 0b0001 0000
|
|
|
|
SchemaAndDataDiff = SchemaOnlyDiff | DataOnlyDiff
|
|
|
|
TabularDiffOutput diffOutput = 1
|
|
SQLDiffOutput diffOutput = 2
|
|
JsonDiffOutput diffOutput = 3
|
|
)
|
|
|
|
var diffDocs = cli.CommandDocumentationContent{
|
|
ShortDesc: "Show changes between commits, commit and working tree, etc",
|
|
LongDesc: `
|
|
Show changes between the working and staged tables, changes between the working tables and the tables within a commit, or changes between tables at two commits.
|
|
|
|
{{.EmphasisLeft}}dolt diff [--options] [<tables>...]{{.EmphasisRight}}
|
|
This form is to view the changes you made relative to the staging area for the next commit. In other words, the differences are what you could tell Dolt to further add but you still haven't. You can stage these changes by using dolt add.
|
|
|
|
{{.EmphasisLeft}}dolt diff [--options] [--merge-base] <commit> [<tables>...]{{.EmphasisRight}}
|
|
This form is to view the changes you have in your working tables relative to the named {{.LessThan}}commit{{.GreaterThan}}. You can use HEAD to compare it with the latest commit, or a branch name to compare with the tip of a different branch. If {{.EmphasisLeft}}--merge-base{{.EmphasisRight}} is given, instead of using {{.LessThan}}commit{{.GreaterThan}}, use the merge base of {{.LessThan}}commit{{.GreaterThan}} and HEAD. {{.EmphasisLeft}}dolt diff --merge-base A{{.EmphasisRight}} is equivalent to {{.EmphasisLeft}}dolt diff $(dolt merge-base A HEAD){{.EmphasisRight}} and {{.EmphasisLeft}}dolt diff A...HEAD{{.EmphasisRight}}.
|
|
|
|
{{.EmphasisLeft}}dolt diff [--options] [--merge-base] <commit> <commit> [<tables>...]{{.EmphasisRight}}
|
|
This is to view the changes between two arbitrary {{.EmphasisLeft}}commit{{.EmphasisRight}}. If {{.EmphasisLeft}}--merge-base{{.EmphasisRight}} is given, use the merge base of the two commits for the "before" side. {{.EmphasisLeft}}dolt diff --merge-base A B{{.EmphasisRight}} is equivalent to {{.EmphasisLeft}}dolt diff $(dolt merge-base A B) B{{.EmphasisRight}} and {{.EmphasisLeft}}dolt diff A...B{{.EmphasisRight}}.
|
|
|
|
{{.EmphasisLeft}}dolt diff [--options] <commit>..<commit> [<tables>...]{{.EmphasisRight}}
|
|
This is synonymous to the above form (without the ..) to view the changes between two arbitrary {{.EmphasisLeft}}commit{{.EmphasisRight}}.
|
|
|
|
{{.EmphasisLeft}}dolt diff [--options] <commit>...<commit> [<tables>...]{{.EmphasisRight}}
|
|
This is to view the changes on the branch containing and up to the second {{.LessThan}}commit{{.GreaterThan}}, starting at a common ancestor of both {{.LessThan}}commit{{.GreaterThan}}. {{.EmphasisLeft}}dolt diff A...B{{.EmphasisRight}} is equivalent to {{.EmphasisLeft}}dolt diff $(dolt merge-base A B) B{{.EmphasisRight}} and {{.EmphasisLeft}}dolt diff --merge-base A B{{.EmphasisRight}}. You can omit any one of {{.LessThan}}commit{{.GreaterThan}}, which has the same effect as using HEAD instead.
|
|
|
|
The diffs displayed can be limited to show the first N by providing the parameter {{.EmphasisLeft}}--limit N{{.EmphasisRight}} where {{.EmphasisLeft}}N{{.EmphasisRight}} is the number of diffs to display.
|
|
|
|
To filter which data rows are displayed, use {{.EmphasisLeft}}--where <SQL expression>{{.EmphasisRight}}. Table column names in the filter expression must be prefixed with {{.EmphasisLeft}}from_{{.EmphasisRight}} or {{.EmphasisLeft}}to_{{.EmphasisRight}}, e.g. {{.EmphasisLeft}}to_COLUMN_NAME > 100{{.EmphasisRight}} or {{.EmphasisLeft}}from_COLUMN_NAME + to_COLUMN_NAME = 0{{.EmphasisRight}}.
|
|
|
|
To filter diff output by change type, use {{.EmphasisLeft}}--filter <type>{{.EmphasisRight}} where {{.EmphasisLeft}}<type>{{.EmphasisRight}} is one of {{.EmphasisLeft}}added{{.EmphasisRight}}, {{.EmphasisLeft}}modified{{.EmphasisRight}}, {{.EmphasisLeft}}renamed{{.EmphasisRight}}, or {{.EmphasisLeft}}dropped{{.EmphasisRight}}. The {{.EmphasisLeft}}added{{.EmphasisRight}} filter shows only additions (new tables or rows), {{.EmphasisLeft}}modified{{.EmphasisRight}} shows only schema modifications or row updates, {{.EmphasisLeft}}renamed{{.EmphasisRight}} shows only renamed tables, and {{.EmphasisLeft}}dropped{{.EmphasisRight}} shows only deletions (dropped tables or deleted rows). You can also use {{.EmphasisLeft}}removed{{.EmphasisRight}} as an alias for {{.EmphasisLeft}}dropped{{.EmphasisRight}}. For example, {{.EmphasisLeft}}dolt diff --filter=dropped{{.EmphasisRight}} shows only deleted rows and dropped tables.
|
|
|
|
The {{.EmphasisLeft}}--diff-mode{{.EmphasisRight}} argument controls how modified rows are presented when the format output is set to {{.EmphasisLeft}}tabular{{.EmphasisRight}}. When set to {{.EmphasisLeft}}row{{.EmphasisRight}}, modified rows are presented as old and new rows. When set to {{.EmphasisLeft}}line{{.EmphasisRight}}, modified rows are presented as a single row, and changes are presented using "+" and "-" within the column. When set to {{.EmphasisLeft}}in-place{{.EmphasisRight}}, modified rows are presented as a single row, and changes are presented side-by-side with a color distinction (requires a color-enabled terminal). When set to {{.EmphasisLeft}}context{{.EmphasisRight}}, rows that contain at least one column that spans multiple lines uses {{.EmphasisLeft}}line{{.EmphasisRight}}, while all other rows use {{.EmphasisLeft}}row{{.EmphasisRight}}. The default value is {{.EmphasisLeft}}context{{.EmphasisRight}}.
|
|
`,
|
|
Synopsis: []string{
|
|
`[options] [{{.LessThan}}commit{{.GreaterThan}}] [{{.LessThan}}tables{{.GreaterThan}}...]`,
|
|
`[options] {{.LessThan}}commit{{.GreaterThan}} {{.LessThan}}commit{{.GreaterThan}} [{{.LessThan}}tables{{.GreaterThan}}...]`,
|
|
},
|
|
}
|
|
|
|
type diffDisplaySettings struct {
|
|
diffParts diffPart
|
|
diffOutput diffOutput
|
|
diffMode diff.Mode
|
|
limit int
|
|
where string
|
|
skinny bool
|
|
includeCols []string
|
|
filter *diffTypeFilter
|
|
}
|
|
|
|
type diffDatasets struct {
|
|
fromRef string
|
|
toRef string
|
|
}
|
|
|
|
// hasWorkingSet reports whether either side of the diff references the working set or staging area.
|
|
func (d *diffDatasets) hasWorkingSet() bool {
|
|
return doltdb.IsWorkingSetRef(d.fromRef) || doltdb.IsWorkingSetRef(d.toRef)
|
|
}
|
|
|
|
type diffArgs struct {
|
|
*diffDisplaySettings
|
|
*diffDatasets
|
|
tableSet *set.StrSet
|
|
}
|
|
|
|
type diffStatistics struct {
|
|
TableName string
|
|
RowsUnmodified uint64
|
|
RowsAdded uint64
|
|
RowsDeleted uint64
|
|
RowsModified uint64
|
|
CellsAdded uint64
|
|
CellsDeleted uint64
|
|
CellsModified uint64
|
|
OldRowCount uint64
|
|
NewRowCount uint64
|
|
OldCellCount uint64
|
|
NewCellCount uint64
|
|
}
|
|
|
|
// diffTypeFilter manages which diff types should be included in the output.
|
|
// When filters is nil or empty, all types are included.
|
|
type diffTypeFilter struct {
|
|
// Map of diff type -> should include
|
|
// If nil or empty, includes all types
|
|
filters map[string]bool
|
|
}
|
|
|
|
// newDiffTypeFilter creates a filter for the specified diff type.
|
|
// Pass diff.DiffTypeAll or empty string to include all types.
|
|
// Accepts "removed" as an alias for "dropped" for user convenience.
|
|
func newDiffTypeFilter(filterType string) *diffTypeFilter {
|
|
if filterType == "" || filterType == diff.DiffTypeAll {
|
|
return &diffTypeFilter{filters: nil} // nil means include all
|
|
}
|
|
|
|
// Map "removed" to "dropped" (alias for user convenience)
|
|
internalFilterType := filterType
|
|
if filterType == "removed" {
|
|
internalFilterType = diff.DiffTypeDropped
|
|
}
|
|
|
|
return &diffTypeFilter{
|
|
filters: map[string]bool{
|
|
internalFilterType: true,
|
|
},
|
|
}
|
|
}
|
|
|
|
// shouldInclude checks if the given diff type should be included.
|
|
// Uses TableDeltaSummary.DiffType field for table-level filtering.
|
|
func (df *diffTypeFilter) shouldInclude(diffType string) bool {
|
|
// nil or empty filters means include everything
|
|
if df.filters == nil || len(df.filters) == 0 {
|
|
return true
|
|
}
|
|
|
|
return df.filters[diffType]
|
|
}
|
|
|
|
// isValid validates the filter configuration
|
|
func (df *diffTypeFilter) isValid() bool {
|
|
if df.filters == nil {
|
|
return true
|
|
}
|
|
|
|
for filterType := range df.filters {
|
|
if filterType != diff.DiffTypeAdded &&
|
|
filterType != diff.DiffTypeModified &&
|
|
filterType != diff.DiffTypeRenamed &&
|
|
filterType != diff.DiffTypeDropped {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// shouldSkipRow checks if a row should be skipped based on the filter settings.
|
|
// Uses the DiffType infrastructure for consistency with table-level filtering.
|
|
func shouldSkipRow(filter *diffTypeFilter, rowChangeType diff.ChangeType) bool {
|
|
if filter == nil {
|
|
return false
|
|
}
|
|
|
|
// Don't filter None - it represents "no row" on one side of the diff
|
|
if rowChangeType == diff.None {
|
|
return false
|
|
}
|
|
|
|
// Convert row-level ChangeType to table-level DiffType string
|
|
diffType := diff.ChangeTypeToDiffType(rowChangeType)
|
|
|
|
// Use the map-based shouldInclude method
|
|
return !filter.shouldInclude(diffType)
|
|
}
|
|
|
|
// shouldUseLazyHeader determines if we should delay printing the table header
|
|
// until we know there are rows to display. This prevents empty headers when
|
|
// all rows are filtered out in data-only diffs.
|
|
func shouldUseLazyHeader(dArgs *diffArgs, tableSummary diff.TableDeltaSummary) bool {
|
|
return dArgs.filter != nil && dArgs.filter.filters != nil &&
|
|
!tableSummary.SchemaChange && !tableSummary.IsRename()
|
|
}
|
|
|
|
// lazyRowWriter wraps a SqlRowDiffWriter and delays calling BeginTable
|
|
// until the first row is actually written. This prevents empty table headers
|
|
// when all rows are filtered out.
|
|
type lazyRowWriter struct {
|
|
writer diff.SqlRowDiffWriter
|
|
|
|
// Callback to invoke before first write
|
|
// Set to nil after first call
|
|
onFirstWrite func() error
|
|
}
|
|
|
|
// newLazyRowWriter creates a lazy writer that wraps the given writer.
|
|
// The onFirstWrite callback is invoked exactly once before the first write.
|
|
func newLazyRowWriter(writer diff.SqlRowDiffWriter, onFirstWrite func() error) *lazyRowWriter {
|
|
return &lazyRowWriter{
|
|
writer: writer,
|
|
onFirstWrite: onFirstWrite,
|
|
}
|
|
}
|
|
|
|
// WriteRow implements diff.SqlRowDiffWriter
|
|
func (l *lazyRowWriter) WriteRow(ctx *sql.Context, row sql.Row, diffType diff.ChangeType, colDiffTypes []diff.ChangeType) error {
|
|
// Initialize on first write
|
|
if l.onFirstWrite != nil {
|
|
if err := l.onFirstWrite(); err != nil {
|
|
return err
|
|
}
|
|
l.onFirstWrite = nil // Prevent double-initialization
|
|
}
|
|
|
|
return l.writer.WriteRow(ctx, row, diffType, colDiffTypes)
|
|
}
|
|
|
|
// WriteCombinedRow implements diff.SqlRowDiffWriter
|
|
func (l *lazyRowWriter) WriteCombinedRow(ctx *sql.Context, oldRow, newRow sql.Row, mode diff.Mode) error {
|
|
// Initialize on first write
|
|
if l.onFirstWrite != nil {
|
|
if err := l.onFirstWrite(); err != nil {
|
|
return err
|
|
}
|
|
l.onFirstWrite = nil
|
|
}
|
|
|
|
return l.writer.WriteCombinedRow(ctx, oldRow, newRow, mode)
|
|
}
|
|
|
|
// Close implements diff.SqlRowDiffWriter
|
|
func (l *lazyRowWriter) Close(ctx context.Context) error {
|
|
return l.writer.Close(ctx)
|
|
}
|
|
|
|
type DiffCmd struct{}
|
|
|
|
// Name is returns the name of the Dolt cli command. This is what is used on the command line to invoke the command
|
|
func (cmd DiffCmd) Name() string {
|
|
return "diff"
|
|
}
|
|
|
|
// Description returns a description of the command
|
|
func (cmd DiffCmd) Description() string {
|
|
return "Diff a table."
|
|
}
|
|
|
|
// EventType returns the type of the event to log
|
|
func (cmd DiffCmd) EventType() eventsapi.ClientEventType {
|
|
return eventsapi.ClientEventType_DIFF
|
|
}
|
|
|
|
func (cmd DiffCmd) Docs() *cli.CommandDocumentation {
|
|
ap := cmd.ArgParser()
|
|
return cli.NewCommandDocumentation(diffDocs, ap)
|
|
}
|
|
|
|
func (cmd DiffCmd) ArgParser() *argparser.ArgParser {
|
|
return cli.CreateDiffArgParser(false)
|
|
}
|
|
|
|
// Exec executes the command
|
|
func (cmd DiffCmd) Exec(ctx context.Context, commandStr string, args []string, _ *env.DoltEnv, cliCtx cli.CliContext) int {
|
|
ap := cmd.ArgParser()
|
|
apr, usage, terminate, status := ParseArgsOrPrintHelp(ap, commandStr, args, diffDocs)
|
|
if terminate {
|
|
return status
|
|
}
|
|
|
|
verr := cmd.validateArgs(apr)
|
|
if verr != nil {
|
|
return HandleVErrAndExitCode(verr, usage)
|
|
}
|
|
|
|
queryist, err := cliCtx.QueryEngine(ctx)
|
|
if err != nil {
|
|
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
|
}
|
|
|
|
updateSystemVar, err := cli.SetSystemVar(queryist.Queryist, queryist.Context, apr.Contains(cli.SystemFlag))
|
|
if err != nil {
|
|
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
|
}
|
|
|
|
dArgs, err := parseDiffArgs(queryist.Queryist, queryist.Context, apr)
|
|
if err != nil {
|
|
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
|
}
|
|
|
|
verr = diffUserTables(queryist.Queryist, queryist.Context, dArgs)
|
|
if verr != nil {
|
|
return HandleVErrAndExitCode(verr, usage)
|
|
}
|
|
|
|
if updateSystemVar != nil {
|
|
err = updateSystemVar()
|
|
}
|
|
|
|
return HandleVErrAndExitCode(errhand.VerboseErrorFromError(err), usage)
|
|
}
|
|
|
|
func (cmd DiffCmd) validateArgs(apr *argparser.ArgParseResults) errhand.VerboseError {
|
|
if apr.Contains(cli.StatFlag) || apr.Contains(cli.SummaryFlag) {
|
|
if apr.Contains(cli.SchemaFlag) || apr.Contains(cli.DataFlag) {
|
|
return errhand.BuildDError("invalid Arguments: --stat and --summary cannot be combined with --schema or --data").Build()
|
|
}
|
|
}
|
|
|
|
if apr.Contains(cli.NameOnlyFlag) {
|
|
if apr.Contains(cli.SchemaFlag) || apr.Contains(cli.DataFlag) || apr.Contains(cli.StatFlag) || apr.Contains(cli.SummaryFlag) {
|
|
return errhand.BuildDError("invalid Arguments: --name-only cannot be combined with --schema, --data, --stat, or --summary").Build()
|
|
}
|
|
}
|
|
|
|
f, _ := apr.GetValue(FormatFlag)
|
|
switch strings.ToLower(f) {
|
|
case "tabular", "sql", "json", "":
|
|
default:
|
|
return errhand.BuildDError("invalid output format: %s", f).Build()
|
|
}
|
|
|
|
filterValue, hasFilter := apr.GetValue(cli.FilterParam)
|
|
if hasFilter {
|
|
filter := newDiffTypeFilter(filterValue)
|
|
if !filter.isValid() {
|
|
return errhand.BuildDError("invalid filter: %s. Valid values are: %s, %s, %s, %s (or %s)",
|
|
filterValue, diff.DiffTypeAdded, diff.DiffTypeModified, diff.DiffTypeRenamed, diff.DiffTypeDropped, "removed").Build()
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func parseDiffDisplaySettings(apr *argparser.ArgParseResults) *diffDisplaySettings {
|
|
displaySettings := &diffDisplaySettings{}
|
|
|
|
displaySettings.diffParts = SchemaAndDataDiff
|
|
if apr.Contains(cli.DataFlag) && !apr.Contains(cli.SchemaFlag) {
|
|
displaySettings.diffParts = DataOnlyDiff
|
|
} else if apr.Contains(cli.SchemaFlag) && !apr.Contains(cli.DataFlag) {
|
|
displaySettings.diffParts = SchemaOnlyDiff
|
|
} else if apr.Contains(cli.StatFlag) {
|
|
displaySettings.diffParts = Stat
|
|
} else if apr.Contains(cli.SummaryFlag) {
|
|
displaySettings.diffParts = Summary
|
|
} else if apr.Contains(cli.NameOnlyFlag) {
|
|
displaySettings.diffParts = NameOnlyDiff
|
|
}
|
|
|
|
displaySettings.skinny = apr.Contains(cli.SkinnyFlag)
|
|
|
|
if cols, ok := apr.GetValueList(cli.IncludeCols); ok {
|
|
displaySettings.includeCols = cols
|
|
}
|
|
|
|
f := apr.GetValueOrDefault(FormatFlag, "tabular")
|
|
switch strings.ToLower(f) {
|
|
case "tabular":
|
|
displaySettings.diffOutput = TabularDiffOutput
|
|
switch strings.ToLower(apr.GetValueOrDefault(cli.DiffMode, "context")) {
|
|
case "row":
|
|
displaySettings.diffMode = diff.ModeRow
|
|
case "line":
|
|
displaySettings.diffMode = diff.ModeLine
|
|
case "in-place":
|
|
displaySettings.diffMode = diff.ModeInPlace
|
|
case "context":
|
|
displaySettings.diffMode = diff.ModeContext
|
|
}
|
|
case "sql":
|
|
displaySettings.diffOutput = SQLDiffOutput
|
|
case "json":
|
|
displaySettings.diffOutput = JsonDiffOutput
|
|
}
|
|
|
|
displaySettings.limit, _ = apr.GetInt(cli.LimitParam)
|
|
displaySettings.where = apr.GetValueOrDefault(cli.WhereParam, "")
|
|
|
|
filterValue := apr.GetValueOrDefault(cli.FilterParam, diff.DiffTypeAll)
|
|
displaySettings.filter = newDiffTypeFilter(filterValue)
|
|
|
|
return displaySettings
|
|
}
|
|
|
|
func parseDiffArgs(queryist cli.Queryist, sqlCtx *sql.Context, apr *argparser.ArgParseResults) (*diffArgs, error) {
|
|
dArgs := &diffArgs{
|
|
diffDisplaySettings: parseDiffDisplaySettings(apr),
|
|
}
|
|
|
|
staged := apr.Contains(cli.StagedFlag) || apr.Contains(cli.CachedFlag)
|
|
|
|
tableNames, err := dArgs.applyDiffRoots(queryist, sqlCtx, apr.Args, staged, apr.Contains(cli.MergeBase))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if apr.Contains(cli.ReverseFlag) {
|
|
dArgs.diffDatasets = &diffDatasets{
|
|
fromRef: dArgs.toRef,
|
|
toRef: dArgs.fromRef,
|
|
}
|
|
}
|
|
|
|
tableSet, err := parseDiffTableSetSql(queryist, sqlCtx, dArgs.diffDatasets, tableNames)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
dArgs.tableSet = tableSet
|
|
|
|
return dArgs, nil
|
|
}
|
|
|
|
func parseDiffTableSetSql(queryist cli.Queryist, sqlCtx *sql.Context, datasets *diffDatasets, tableNames []string) (*set.StrSet, error) {
|
|
|
|
tablesAtFromRef, err := getTableNamesAtRef(queryist, sqlCtx, datasets.fromRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tablesAtToRef, err := getTableNamesAtRef(queryist, sqlCtx, datasets.toRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
tableSet := set.NewStrSet(nil)
|
|
|
|
for _, tableName := range tableNames {
|
|
// verify table args exist in at least one root
|
|
_, ok := tablesAtFromRef[tableName]
|
|
if ok {
|
|
tableSet.Add(tableName)
|
|
continue
|
|
}
|
|
|
|
_, ok = tablesAtToRef[tableName]
|
|
if ok {
|
|
tableSet.Add(tableName)
|
|
continue
|
|
}
|
|
|
|
return nil, fmt.Errorf("table %s does not exist in either revision", tableName)
|
|
}
|
|
|
|
// if no tables or docs were specified as args, diff all tables and docs
|
|
if len(tableNames) == 0 {
|
|
seenTableNames := make(map[string]bool)
|
|
for _, tables := range []map[string]bool{tablesAtFromRef, tablesAtToRef} {
|
|
for tableName := range tables {
|
|
if _, ok := seenTableNames[tableName]; !ok {
|
|
seenTableNames[tableName] = true
|
|
tableSet.Add(tableName)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return tableSet, nil
|
|
}
|
|
|
|
var doltSystemTables = []string{
|
|
"dolt_procedures",
|
|
"dolt_schemas",
|
|
"dolt_tests",
|
|
}
|
|
|
|
func getTableNamesAtRef(queryist cli.Queryist, sqlCtx *sql.Context, ref string) (map[string]bool, error) {
|
|
// query for user-created tables
|
|
q, err := dbr.InterpolateForDialect("SHOW FULL TABLES AS OF ?", []interface{}{ref}, dialect.MySQL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
rows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tableNames := make(map[string]bool)
|
|
for _, row := range rows {
|
|
tableName, ok, err := sql.Unwrap[string](sqlCtx, row[0])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !ok {
|
|
return nil, fmt.Errorf("unexpected type for table name, expected string, found %T", row[0])
|
|
}
|
|
tableType, ok, err := sql.Unwrap[string](sqlCtx, row[1])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !ok {
|
|
return nil, fmt.Errorf("unexpected type for table type, expected string, found %T", row[1])
|
|
}
|
|
isTable := tableType == "BASE TABLE"
|
|
if isTable {
|
|
tableNames[tableName] = true
|
|
}
|
|
}
|
|
|
|
// add system tables, if they exist at this ref
|
|
for _, sysTable := range doltSystemTables {
|
|
interpolatedQuery, err := dbr.InterpolateForDialect("SHOW CREATE TABLE ? AS OF ?", []interface{}{dbr.I(sysTable), ref}, dialect.MySQL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
result, err := cli.GetRowsForSql(queryist, sqlCtx, interpolatedQuery)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error getting system table %s: %w", sysTable, err)
|
|
}
|
|
|
|
if len(result) > 0 {
|
|
tableNames[sysTable] = true
|
|
}
|
|
}
|
|
|
|
return tableNames, nil
|
|
}
|
|
|
|
// applyDiffRoots applies the appropriate |from| and |to| root values to the receiver and returns the table names
|
|
// (if any) given to the command.
|
|
func (dArgs *diffArgs) applyDiffRoots(queryist cli.Queryist, sqlCtx *sql.Context, args []string, isCached, useMergeBase bool) ([]string, error) {
|
|
dArgs.diffDatasets = &diffDatasets{
|
|
fromRef: doltdb.Staged,
|
|
toRef: doltdb.Working,
|
|
}
|
|
|
|
if isCached {
|
|
dArgs.fromRef = "HEAD"
|
|
dArgs.toRef = doltdb.Staged
|
|
}
|
|
|
|
if len(args) == 0 {
|
|
if useMergeBase {
|
|
return nil, errors.New("Must supply at least one revision when using --merge-base flag")
|
|
}
|
|
// `dolt diff`
|
|
return nil, nil
|
|
}
|
|
|
|
if strings.Contains(args[0], "..") {
|
|
if useMergeBase {
|
|
return nil, errors.New("Cannot use `..` or `...` with --merge-base flag")
|
|
}
|
|
err := dArgs.applyDotRevisions(queryist, sqlCtx, args)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return args[1:], err
|
|
}
|
|
|
|
fromRef := args[0]
|
|
// treat the first arg as a ref spec
|
|
_, err := getTableNamesAtRef(queryist, sqlCtx, fromRef)
|
|
if errors.Is(err, doltdb.ErrGhostCommitEncountered) {
|
|
return nil, err
|
|
}
|
|
// if it doesn't resolve, treat it as a table name
|
|
if err != nil {
|
|
// `dolt diff table`
|
|
if useMergeBase {
|
|
return nil, errors.New("Must supply at least one revision when using --merge-base flag")
|
|
}
|
|
return args, nil
|
|
}
|
|
dArgs.fromRef = fromRef
|
|
|
|
if len(args) == 1 {
|
|
// `dolt diff from_commit`
|
|
if useMergeBase {
|
|
err := dArgs.applyMergeBase(queryist, sqlCtx, args[0], "HEAD")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return nil, nil
|
|
}
|
|
|
|
toRef := args[1]
|
|
// treat the first arg as a ref spec
|
|
_, err = getTableNamesAtRef(queryist, sqlCtx, toRef)
|
|
// if it doesn't resolve, treat it as a table name
|
|
if err != nil {
|
|
// `dolt diff from_commit [...tables]`
|
|
if useMergeBase {
|
|
err := dArgs.applyMergeBase(queryist, sqlCtx, args[0], "HEAD")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return args[1:], nil
|
|
}
|
|
dArgs.toRef = toRef
|
|
|
|
if useMergeBase {
|
|
err := dArgs.applyMergeBase(queryist, sqlCtx, args[0], args[1])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
// `dolt diff from_commit to_commit [...tables]`
|
|
return args[2:], nil
|
|
}
|
|
|
|
// applyMergeBase applies the merge base of two revisions to the |from| root
|
|
// values.
|
|
func (dArgs *diffArgs) applyMergeBase(queryist cli.Queryist, sqlCtx *sql.Context, leftStr, rightStr string) error {
|
|
mergeBaseStr, err := getCommonAncestor(queryist, sqlCtx, leftStr, rightStr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dArgs.fromRef = mergeBaseStr
|
|
|
|
return nil
|
|
}
|
|
|
|
func getCommonAncestor(queryist cli.Queryist, sqlCtx *sql.Context, c1, c2 string) (string, error) {
|
|
q, err := dbr.InterpolateForDialect("select dolt_merge_base(?, ?)", []interface{}{c1, c2}, dialect.MySQL)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
rows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
if len(rows) != 1 {
|
|
return "", errors.New("unexpected number of rows returned from dolt_merge_base")
|
|
}
|
|
ancestor := rows[0][0].(string)
|
|
return ancestor, nil
|
|
}
|
|
|
|
// applyDotRevisions applies the appropriate |from| and |to| root values to the
|
|
// receiver for arguments containing `..` or `...`
|
|
func (dArgs *diffArgs) applyDotRevisions(queryist cli.Queryist, sqlCtx *sql.Context, args []string) error {
|
|
// `dolt diff from_commit...to_commit [...tables]`
|
|
if strings.Contains(args[0], "...") {
|
|
refs := strings.Split(args[0], "...")
|
|
|
|
if len(refs[0]) > 0 {
|
|
right := refs[1]
|
|
// Use current HEAD if right side of `...` does not exist
|
|
if len(refs[1]) == 0 {
|
|
right = "HEAD"
|
|
}
|
|
|
|
err := dArgs.applyMergeBase(queryist, sqlCtx, refs[0], right)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
if len(refs[1]) > 0 {
|
|
dArgs.toRef = refs[1]
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// `dolt diff from_commit..to_commit [...tables]`
|
|
if strings.Contains(args[0], "..") {
|
|
refs := strings.Split(args[0], "..")
|
|
|
|
if len(refs[0]) > 0 {
|
|
dArgs.fromRef = refs[0]
|
|
}
|
|
|
|
if len(refs[1]) > 0 {
|
|
dArgs.toRef = refs[1]
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
var diffSummarySchema = sql.Schema{
|
|
&sql.Column{Name: "Table name", Type: types.Text, Nullable: false},
|
|
&sql.Column{Name: "Diff type", Type: types.Text, Nullable: false},
|
|
&sql.Column{Name: "Data change", Type: types.Boolean, Nullable: false},
|
|
&sql.Column{Name: "Schema change", Type: types.Boolean, Nullable: false},
|
|
}
|
|
|
|
func printDiffSummary(ctx *sql.Context, diffSummaries []diff.TableDeltaSummary, dArgs *diffArgs) errhand.VerboseError {
|
|
cliWR := iohelp.NopWrCloser(cli.OutStream)
|
|
wr := tabular.NewFixedWidthTableWriter(diffSummarySchema, cliWR, 100)
|
|
defer wr.Close(ctx)
|
|
|
|
for _, diffSummary := range diffSummaries {
|
|
|
|
// TODO: schema name
|
|
shouldPrintTables := dArgs.tableSet.Contains(diffSummary.FromTableName.Name) || dArgs.tableSet.Contains(diffSummary.ToTableName.Name)
|
|
if !shouldPrintTables {
|
|
return nil
|
|
}
|
|
|
|
tableName := diffSummary.TableName.Name
|
|
if diffSummary.DiffType == "renamed" {
|
|
tableName = fmt.Sprintf("%s -> %s", diffSummary.FromTableName.Name, diffSummary.ToTableName.Name)
|
|
}
|
|
err := wr.WriteSqlRow(ctx, sql.Row{tableName, diffSummary.DiffType, diffSummary.DataChange, diffSummary.SchemaChange})
|
|
if err != nil {
|
|
return errhand.BuildDError("could not write table delta summary").AddCause(err).Build()
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getDeltasBetweenRefs(queryist cli.Queryist, sqlCtx *sql.Context, fromRef, toRef string) ([]diff.TableDeltaSummary, error) {
|
|
diffSummaries, err := getDiffSummariesBetweenRefs(queryist, sqlCtx, fromRef, toRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
schemaSummaries, err := getSchemaDiffSummariesBetweenRefs(queryist, sqlCtx, fromRef, toRef)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
allSummaries := []diff.TableDeltaSummary{}
|
|
allSummaries = append(allSummaries, diffSummaries...)
|
|
|
|
for _, schemaSummary := range schemaSummaries {
|
|
deltaExists := false
|
|
for i, summary := range allSummaries {
|
|
deltaExists = summary.FromTableName == schemaSummary.FromTableName && summary.ToTableName == schemaSummary.ToTableName
|
|
if deltaExists {
|
|
existingSummary := allSummaries[i]
|
|
existingSummary.SchemaChange = true
|
|
existingSummary.AlterStmts = append(existingSummary.AlterStmts, schemaSummary.AlterStmts...)
|
|
allSummaries[i] = existingSummary
|
|
break
|
|
}
|
|
}
|
|
if !deltaExists {
|
|
allSummaries = append(allSummaries, schemaSummary)
|
|
}
|
|
}
|
|
|
|
return allSummaries, nil
|
|
}
|
|
|
|
func getSchemaDiffSummariesBetweenRefs(queryist cli.Queryist, sqlCtx *sql.Context, fromRef, toRef string) ([]diff.TableDeltaSummary, error) {
|
|
q, err := dbr.InterpolateForDialect("select * from dolt_schema_diff(?, ?)", []interface{}{fromRef, toRef}, dialect.MySQL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to interpolate query: %w", err)
|
|
}
|
|
schemaDiffRows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to get schema diff from %s to %s: %w", fromRef, toRef, err)
|
|
}
|
|
|
|
var summaries []diff.TableDeltaSummary
|
|
for _, row := range schemaDiffRows {
|
|
fromTable := row[0].(string)
|
|
toTable := row[1].(string)
|
|
fromCreateStmt := row[2].(string)
|
|
toCreateStmt := row[3].(string)
|
|
var diffType = ""
|
|
var tableName = ""
|
|
switch {
|
|
case fromTable == toTable:
|
|
if fromCreateStmt != toCreateStmt {
|
|
diffType = "modified"
|
|
tableName = fromTable
|
|
}
|
|
case fromTable == "":
|
|
diffType = diff.DiffTypeAdded
|
|
tableName = toTable
|
|
case toTable == "":
|
|
diffType = diff.DiffTypeDropped
|
|
tableName = fromTable
|
|
case fromTable != "" && toTable != "" && fromTable != toTable:
|
|
diffType = diff.DiffTypeRenamed
|
|
tableName = toTable
|
|
default:
|
|
return nil, fmt.Errorf("error: unexpected schema diff case: fromTable='%s', toTable='%s'", fromTable, toTable)
|
|
}
|
|
|
|
q, err := dbr.InterpolateForDialect(
|
|
"select statement_order, statement from dolt_patch(?, ?) where diff_type='schema' and (table_name=? or table_name=?) order by statement_order asc",
|
|
[]interface{}{fromRef, toRef, fromTable, toTable},
|
|
dialect.MySQL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to interpolate dolt_patch query: %w", err)
|
|
}
|
|
patchRows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to get dolt_patch rows from %s to %s: %w", fromRef, toRef, err)
|
|
}
|
|
alterStmts := []string{}
|
|
for _, row := range patchRows {
|
|
alterStmts = append(alterStmts, row[1].(string))
|
|
}
|
|
|
|
summary := diff.TableDeltaSummary{
|
|
TableName: doltdb.TableName{Name: tableName},
|
|
FromTableName: doltdb.TableName{Name: fromTable},
|
|
ToTableName: doltdb.TableName{Name: toTable},
|
|
DiffType: diffType,
|
|
DataChange: false,
|
|
SchemaChange: true,
|
|
AlterStmts: alterStmts,
|
|
}
|
|
|
|
summaries = append(summaries, summary)
|
|
}
|
|
|
|
return summaries, nil
|
|
}
|
|
|
|
func getDiffSummariesBetweenRefs(queryist cli.Queryist, sqlCtx *sql.Context, fromRef, toRef string) ([]diff.TableDeltaSummary, error) {
|
|
q, err := dbr.InterpolateForDialect("select * from dolt_diff_summary(?, ?)", []interface{}{fromRef, toRef}, dialect.MySQL)
|
|
dataDiffRows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to get diff summary from %s to %s: %w", fromRef, toRef, err)
|
|
}
|
|
|
|
summaries := []diff.TableDeltaSummary{}
|
|
|
|
for _, row := range dataDiffRows {
|
|
summary := diff.TableDeltaSummary{}
|
|
summary.FromTableName.Name = row[0].(string)
|
|
summary.ToTableName.Name = row[1].(string)
|
|
summary.DiffType = row[2].(string)
|
|
summary.DataChange, err = cli.QueryValueAsBool(row[3])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to parse data change value '%s': %w", row[3], err)
|
|
}
|
|
summary.SchemaChange, err = cli.QueryValueAsBool(row[4])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error: unable to parse schema change value '%s': %w", row[4], err)
|
|
}
|
|
|
|
switch summary.DiffType {
|
|
case diff.DiffTypeDropped:
|
|
summary.TableName = summary.FromTableName
|
|
case diff.DiffTypeAdded:
|
|
summary.TableName = summary.ToTableName
|
|
case diff.DiffTypeRenamed:
|
|
summary.TableName = summary.ToTableName
|
|
case diff.DiffTypeModified:
|
|
summary.TableName = summary.FromTableName
|
|
default:
|
|
return nil, fmt.Errorf("error: unexpected diff type '%s'", summary.DiffType)
|
|
}
|
|
|
|
summaries = append(summaries, summary)
|
|
}
|
|
|
|
return summaries, nil
|
|
}
|
|
|
|
func diffUserTables(queryist cli.Queryist, sqlCtx *sql.Context, dArgs *diffArgs) errhand.VerboseError {
|
|
var err error
|
|
|
|
deltas, err := getDeltasBetweenRefs(queryist, sqlCtx, dArgs.fromRef, dArgs.toRef)
|
|
if err != nil {
|
|
return errhand.BuildDError("error: unable to get diff summary").AddCause(err).Build()
|
|
}
|
|
|
|
// Validate where clause even if there are no deltas
|
|
if len(dArgs.where) > 0 && len(deltas) == 0 {
|
|
if verr := validateWhereClause(queryist, sqlCtx, dArgs); verr != nil {
|
|
return verr
|
|
}
|
|
}
|
|
|
|
if dArgs.diffParts&Summary != 0 {
|
|
return printDiffSummary(sqlCtx, deltas, dArgs)
|
|
}
|
|
|
|
// Ignore patterns govern the staging of untracked tables and do not apply to committed history,
|
|
// so they are only loaded when the diff references the working set or staging area.
|
|
var ignoredTablePatterns doltdb.IgnorePatterns
|
|
if dArgs.hasWorkingSet() {
|
|
ignoredTablePatterns, err = getIgnoredTablePatternsFromSql(queryist, sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(fmt.Errorf("couldn't get ignored table patterns, cause: %w", err))
|
|
}
|
|
}
|
|
|
|
dw, err := newDiffWriter(dArgs.diffOutput)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
doltSchemasChanged := false
|
|
var hasdiffErrors bool
|
|
for _, delta := range deltas {
|
|
if doltdb.IsFullTextTable(delta.TableName.Name) {
|
|
continue
|
|
}
|
|
|
|
ignore, err := ignoredTablePatterns.ShouldIgnoreDelta(delta.IsAdd(), delta.IsDrop(), delta.ToTableName, delta.FromTableName)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
if ignore {
|
|
continue
|
|
}
|
|
|
|
if !shouldPrintTableDelta(dArgs.tableSet, delta.ToTableName.Name, delta.FromTableName.Name) {
|
|
continue
|
|
}
|
|
|
|
// Apply table-level filtering based on diff type
|
|
if dArgs.filter != nil && dArgs.filter.filters != nil {
|
|
// For data-only changes (no schema/rename), always let them through for row-level filtering
|
|
isDataOnlyChange := !delta.SchemaChange && !delta.IsRename() && delta.DataChange
|
|
|
|
if !isDataOnlyChange && !dArgs.filter.shouldInclude(delta.DiffType) {
|
|
continue // Skip this table
|
|
}
|
|
}
|
|
|
|
if strings.HasPrefix(delta.ToTableName.Name, diff.DBPrefix) {
|
|
verr := diffDatabase(queryist, sqlCtx, delta, dArgs, dw)
|
|
if verr != nil {
|
|
return verr
|
|
}
|
|
continue
|
|
}
|
|
|
|
if isDoltSchemasTable(delta.ToTableName.Name, delta.FromTableName.Name) {
|
|
// save dolt_schemas table diff for last in diff output
|
|
doltSchemasChanged = true
|
|
} else {
|
|
verr := diffUserTable(queryist, sqlCtx, delta, dArgs, dw)
|
|
if verr != nil {
|
|
cli.PrintErrln(verr.Verbose())
|
|
hasdiffErrors = true
|
|
}
|
|
}
|
|
}
|
|
|
|
if doltSchemasChanged {
|
|
verr := diffDoltSchemasTable(queryist, sqlCtx, dArgs, dw)
|
|
if verr != nil {
|
|
return verr
|
|
}
|
|
}
|
|
|
|
err = dw.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
if hasdiffErrors {
|
|
errorBuilder := errhand.BuildDError("error: encountered errors during diff")
|
|
return errorBuilder.Build()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func shouldPrintTableDelta(tablesToPrint *set.StrSet, toTableName, fromTableName string) bool {
|
|
// TODO: this should be case insensitive
|
|
return tablesToPrint.Contains(fromTableName) ||
|
|
tablesToPrint.Contains(toTableName) ||
|
|
strings.HasPrefix(fromTableName, diff.DBPrefix) ||
|
|
strings.HasPrefix(toTableName, diff.DBPrefix)
|
|
}
|
|
|
|
func isDoltSchemasTable(toTableName, fromTableName string) bool {
|
|
return fromTableName == doltdb.SchemasTableName || toTableName == doltdb.SchemasTableName
|
|
}
|
|
|
|
func getTableInfoAtRef(queryist cli.Queryist, sqlCtx *sql.Context, tableName string, ref string) (diff.TableInfo, error) {
|
|
sch, createStmt, err := getTableSchemaAtRef(queryist, sqlCtx, tableName, ref)
|
|
if err != nil {
|
|
return diff.TableInfo{}, fmt.Errorf("error: unable to get schema for table '%s': %w", tableName, err)
|
|
}
|
|
|
|
tableInfo := diff.TableInfo{
|
|
Name: tableName,
|
|
Sch: sch,
|
|
CreateStmt: createStmt,
|
|
}
|
|
return tableInfo, nil
|
|
}
|
|
|
|
func getTableSchemaAtRef(queryist cli.Queryist, sqlCtx *sql.Context, tableName string, ref string) (sch schema.Schema, createStmt string, err error) {
|
|
var rows []sql.Row
|
|
interpolatedQuery, err := dbr.InterpolateForDialect("SHOW CREATE TABLE ? AS OF ?", []interface{}{dbr.I(tableName), ref}, dialect.MySQL)
|
|
if err != nil {
|
|
return sch, createStmt, fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
rows, err = cli.GetRowsForSql(queryist, sqlCtx, interpolatedQuery)
|
|
if err != nil {
|
|
return sch, createStmt, fmt.Errorf("error: unable to get create table statement for table '%s': %w", tableName, err)
|
|
}
|
|
|
|
if len(rows) != 1 {
|
|
return sch, createStmt, fmt.Errorf("creating schema, expected 1 row, got %d", len(rows))
|
|
}
|
|
createStmt = rows[0][1].(string)
|
|
|
|
// append ; at the end, if one isn't there yet
|
|
if createStmt[len(createStmt)-1] != ';' {
|
|
createStmt += ";"
|
|
}
|
|
|
|
sch, err = schemaFromCreateTableStmt(createStmt)
|
|
if err != nil {
|
|
return sch, createStmt, err
|
|
}
|
|
|
|
return sch, createStmt, nil
|
|
}
|
|
|
|
func getDatabaseInfoAtRef(queryist cli.Queryist, sqlCtx *sql.Context, tableName string, ref string) (diff.TableInfo, error) {
|
|
createStmt, err := getDatabaseSchemaAtRef(queryist, sqlCtx, tableName, ref)
|
|
if err != nil {
|
|
return diff.TableInfo{}, fmt.Errorf("error: unable to get schema for table '%s': %w", tableName, err)
|
|
}
|
|
|
|
tableInfo := diff.TableInfo{
|
|
Name: tableName,
|
|
Sch: schema.EmptySchema,
|
|
CreateStmt: createStmt,
|
|
}
|
|
return tableInfo, nil
|
|
}
|
|
|
|
func getDatabaseSchemaAtRef(queryist cli.Queryist, sqlCtx *sql.Context, tableName string, ref string) (string, error) {
|
|
var rows []sql.Row
|
|
// TODO: implement `show create database as of ...`
|
|
tableName = strings.TrimPrefix(tableName, diff.DBPrefix)
|
|
interpolatedQuery, err := dbr.InterpolateForDialect("SHOW CREATE DATABASE ?", []interface{}{dbr.I(tableName)}, dialect.MySQL)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
rows, err = cli.GetRowsForSql(queryist, sqlCtx, interpolatedQuery)
|
|
if err != nil {
|
|
return "", fmt.Errorf("error: unable to get create database statement for database '%s': %w", tableName, err)
|
|
}
|
|
if len(rows) != 1 {
|
|
return "", fmt.Errorf("creating schema, expected 1 row, got %d", len(rows))
|
|
}
|
|
createStmt := rows[0][1].(string)
|
|
|
|
// append ; at the end, if one isn't there yet
|
|
if createStmt[len(createStmt)-1] != ';' {
|
|
createStmt += ";"
|
|
}
|
|
|
|
return createStmt, nil
|
|
}
|
|
|
|
// schemaFromCreateTableStmt returns a schema for the CREATE TABLE statement given
|
|
// TODO: this is substantially incorrect, doesn't handle primary key ordering, probably other things too
|
|
func schemaFromCreateTableStmt(createTableStmt string) (schema.Schema, error) {
|
|
parsed, err := ast.Parse(createTableStmt)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
create, ok := parsed.(*ast.DDL)
|
|
if !ok {
|
|
return nil, fmt.Errorf("expected create table, found %T", parsed)
|
|
}
|
|
|
|
primaryCols := make(map[string]bool)
|
|
for _, index := range create.TableSpec.Indexes {
|
|
if index.Info.Primary {
|
|
for _, indexCol := range index.Fields {
|
|
primaryCols[indexCol.Column.Lowered()] = true
|
|
}
|
|
break
|
|
}
|
|
}
|
|
|
|
var cols []schema.Column
|
|
for _, col := range create.TableSpec.Columns {
|
|
internalTyp, err := types.ColumnTypeToType(&col.Type)
|
|
typeInfo, err := typeinfo.FromSqlType(internalTyp)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defBuf := ast.NewTrackedBuffer(nil)
|
|
if col.Type.Default != nil {
|
|
col.Type.Default.Format(defBuf)
|
|
}
|
|
|
|
genBuf := ast.NewTrackedBuffer(nil)
|
|
if col.Type.GeneratedExpr != nil {
|
|
col.Type.GeneratedExpr.Format(genBuf)
|
|
}
|
|
|
|
onUpBuf := ast.NewTrackedBuffer(nil)
|
|
if col.Type.OnUpdate != nil {
|
|
col.Type.OnUpdate.Format(onUpBuf)
|
|
}
|
|
|
|
var comment string
|
|
if col.Type.Comment != nil {
|
|
comment = col.Type.Comment.String()
|
|
}
|
|
|
|
sCol := schema.Column{
|
|
Name: col.Name.String(),
|
|
Kind: typeInfo.NomsKind(),
|
|
IsPartOfPK: primaryCols[col.Name.Lowered()],
|
|
TypeInfo: typeInfo,
|
|
Default: defBuf.String(),
|
|
Generated: "", // TODO
|
|
OnUpdate: "", // TODO
|
|
Virtual: false, // TODO
|
|
AutoIncrement: col.Type.Autoincrement == true,
|
|
Comment: comment,
|
|
}
|
|
cols = append(cols, sCol)
|
|
}
|
|
|
|
sch, err := schema.NewSchema(schema.NewColCollection(cols...), nil, schema.Collation_Default, nil, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return sch, err
|
|
}
|
|
|
|
func getTableDiffStats(queryist cli.Queryist, sqlCtx *sql.Context, tableName, fromRef, toRef string) ([]diffStatistics, error) {
|
|
q, err := dbr.InterpolateForDialect("select * from dolt_diff_stat(?, ?, ?)", []interface{}{fromRef, toRef, tableName}, dialect.MySQL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error interpolating query: %w", err)
|
|
}
|
|
rows, err := cli.GetRowsForSql(queryist, sqlCtx, q)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error running diff stats query: %w", err)
|
|
}
|
|
|
|
allStats := []diffStatistics{}
|
|
for _, row := range rows {
|
|
rowsUnmodified, err := coallesceNilToUint64(row[1])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rowsAdded, err := coallesceNilToUint64(row[2])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rowsDeleted, err := coallesceNilToUint64(row[3])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rowsModified, err := coallesceNilToUint64(row[4])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cellsAdded, err := coallesceNilToUint64(row[5])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cellsDeleted, err := coallesceNilToUint64(row[6])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
cellsModified, err := coallesceNilToUint64(row[7])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
oldRowCount, err := coallesceNilToUint64(row[8])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
newRowCount, err := coallesceNilToUint64(row[9])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
oldCellCount, err := coallesceNilToUint64(row[10])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
newCellCount, err := coallesceNilToUint64(row[11])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
stats := diffStatistics{
|
|
TableName: row[0].(string),
|
|
RowsUnmodified: rowsUnmodified,
|
|
RowsAdded: rowsAdded,
|
|
RowsDeleted: rowsDeleted,
|
|
RowsModified: rowsModified,
|
|
CellsAdded: cellsAdded,
|
|
CellsDeleted: cellsDeleted,
|
|
CellsModified: cellsModified,
|
|
OldRowCount: oldRowCount,
|
|
NewRowCount: newRowCount,
|
|
OldCellCount: oldCellCount,
|
|
NewCellCount: newCellCount,
|
|
}
|
|
allStats = append(allStats, stats)
|
|
}
|
|
return allStats, nil
|
|
}
|
|
|
|
func coallesceNilToUint64(val interface{}) (uint64, error) {
|
|
if val == nil {
|
|
return 0, nil
|
|
}
|
|
return getUint64ColAsUint64(val)
|
|
}
|
|
|
|
func diffUserTable(
|
|
queryist cli.Queryist,
|
|
sqlCtx *sql.Context,
|
|
tableSummary diff.TableDeltaSummary,
|
|
dArgs *diffArgs,
|
|
dw diffWriter,
|
|
) errhand.VerboseError {
|
|
fromTable := tableSummary.FromTableName
|
|
toTable := tableSummary.ToTableName
|
|
|
|
if dArgs.diffParts&NameOnlyDiff == 0 && !shouldUseLazyHeader(dArgs, tableSummary) {
|
|
// TODO: schema names
|
|
err := dw.BeginTable(sqlCtx, tableSummary.FromTableName.Name, tableSummary.ToTableName.Name, tableSummary.IsAdd(), tableSummary.IsDrop())
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
}
|
|
|
|
var fromTableInfo, toTableInfo *diff.TableInfo
|
|
|
|
from, err := getTableInfoAtRef(queryist, sqlCtx, fromTable.Name, dArgs.fromRef)
|
|
if err == nil {
|
|
fromTableInfo = &from
|
|
}
|
|
to, err := getTableInfoAtRef(queryist, sqlCtx, toTable.Name, dArgs.toRef)
|
|
if err == nil {
|
|
toTableInfo = &to
|
|
}
|
|
|
|
tableName := fromTable
|
|
if tableName.Name == "" {
|
|
tableName = toTable
|
|
}
|
|
|
|
if dArgs.diffParts&NameOnlyDiff != 0 {
|
|
cli.Println(tableName)
|
|
return errhand.VerboseErrorFromError(nil)
|
|
}
|
|
|
|
if dArgs.diffParts&Stat != 0 {
|
|
var areTablesKeyless = false
|
|
|
|
var fromColLen = 0
|
|
var fromKeyless = false
|
|
if fromTableInfo != nil {
|
|
fromKeyless = schema.IsKeyless(fromTableInfo.Sch)
|
|
fromColLen = fromTableInfo.Sch.GetAllCols().Size()
|
|
}
|
|
var toColLen = 0
|
|
var toKeyless = false
|
|
if toTableInfo != nil {
|
|
toKeyless = schema.IsKeyless(toTableInfo.Sch)
|
|
toColLen = toTableInfo.Sch.GetAllCols().Size()
|
|
}
|
|
|
|
// nil table is neither keyless nor keyed
|
|
if fromTableInfo == nil {
|
|
areTablesKeyless = toKeyless
|
|
} else if toTableInfo == nil {
|
|
areTablesKeyless = fromKeyless
|
|
} else {
|
|
if fromKeyless && toKeyless {
|
|
areTablesKeyless = true
|
|
} else if !fromKeyless && !toKeyless {
|
|
areTablesKeyless = false
|
|
} else {
|
|
return errhand.BuildDError("mismatched keyless and keyed schemas for table %s", tableName).Build()
|
|
}
|
|
}
|
|
|
|
var diffStats []diffStatistics
|
|
diffStats, err = getTableDiffStats(queryist, sqlCtx, tableName.Name, dArgs.fromRef, dArgs.toRef)
|
|
if err != nil {
|
|
return errhand.BuildDError("cannot retrieve diff stats between '%s' and '%s'", dArgs.fromRef, dArgs.toRef).AddCause(err).Build()
|
|
}
|
|
|
|
err = dw.WriteTableDiffStats(sqlCtx, diffStats, fromColLen, toColLen, areTablesKeyless)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
if dArgs.diffParts&SchemaOnlyDiff != 0 {
|
|
err = dw.WriteTableSchemaDiff(sqlCtx, fromTableInfo, toTableInfo, tableSummary)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
}
|
|
|
|
if tableSummary.IsDrop() && dArgs.diffOutput == SQLDiffOutput {
|
|
return nil // don't output DELETE FROM statements after DROP TABLE
|
|
}
|
|
|
|
verr := diffRows(queryist, sqlCtx, tableSummary, fromTableInfo, toTableInfo, dArgs, dw)
|
|
if verr != nil {
|
|
return verr
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func diffDoltSchemasTable(
|
|
queryist cli.Queryist,
|
|
sqlCtx *sql.Context,
|
|
dArgs *diffArgs,
|
|
dw diffWriter,
|
|
) errhand.VerboseError {
|
|
query, err := dbr.InterpolateForDialect("select from_name,to_name,from_type,to_type,from_fragment,to_fragment "+
|
|
"from dolt_diff(?, ?, ?) "+
|
|
"order by coalesce(from_type, to_type), coalesce(from_name, to_name)",
|
|
[]interface{}{dArgs.fromRef, dArgs.toRef, doltdb.SchemasTableName}, dialect.MySQL)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error building diff query").AddCause(err).Build()
|
|
}
|
|
|
|
_, rowIter, _, err := queryist.Query(sqlCtx, query)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error running diff query:\n%s", query).AddCause(err).Build()
|
|
}
|
|
|
|
defer rowIter.Close(sqlCtx)
|
|
for {
|
|
row, err := rowIter.Next(sqlCtx)
|
|
if err == io.EOF {
|
|
break
|
|
} else if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
var vErr errhand.VerboseError
|
|
unwrapString := func(val interface{}) (string, errhand.VerboseError) {
|
|
s, ok, err := sql.Unwrap[string](sqlCtx, val)
|
|
if err != nil {
|
|
return "", errhand.VerboseErrorFromError(err)
|
|
}
|
|
if !ok {
|
|
return "", errhand.BuildDError("expected string, got %T", val).Build()
|
|
}
|
|
return s, nil
|
|
}
|
|
var fragmentName string
|
|
if row[0] != nil {
|
|
fragmentName, vErr = unwrapString(row[0])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
} else {
|
|
fragmentName, vErr = unwrapString(row[1])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
}
|
|
|
|
var fragmentType string
|
|
if row[2] != nil {
|
|
fragmentType, vErr = unwrapString(row[2])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
} else {
|
|
fragmentType, vErr = unwrapString(row[3])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
}
|
|
|
|
var oldFragment string
|
|
var newFragment string
|
|
if row[4] != nil {
|
|
oldFragment, vErr = unwrapString(row[4])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
// Typically schema fragments have the semicolons stripped, so put it back on
|
|
if len(oldFragment) > 0 && oldFragment[len(oldFragment)-1] != ';' {
|
|
oldFragment += ";"
|
|
}
|
|
}
|
|
if row[5] != nil {
|
|
newFragment, vErr = unwrapString(row[5])
|
|
if vErr != nil {
|
|
return vErr
|
|
}
|
|
// Typically schema fragments have the semicolons stripped, so put it back on
|
|
if len(newFragment) > 0 && newFragment[len(newFragment)-1] != ';' {
|
|
newFragment += ";"
|
|
}
|
|
}
|
|
|
|
switch fragmentType {
|
|
case "event":
|
|
err := dw.WriteEventDiff(sqlCtx, fragmentName, oldFragment, newFragment)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
case "trigger":
|
|
err := dw.WriteTriggerDiff(sqlCtx, fragmentName, oldFragment, newFragment)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
case "view":
|
|
err := dw.WriteViewDiff(sqlCtx, fragmentName, oldFragment, newFragment)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
default:
|
|
cli.PrintErrf("Unrecognized schema element type: %s", fragmentType)
|
|
continue
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func diffDatabase(
|
|
queryist cli.Queryist,
|
|
sqlCtx *sql.Context,
|
|
tableSummary diff.TableDeltaSummary,
|
|
dArgs *diffArgs,
|
|
dw diffWriter,
|
|
) errhand.VerboseError {
|
|
if dArgs.diffParts&NameOnlyDiff != 0 {
|
|
cli.Println(tableSummary.FromTableName)
|
|
return nil
|
|
}
|
|
|
|
err := dw.BeginTable(sqlCtx, tableSummary.FromTableName.Name, tableSummary.ToTableName.Name, tableSummary.IsAdd(), tableSummary.IsDrop())
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
if dArgs.diffParts&SchemaOnlyDiff == 0 {
|
|
return nil
|
|
}
|
|
|
|
fromTable := tableSummary.FromTableName
|
|
var fromTableInfo *diff.TableInfo
|
|
from, err := getDatabaseInfoAtRef(queryist, sqlCtx, fromTable.Name, dArgs.fromRef)
|
|
if err == nil {
|
|
// TODO: implement show create database as of ...
|
|
fromTableInfo = &from
|
|
fromTableInfo.CreateStmt = ""
|
|
}
|
|
|
|
toTable := tableSummary.ToTableName
|
|
var toTableInfo *diff.TableInfo
|
|
to, err := getDatabaseInfoAtRef(queryist, sqlCtx, toTable.Name, dArgs.toRef)
|
|
if err == nil {
|
|
toTableInfo = &to
|
|
}
|
|
|
|
err = dw.WriteTableSchemaDiff(sqlCtx, fromTableInfo, toTableInfo, tableSummary)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
verr := diffRows(queryist, sqlCtx, tableSummary, fromTableInfo, toTableInfo, dArgs, dw)
|
|
if verr != nil {
|
|
return verr
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// areValueSetsDiffable checks if it's possible to meaningfully diff the tuples stored in a table
|
|
// between two commits. This is possible for some schema changes but not others.
|
|
func areValueSetsDiffable(fromSch, toSch schema.Schema) bool {
|
|
if fromSch == nil || fromSch.GetAllCols().Size() == 0 ||
|
|
toSch == nil || toSch.GetAllCols().Size() == 0 {
|
|
// If one side doesn't have a schema, the diff is trivial.
|
|
return true
|
|
}
|
|
|
|
fromCols := fromSch.GetNonPKCols()
|
|
toCols := toSch.GetNonPKCols()
|
|
|
|
var sharedCols int
|
|
if fromCols.Size() < toCols.Size() {
|
|
sharedCols = fromCols.Size()
|
|
for i := fromCols.Size(); i < toCols.Size(); i++ {
|
|
if toCols.GetColumns()[i].Default != "" {
|
|
// If a column with a default value was added, it is not possible to generate a data diff
|
|
// without running the schema change in a SQL engine.
|
|
return false
|
|
}
|
|
}
|
|
} else {
|
|
sharedCols = toCols.Size()
|
|
}
|
|
|
|
compatChecker := typecompatibility.NewTypeCompatabilityChecker()
|
|
for i := 0; i < sharedCols; i++ {
|
|
fromCol := fromCols.GetByIndex(i)
|
|
toCol := toCols.GetByIndex(i)
|
|
// TODO: We might get more accurate results from calling MapSchemaBasedOnTagAndName
|
|
if fromCol.Name != toCol.Name && fromCol.Tag != toCol.Tag {
|
|
// If fromCol.Tag == toCol.Tag, this will display a rename.
|
|
// We ought to be able to generate a data diff here, but we currently panic if a row has changed a value in this column.
|
|
// TODO: Generate SQL data diffs for renamed columns.
|
|
|
|
// If fromCol.Tag == toCol.Tag, this will display a drop and an add.
|
|
// In this case, we would need to generate an update statement for every row where this new column is not NULL.
|
|
// We cannot currently do this for rows whose bytes have not changed.
|
|
return false
|
|
}
|
|
typeChangeInfo := compatChecker.IsTypeChangeCompatible(fromCol.TypeInfo, toCol.TypeInfo)
|
|
binaryCompatible := typeChangeInfo.Compatible && !typeChangeInfo.RewriteRows
|
|
if !binaryCompatible {
|
|
// It is not possible to semantically compare the values of this column at these two commits.
|
|
// This means that it's possible for the column to have the same bytes at both commits, while not
|
|
// representing the same value. We do not have enough context to produce a correct data diff.
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func diffRows(
|
|
queryist cli.Queryist,
|
|
sqlCtx *sql.Context,
|
|
tableSummary diff.TableDeltaSummary,
|
|
fromTableInfo, toTableInfo *diff.TableInfo,
|
|
dArgs *diffArgs,
|
|
dw diffWriter,
|
|
) errhand.VerboseError {
|
|
var fromSch schema.Schema = nil
|
|
var toSch schema.Schema = nil
|
|
if fromTableInfo != nil {
|
|
fromSch = fromTableInfo.Sch
|
|
}
|
|
if toTableInfo != nil {
|
|
toSch = toTableInfo.Sch
|
|
}
|
|
|
|
var toSqlSch, fromSqlSch sql.Schema
|
|
if fromTableInfo != nil {
|
|
pkSch, err := sqlutil.FromDoltSchema(sqlCtx, sqlCtx.GetCurrentDatabase(), fromTableInfo.Name, fromSch)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
fromSqlSch = pkSch.Schema
|
|
}
|
|
if toTableInfo != nil {
|
|
pkSch, err := sqlutil.FromDoltSchema(sqlCtx, sqlCtx.GetCurrentDatabase(), toTableInfo.Name, toSch)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
toSqlSch = pkSch.Schema
|
|
}
|
|
|
|
var unionSch sql.Schema
|
|
if dArgs.diffOutput == SQLDiffOutput {
|
|
// When generating SQL output, it doesn't make sense to generate a data diff for columns that no longer exist.
|
|
// Thus, we always use the schema on the target commit.
|
|
unionSch = toSqlSch
|
|
} else if fromSqlSch.Equals(toSqlSch) {
|
|
unionSch = fromSqlSch
|
|
} else {
|
|
unionSch = unionSchemas(fromSqlSch, toSqlSch)
|
|
}
|
|
|
|
// We always instantiate a RowWriter in case the diffWriter needs it to close off any work from schema output
|
|
var rowWriter diff.SqlRowDiffWriter
|
|
realWriter, err := dw.RowWriter(sqlCtx, fromTableInfo, toTableInfo, tableSummary, unionSch)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
|
|
if shouldUseLazyHeader(dArgs, tableSummary) {
|
|
// Wrap with lazy writer to delay BeginTable until first row write
|
|
onFirstWrite := func() error {
|
|
return dw.BeginTable(
|
|
sqlCtx,
|
|
tableSummary.FromTableName.Name,
|
|
tableSummary.ToTableName.Name,
|
|
tableSummary.IsAdd(),
|
|
tableSummary.IsDrop(),
|
|
)
|
|
}
|
|
rowWriter = newLazyRowWriter(realWriter, onFirstWrite)
|
|
} else {
|
|
rowWriter = realWriter
|
|
}
|
|
|
|
if !schema.ArePrimaryKeySetsDiffable(fromSch, toSch) {
|
|
err := rowWriter.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
err = fmt.Errorf("Primary key sets differ between revisions for table '%s', skipping data diff", tableSummary.ToTableName)
|
|
// When outputting SQL, it's an error if we can't generate a data diff. Otherwise we write to stderr and continue.
|
|
if dArgs.diffOutput == SQLDiffOutput {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
cli.PrintErrln(err)
|
|
return nil
|
|
}
|
|
|
|
if dArgs.diffOutput == SQLDiffOutput && !areValueSetsDiffable(fromSch, toSch) {
|
|
err := rowWriter.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
return errhand.VerboseErrorFromError(fmt.Errorf("Incompatible schema change, skipping data diff for table '%s'", tableSummary.ToTableName))
|
|
}
|
|
|
|
// no data diff requested
|
|
if dArgs.diffParts&DataOnlyDiff == 0 {
|
|
err = rowWriter.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// do the data diff
|
|
// TODO: schema names
|
|
tableName := tableSummary.ToTableName.Name
|
|
if len(tableName) == 0 {
|
|
tableName = tableSummary.FromTableName.Name
|
|
}
|
|
|
|
if strings.HasPrefix(tableName, diff.DBPrefix) {
|
|
err = rowWriter.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
defer rowWriter.Close(sqlCtx)
|
|
|
|
columnNames, format := getColumnNames(fromTableInfo, toTableInfo)
|
|
query := fmt.Sprintf("select %s ? from dolt_diff(?, ?, ?)", format)
|
|
var params []interface{}
|
|
for _, col := range columnNames {
|
|
params = append(params, dbr.I(col))
|
|
}
|
|
params = append(params, dbr.I("diff_type"), dArgs.fromRef, dArgs.toRef, tableName)
|
|
|
|
if len(dArgs.where) > 0 {
|
|
query += " where ?"
|
|
params = append(params, dbr.Expr(dArgs.where))
|
|
}
|
|
|
|
if dArgs.limit >= 0 {
|
|
query += " limit ?"
|
|
params = append(params, dbr.Expr(strconv.Itoa(dArgs.limit)))
|
|
}
|
|
|
|
interpolatedQuery, err := dbr.InterpolateForDialect(query, params, dialect.MySQL)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error building diff query:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
|
|
sch, rowIter, _, err := queryist.Query(sqlCtx, interpolatedQuery)
|
|
if sql.ErrSyntaxError.Is(err) {
|
|
return errhand.BuildDError("Failed to parse diff query. Invalid where clause?\nDiff query: %s", interpolatedQuery).AddCause(err).Build()
|
|
} else if err != nil {
|
|
return errhand.BuildDError("Error running diff query:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
|
|
defer rowIter.Close(sqlCtx)
|
|
|
|
var modifiedColNames map[string]bool
|
|
if dArgs.skinny {
|
|
modifiedColNames, err = getModifiedCols(sqlCtx, rowIter, unionSch, sch)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error running diff query:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
for _, col := range dArgs.includeCols {
|
|
modifiedColNames[col] = true // ensure included columns are always present
|
|
}
|
|
// instantiate a new schema that only contains the columns with changes
|
|
var filteredUnionSch sql.Schema
|
|
for _, s := range unionSch {
|
|
for colName := range modifiedColNames {
|
|
if s.Name == colName {
|
|
filteredUnionSch = append(filteredUnionSch, s)
|
|
}
|
|
}
|
|
}
|
|
|
|
// instantiate a new RowWriter with the new schema that only contains the columns with changes
|
|
rowWriter, err = dw.RowWriter(sqlCtx, fromTableInfo, toTableInfo, tableSummary, filteredUnionSch)
|
|
if err != nil {
|
|
return errhand.VerboseErrorFromError(err)
|
|
}
|
|
defer rowWriter.Close(sqlCtx)
|
|
|
|
// reset the row iterator
|
|
err = rowIter.Close(sqlCtx)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error closing row iterator:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
_, rowIter, _, err = queryist.Query(sqlCtx, interpolatedQuery)
|
|
defer rowIter.Close(sqlCtx)
|
|
if sql.ErrSyntaxError.Is(err) {
|
|
return errhand.BuildDError("Failed to parse diff query. Invalid where clause?\nDiff query: %s", interpolatedQuery).AddCause(err).Build()
|
|
} else if err != nil {
|
|
return errhand.BuildDError("Error running diff query:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
}
|
|
|
|
err = writeDiffResults(sqlCtx, sch, unionSch, rowIter, rowWriter, modifiedColNames, dArgs)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error running diff query:\n%s", interpolatedQuery).AddCause(err).Build()
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func unionSchemas(s1 sql.Schema, s2 sql.Schema) sql.Schema {
|
|
var union sql.Schema
|
|
union = append(union, s1...)
|
|
for i := range s2 {
|
|
idx := union.IndexOfColName(s2[i].Name)
|
|
if idx < 0 {
|
|
union = append(union, s2[i])
|
|
} else {
|
|
// have same name. choose the one with the most 'flexible' type
|
|
orig := union[idx]
|
|
orig.Type = chooseMostFlexibleType(orig.Type, s2[i].Type)
|
|
}
|
|
}
|
|
return union
|
|
}
|
|
|
|
// chooseMostFlexibleType returns the type that is more 'flexible' than the other type, for the purposes of printing
|
|
// a diff when the type of the column has changed. There are a ton of different ways we could slice this. We'll stick to
|
|
// the following rules for the time being:
|
|
// - Going from any integer to a float, always take the float.
|
|
// - Going from any integer to a decimal, always take the decimal.
|
|
// - Going from any decimal to a decimal (with different precision/scale), always take the new decimal.
|
|
// - Going from a low precision float to a high precision float, we'll always take the high precision float.
|
|
// - Going from a low precision integer to a high precision integer, we'll always take the high precision integer.
|
|
// Currently, we only support this if the signage is the same.
|
|
// - Going from a DATE, TIME, or DATETIME to a TIMESTAMP, we'll always take the TIMESTAMP.
|
|
//
|
|
// If none of these rules apply, we'll just take the `a` type.
|
|
//
|
|
// Note this is only for printing the diff. This is not robust for other purposes.
|
|
func chooseMostFlexibleType(origA, origB sql.Type) sql.Type {
|
|
if origA.Equals(origB) {
|
|
return origA
|
|
}
|
|
|
|
at := origA.Type()
|
|
bt := origB.Type()
|
|
|
|
// If both are numbers, we'll take the float.
|
|
if sqltypes.IsIntegral(at) && sqltypes.IsFloat(bt) {
|
|
return origB
|
|
}
|
|
if sqltypes.IsIntegral(bt) && sqltypes.IsFloat(at) {
|
|
return origA
|
|
}
|
|
|
|
if bt == sqltypes.Decimal && sqltypes.IsIntegral(at) {
|
|
return origB
|
|
}
|
|
|
|
if sqltypes.IsFloat(at) && sqltypes.IsFloat(bt) {
|
|
// There are only two float types, so we'll always end up with a float64 here.
|
|
return origA.Promote()
|
|
}
|
|
|
|
if sqltypes.IsIntegral(at) && sqltypes.IsIntegral(bt) {
|
|
if (sqltypes.IsUnsigned(at) && sqltypes.IsUnsigned(bt)) || (!sqltypes.IsUnsigned(at) && !sqltypes.IsUnsigned(bt)) {
|
|
// Vitess definitions are ordered in the even that both are signed or unsigned, so take the higher one.
|
|
if bt > at {
|
|
return origB
|
|
}
|
|
return origA
|
|
}
|
|
|
|
// TODO: moving from unsigned to signed or vice versa.
|
|
}
|
|
|
|
if bt == sqltypes.Decimal && at == sqltypes.Decimal {
|
|
return origB
|
|
}
|
|
|
|
if bt == sqltypes.Timestamp && (at == sqltypes.Date || at == sqltypes.Time || at == sqltypes.Datetime) {
|
|
return origB
|
|
}
|
|
|
|
return origA
|
|
}
|
|
|
|
func getColumnNames(fromTableInfo, toTableInfo *diff.TableInfo) (colNames []string, formatText string) {
|
|
var fromSch, toSch schema.Schema
|
|
if fromTableInfo != nil {
|
|
fromSch = fromTableInfo.Sch
|
|
}
|
|
if toTableInfo != nil {
|
|
toSch = toTableInfo.Sch
|
|
}
|
|
|
|
var cols []string
|
|
if fromSch != nil {
|
|
fromSch.GetAllCols().Iter(func(_ uint64, col schema.Column) (stop bool, err error) {
|
|
cols = append(cols, fmt.Sprintf("from_%s", col.Name))
|
|
return false, nil
|
|
})
|
|
}
|
|
if toSch != nil {
|
|
toSch.GetAllCols().Iter(func(_ uint64, col schema.Column) (stop bool, err error) {
|
|
cols = append(cols, fmt.Sprintf("to_%s", col.Name))
|
|
return false, nil
|
|
})
|
|
}
|
|
|
|
colNames = cols
|
|
formatText = strings.Repeat("?, ", len(cols))
|
|
return colNames, formatText
|
|
}
|
|
|
|
func writeDiffResults(
|
|
ctx *sql.Context,
|
|
diffQuerySch sql.Schema,
|
|
targetSch sql.Schema,
|
|
iter sql.RowIter,
|
|
writer diff.SqlRowDiffWriter,
|
|
modifiedColNames map[string]bool,
|
|
dArgs *diffArgs,
|
|
) error {
|
|
ds, err := diff.NewDiffSplitter(diffQuerySch, targetSch)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for {
|
|
r, err := iter.Next(ctx)
|
|
if err == io.EOF {
|
|
return nil
|
|
} else if err != nil {
|
|
return err
|
|
}
|
|
|
|
oldRow, newRow, err := ds.SplitDiffResultRow(ctx, r)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Apply row-level filtering based on diff type
|
|
if dArgs.filter != nil {
|
|
if shouldSkipRow(dArgs.filter, oldRow.RowDiff) || shouldSkipRow(dArgs.filter, newRow.RowDiff) {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if dArgs.skinny {
|
|
var filteredOldRow, filteredNewRow diff.RowDiff
|
|
for i, changeType := range newRow.ColDiffs {
|
|
if (changeType == diff.Added|diff.Removed) || modifiedColNames[targetSch[i].Name] {
|
|
if i < len(oldRow.Row) {
|
|
filteredOldRow.Row = append(filteredOldRow.Row, oldRow.Row[i])
|
|
filteredOldRow.ColDiffs = append(filteredOldRow.ColDiffs, oldRow.ColDiffs[i])
|
|
filteredOldRow.RowDiff = oldRow.RowDiff
|
|
}
|
|
|
|
if i < len(newRow.Row) {
|
|
filteredNewRow.Row = append(filteredNewRow.Row, newRow.Row[i])
|
|
filteredNewRow.ColDiffs = append(filteredNewRow.ColDiffs, newRow.ColDiffs[i])
|
|
filteredNewRow.RowDiff = newRow.RowDiff
|
|
}
|
|
}
|
|
}
|
|
|
|
oldRow = filteredOldRow
|
|
newRow = filteredNewRow
|
|
}
|
|
|
|
// We are guaranteed to have "ModeRow" for writers that do not support combined rows
|
|
if dArgs.diffMode != diff.ModeRow && oldRow.RowDiff == diff.ModifiedOld && newRow.RowDiff == diff.ModifiedNew {
|
|
if err = writer.WriteCombinedRow(ctx, oldRow.Row, newRow.Row, dArgs.diffMode); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
if oldRow.Row != nil {
|
|
if err = writer.WriteRow(ctx, oldRow.Row, oldRow.RowDiff, oldRow.ColDiffs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if newRow.Row != nil {
|
|
if err = writer.WriteRow(ctx, newRow.Row, newRow.RowDiff, newRow.ColDiffs); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// getModifiedCols returns a set of the names of columns that are modified, as well as the name of the primary key for a particular row iterator and schema.
|
|
// In the case where rows are added or removed, all columns will be included
|
|
// unionSch refers to a joint schema between the schema before and after any schema changes pertaining to the diff,
|
|
// while diffQuerySch refers to the schema returned by the "dolt_diff" sql query.
|
|
func getModifiedCols(
|
|
ctx *sql.Context,
|
|
iter sql.RowIter,
|
|
unionSch sql.Schema,
|
|
diffQuerySch sql.Schema,
|
|
) (map[string]bool, error) {
|
|
modifiedColNames := make(map[string]bool)
|
|
for {
|
|
r, err := iter.Next(ctx)
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
|
|
ds, err := diff.NewDiffSplitter(diffQuerySch, unionSch)
|
|
if err != nil {
|
|
return modifiedColNames, err
|
|
}
|
|
|
|
oldRow, newRow, err := ds.SplitDiffResultRow(ctx, r)
|
|
if err != nil {
|
|
return modifiedColNames, err
|
|
}
|
|
|
|
for i, changeType := range newRow.ColDiffs {
|
|
if changeType != diff.None || unionSch[i].PrimaryKey {
|
|
modifiedColNames[unionSch[i].Name] = true
|
|
}
|
|
}
|
|
|
|
for i, changeType := range oldRow.ColDiffs {
|
|
if changeType != diff.None || unionSch[i].PrimaryKey {
|
|
modifiedColNames[unionSch[i].Name] = true
|
|
}
|
|
}
|
|
}
|
|
|
|
return modifiedColNames, nil
|
|
}
|
|
|
|
// validateWhereClause validates the where clause by attempting a minimal syntax check
|
|
// against the dolt_diff schema. This ensures the where clause is syntactically correct
|
|
// and references valid column patterns, even when there are no changes to show.
|
|
func validateWhereClause(queryist cli.Queryist, sqlCtx *sql.Context, dArgs *diffArgs) errhand.VerboseError {
|
|
// Build a minimal validation query that doesn't depend on having actual tables
|
|
// We use a subquery approach so the aliased columns are available in the WHERE clause
|
|
query := "select * from (select 1 as diff_type, 1 as from_pk, 1 as to_pk) as diff_validation where " + dArgs.where + " limit 0"
|
|
_, rowIter, _, err := queryist.Query(sqlCtx, query)
|
|
if err != nil {
|
|
return errhand.BuildDError("Error running diff query").AddCause(err).Build()
|
|
}
|
|
|
|
// Close the iterator since we only wanted to validate
|
|
if rowIter != nil {
|
|
rowIter.Close(sqlCtx)
|
|
}
|
|
|
|
return nil
|
|
}
|