milvus-io--milvus/internal/util/function/validator/validator.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validator

import (
	"github.com/samber/lo"

	"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
	"github.com/milvus-io/milvus/internal/util/function/embedding"
	"github.com/milvus-io/milvus/internal/util/function/models"
	"github.com/milvus-io/milvus/pkg/v3/util/merr"
	"github.com/milvus-io/milvus/pkg/v3/util/paramtable"
	"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
)

func ValidateFunction(coll *schemapb.CollectionSchema, needValidateFunctionName string, disableRuntimeCheck bool) error {
	nameMap := lo.SliceToMap(coll.GetFields(), func(field *schemapb.FieldSchema) (string, *schemapb.FieldSchema) {
		return field.GetName(), field
	})
	usedOutputField := typeutil.NewSet[string]()
	usedFunctionName := typeutil.NewSet[string]()

	for _, function := range coll.GetFunctions() {
		if err := CheckFunctionBasicParams(function); err != nil {
			return err
		}

		if usedFunctionName.Contain(function.GetName()) {
			return merr.WrapErrParameterInvalidMsg("duplicate function name: %s", function.GetName())
		}

		usedFunctionName.Insert(function.GetName())
		inputFields := []*schemapb.FieldSchema{}
		for _, name := range function.GetInputFieldNames() {
			inputField, ok := nameMap[name]
			if !ok {
				return merr.WrapErrParameterInvalidMsg("function input field not found: %s", name)
			}
			inputFields = append(inputFields, inputField)
		}

		if err := CheckFunctionInputField(function, inputFields); err != nil {
			return err
		}

		outputFields := make([]*schemapb.FieldSchema, len(function.GetOutputFieldNames()))
		for i, name := range function.GetOutputFieldNames() {
			outputField, ok := nameMap[name]
			if !ok {
				return merr.WrapErrParameterInvalidMsg("function output field not found: %s", name)
			}

			if outputField.GetIsPrimaryKey() {
				return merr.WrapErrParameterInvalidMsg("function output field cannot be primary key: function %s, field %s", function.GetName(), outputField.GetName())
			}

			if outputField.GetIsPartitionKey() || outputField.GetIsClusteringKey() {
				return merr.WrapErrParameterInvalidMsg("function output field cannot be partition key or clustering key: function %s, field %s", function.GetName(), outputField.GetName())
			}

			if outputField.GetNullable() {
				return merr.WrapErrParameterInvalidMsg("function output field cannot be nullable: function %s, field %s", function.GetName(), outputField.GetName())
			}

			outputFields[i] = outputField
			if usedOutputField.Contain(name) {
				return merr.WrapErrParameterInvalidMsg("duplicate function output field: function %s, field %s", function.GetName(), name)
			}
			usedOutputField.Insert(name)
		}

		if err := CheckFunctionOutputField(function, outputFields); err != nil {
			return err
		}
	}
	if !disableRuntimeCheck {
		if err := embedding.ValidateFunctions(coll, needValidateFunctionName, &models.ModelExtraInfo{ClusterID: paramtable.Get().CommonCfg.ClusterPrefix.GetValue(), DBName: coll.DbName}); err != nil {
			return err
		}
	}
	return nil
}

func NormalizeFunctionOutputFields(coll *schemapb.CollectionSchema) error {
	if coll == nil {
		return nil
	}
	fieldsByName := lo.SliceToMap(coll.GetFields(), func(field *schemapb.FieldSchema) (string, *schemapb.FieldSchema) {
		return field.GetName(), field
	})
	fieldsByID := lo.SliceToMap(coll.GetFields(), func(field *schemapb.FieldSchema) (int64, *schemapb.FieldSchema) {
		return field.GetFieldID(), field
	})

	for _, field := range coll.GetFields() {
		field.IsFunctionOutput = false
	}

	for _, function := range coll.GetFunctions() {
		for _, fieldID := range function.GetOutputFieldIds() {
			if fieldID == 0 {
				continue
			}
			field, ok := fieldsByID[fieldID]
			if !ok {
				return merr.WrapErrParameterInvalidMsg("function output field id %d not found in schema", fieldID)
			}
			field.IsFunctionOutput = true
		}
		for _, name := range function.GetOutputFieldNames() {
			if name == "" {
				continue
			}
			field, ok := fieldsByName[name]
			if !ok {
				return merr.WrapErrParameterInvalidMsg("function output field not found: %s", name)
			}
			field.IsFunctionOutput = true
		}
	}
	return nil
}

func CheckFunctionOutputField(fSchema *schemapb.FunctionSchema, fields []*schemapb.FieldSchema) error {
	switch fSchema.GetType() {
	case schemapb.FunctionType_BM25:
		if len(fields) != 1 {
			return merr.WrapErrParameterInvalidMsg("BM25 function only need 1 output field, but got %d", len(fields))
		}

		if !typeutil.IsSparseFloatVectorType(fields[0].GetDataType()) {
			return merr.WrapErrParameterInvalidMsg("BM25 function output field must be a SparseFloatVector field, but got %s", fields[0].DataType.String())
		}
	case schemapb.FunctionType_TextEmbedding:
		if err := embedding.TextEmbeddingOutputsCheck(fields); err != nil {
			return err
		}
	case schemapb.FunctionType_MinHash:
		if len(fields) != 1 {
			return merr.WrapErrParameterInvalidMsg("MinHash function only need 1 output field, but got %d", len(fields))
		}
		if fields[0].GetDataType() != schemapb.DataType_BinaryVector {
			return merr.WrapErrParameterInvalidMsg("MinHash function output field must be a BinaryVector field, but got %s", fields[0].DataType.String())
		}
	default:
		return merr.WrapErrParameterInvalidMsg("check output field for unknown function type")
	}
	return nil
}

func CheckFunctionInputField(function *schemapb.FunctionSchema, fields []*schemapb.FieldSchema) error {
	switch function.GetType() {
	case schemapb.FunctionType_BM25:
		if len(fields) != 1 || (fields[0].DataType != schemapb.DataType_VarChar && fields[0].DataType != schemapb.DataType_Text) {
			return merr.WrapErrParameterInvalidMsg("BM25 function input field must be a VARCHAR/TEXT field, got %d field with type %s",
				len(fields), fields[0].DataType.String())
		}
		h := typeutil.CreateFieldSchemaHelper(fields[0])
		if !h.EnableAnalyzer() {
			return merr.WrapErrParameterInvalidMsg("BM25 function input field must set enable_analyzer to true")
		}
	case schemapb.FunctionType_TextEmbedding:
		if err := embedding.TextEmbeddingInputsCheck(function.GetName(), fields); err != nil {
			return err
		}
	case schemapb.FunctionType_MinHash:
		if len(fields) != 1 || (fields[0].DataType != schemapb.DataType_VarChar && fields[0].DataType != schemapb.DataType_Text) {
			return merr.WrapErrParameterInvalidMsg("MinHash function input field must be a VARCHAR/TEXT field, got %d field with type %s",
				len(fields), fields[0].DataType.String())
		}
	default:
		return merr.WrapErrParameterInvalidMsg("check input field with unknown function type")
	}
	return nil
}

func CheckFunctionBasicParams(function *schemapb.FunctionSchema) error {
	if function.GetName() == "" {
		return merr.WrapErrParameterMissingMsg("function name cannot be empty")
	}
	if len(function.GetInputFieldNames()) == 0 {
		return merr.WrapErrParameterMissingMsg("function input field names cannot be empty, function: %s", function.GetName())
	}
	if len(function.GetOutputFieldNames()) == 0 {
		return merr.WrapErrParameterMissingMsg("function output field names cannot be empty, function: %s", function.GetName())
	}
	for _, input := range function.GetInputFieldNames() {
		if input == "" {
			return merr.WrapErrParameterMissingMsg("function input field name cannot be empty string, function: %s", function.GetName())
		}
		// if input occurs more than once, error
		if lo.Count(function.GetInputFieldNames(), input) > 1 {
			return merr.WrapErrParameterInvalidMsg("each function input field should be used exactly once in the same function, function: %s, input field: %s", function.GetName(), input)
		}
	}
	for _, output := range function.GetOutputFieldNames() {
		if output == "" {
			return merr.WrapErrParameterMissingMsg("function output field name cannot be empty string, function: %s", function.GetName())
		}
		if lo.Count(function.GetInputFieldNames(), output) > 0 {
			return merr.WrapErrParameterInvalidMsg("a single field cannot be both input and output in the same function, function: %s, field: %s", function.GetName(), output)
		}
		if lo.Count(function.GetOutputFieldNames(), output) > 1 {
			return merr.WrapErrParameterInvalidMsg("each function output field should be used exactly once in the same function, function: %s, output field: %s", function.GetName(), output)
		}
	}
	switch function.GetType() {
	case schemapb.FunctionType_BM25:
		if len(function.GetParams()) != 0 {
			return merr.WrapErrParameterInvalidMsg("BM25 function accepts no params")
		}
	case schemapb.FunctionType_TextEmbedding:
		if len(function.GetParams()) == 0 {
			return merr.WrapErrParameterInvalidMsg("TextEmbedding function accepts no params")
		}
	case schemapb.FunctionType_MinHash:
		// MinHash function can accept optional params
		return nil
	default:
		return merr.WrapErrParameterInvalidMsg("check function params with unknown function type")
	}
	return nil
}