Files
wehub-resource-sync 498b235461
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
chore: import upstream snapshot with attribution
2026-07-13 12:31:17 +08:00

1183 lines
40 KiB
Go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package httpserver
import (
"context"
"strconv"
"strings"
"github.com/gin-gonic/gin"
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
"github.com/milvus-io/milvus/internal/json"
"github.com/milvus-io/milvus/internal/util/importutilv2"
"github.com/milvus-io/milvus/pkg/v3/common"
"github.com/milvus-io/milvus/pkg/v3/mlog"
"github.com/milvus-io/milvus/pkg/v3/util/merr"
)
type EmptyReq struct{}
func (req *EmptyReq) GetDbName() string { return "" }
type DatabaseReq struct {
DbName string `json:"dbName"`
}
func (req *DatabaseReq) GetDbName() string { return req.DbName }
type DatabaseReqRequiredName struct {
DbName string `json:"dbName" binding:"required"`
}
func (req *DatabaseReqRequiredName) GetDbName() string { return req.DbName }
type DatabaseReqWithProperties struct {
DbName string `json:"dbName" binding:"required"`
Properties map[string]interface{} `json:"properties"`
}
func (req *DatabaseReqWithProperties) GetDbName() string { return req.DbName }
type DropDatabasePropertiesReq struct {
DbName string `json:"dbName" binding:"required"`
PropertyKeys []string `json:"propertyKeys"`
}
func (req *DropDatabasePropertiesReq) GetDbName() string { return req.DbName }
type CollectionNameReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionNames []string `json:"partitionNames"` // get partitions load state
}
func (req *CollectionNameReq) GetDbName() string {
return req.DbName
}
func (req *CollectionNameReq) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionNameReq) GetPartitionNames() []string {
return req.PartitionNames
}
type CollectionReqWithProperties struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Properties map[string]interface{} `json:"properties"`
}
func (req *CollectionReqWithProperties) GetDbName() string { return req.DbName }
func (req *CollectionReqWithProperties) GetCollectionName() string {
return req.CollectionName
}
type CollectionDropFunction struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
FunctionName string `json:"FunctionName" binding:"required"`
}
func (req *CollectionDropFunction) GetDbName() string { return req.DbName }
func (req *CollectionDropFunction) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionDropFunction) GetFunctionName() string {
return req.FunctionName
}
type CollectionAddFunction struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Function FunctionSchema `json:"function" binding:"required"`
}
func (req *CollectionAddFunction) GetDbName() string { return req.DbName }
func (req *CollectionAddFunction) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionAddFunction) GetFunction() *FunctionSchema {
return &req.Function
}
type CollectionAlterFunction struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
FunctionName string `json:"functionName" binding:"required"`
Function FunctionSchema `json:"function" binding:"required"`
}
func (req *CollectionAlterFunction) GetDbName() string { return req.DbName }
func (req *CollectionAlterFunction) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionAlterFunction) GetFunctionName() string {
return req.FunctionName
}
func (req *CollectionAlterFunction) GetFunction() *FunctionSchema {
return &req.Function
}
type OptionalCollectionNameReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName"`
}
func (req *OptionalCollectionNameReq) GetDbName() string {
return req.DbName
}
func (req *OptionalCollectionNameReq) GetCollectionName() string {
return req.CollectionName
}
type RenameCollectionReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
NewCollectionName string `json:"newCollectionName" binding:"required"`
NewDbName string `json:"newDbName"`
}
func (req *RenameCollectionReq) GetDbName() string { return req.DbName }
func (req *RenameCollectionReq) GetCollectionName() string { return req.CollectionName }
type DropCollectionPropertiesReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PropertyKeys []string `json:"propertyKeys"`
}
func (req *DropCollectionPropertiesReq) GetDbName() string { return req.DbName }
func (req *DropCollectionPropertiesReq) GetCollectionName() string {
return req.CollectionName
}
type CompactReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IsClustering bool `json:"isClustering"`
}
func (req *CompactReq) GetDbName() string { return req.DbName }
func (req *CompactReq) GetCollectionName() string {
return req.CollectionName
}
type GetCompactionStateReq struct {
JobID int64 `json:"jobID"`
}
type FlushReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
}
func (req *FlushReq) GetDbName() string { return req.DbName }
func (req *FlushReq) GetCollectionName() string {
return req.CollectionName
}
type CollectionFieldReqWithParams struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
FieldName string `json:"fieldName" binding:"required"`
FieldParams map[string]interface{} `json:"fieldParams"`
}
func (req *CollectionFieldReqWithParams) GetDbName() string { return req.DbName }
func (req *CollectionFieldReqWithParams) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionFieldReqWithParams) GetFieldName() string {
return req.FieldName
}
type CollectionFieldReqWithSchema struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Schema *FieldSchema `json:"schema" binding:"required"`
}
func (req *CollectionFieldReqWithSchema) GetDbName() string { return req.DbName }
func (req *CollectionFieldReqWithSchema) GetCollectionName() string {
return req.CollectionName
}
func (req *CollectionFieldReqWithSchema) GetFieldName() string {
if req.Schema == nil {
return ""
}
return req.Schema.FieldName
}
type PartitionReq struct {
// CollectionNameReq
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName" binding:"required"`
}
func (req *PartitionReq) GetDbName() string { return req.DbName }
func (req *PartitionReq) GetCollectionName() string { return req.CollectionName }
func (req *PartitionReq) GetPartitionName() string { return req.PartitionName }
type ImportReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName"`
Files [][]string `json:"files" binding:"required"`
Options map[string]string `json:"options"`
}
const (
autoCommitTrue = "true"
autoCommitFalse = "false"
)
func (req *ImportReq) UnmarshalJSON(data []byte) error {
type importReqAlias struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName"`
Files [][]string `json:"files" binding:"required"`
Options map[string]*string `json:"options"`
}
var decoded importReqAlias
if err := json.Unmarshal(data, &decoded); err != nil {
return err
}
options := make(map[string]string, len(decoded.Options))
for key, value := range decoded.Options {
if value == nil {
if key == importutilv2.AutoCommitKey {
return merr.WrapErrParameterInvalidMsg("options.%s must be one of %q or %q", importutilv2.AutoCommitKey, autoCommitTrue, autoCommitFalse)
}
options[key] = ""
continue
}
if key == importutilv2.AutoCommitKey && strings.ToLower(*value) != autoCommitTrue && strings.ToLower(*value) != autoCommitFalse {
return merr.WrapErrParameterInvalidMsg("options.%s must be one of %q or %q", importutilv2.AutoCommitKey, autoCommitTrue, autoCommitFalse)
}
options[key] = *value
}
req.DbName = decoded.DbName
req.CollectionName = decoded.CollectionName
req.PartitionName = decoded.PartitionName
req.Files = decoded.Files
req.Options = options
return nil
}
func (req *ImportReq) GetDbName() string {
return req.DbName
}
func (req *ImportReq) GetCollectionName() string {
return req.CollectionName
}
func (req *ImportReq) GetPartitionName() string {
return req.PartitionName
}
func (req *ImportReq) GetFiles() [][]string {
return req.Files
}
func (req *ImportReq) GetOptions() map[string]string {
return req.Options
}
type JobIDReq struct {
JobID string `json:"jobId" binding:"required"`
}
func (req *JobIDReq) GetJobID() string { return req.JobID }
type RestoreExternalSnapshotReq struct {
DbName string `json:"dbName"`
TargetCollectionName string `json:"targetCollectionName" binding:"required"`
SnapshotMetadataURI string `json:"snapshotMetadataURI" binding:"required"`
ExternalSpec string `json:"externalSpec"`
}
func (req *RestoreExternalSnapshotReq) GetDbName() string { return req.DbName }
type ExportSnapshotReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Name string `json:"snapshotName" binding:"required"`
TargetS3Path string `json:"targetS3Path" binding:"required"`
ExternalSpec string `json:"externalSpec"`
}
func (req *ExportSnapshotReq) GetDbName() string { return req.DbName }
type QueryReqV2 struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionNames []string `json:"partitionNames"`
OutputFields []string `json:"outputFields"`
Filter string `json:"filter"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
// OrderByFields sorts query results by scalar fields; each item is
// "fieldName" or "fieldName:asc" / "fieldName:desc" (default asc).
OrderByFields []string `json:"orderByFields"`
ExprParams map[string]interface{} `json:"exprParams"`
ConsistencyLevel string `json:"consistencyLevel"`
}
func (req *QueryReqV2) GetDbName() string { return req.DbName }
func (req *QueryReqV2) GetCollectionName() string { return req.CollectionName }
type CollectionIDReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName"`
PartitionNames []string `json:"partitionNames"`
OutputFields []string `json:"outputFields"`
ID interface{} `json:"id" binding:"required"`
ConsistencyLevel string `json:"consistencyLevel"`
}
func (req *CollectionIDReq) GetDbName() string { return req.DbName }
func (req *CollectionIDReq) GetCollectionName() string { return req.CollectionName }
type CollectionFilterReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName"`
Filter string `json:"filter" binding:"required"`
ExprParams map[string]interface{} `json:"exprParams"`
}
func (req *CollectionFilterReq) GetDbName() string { return req.DbName }
func (req *CollectionFilterReq) GetCollectionName() string { return req.CollectionName }
type CollectionDataReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionName string `json:"partitionName"`
Data []map[string]interface{} `json:"data" binding:"required"`
PartialUpdate bool `json:"partialUpdate"`
FieldOps []FieldPartialUpdateOpReq `json:"fieldOps"`
}
func (req *CollectionDataReq) GetDbName() string { return req.DbName }
func (req *CollectionDataReq) GetCollectionName() string { return req.CollectionName }
type FieldPartialUpdateOpReq struct {
FieldName string `json:"fieldName"`
Op string `json:"op"`
}
func buildFieldPartialUpdateOps(fieldOps []FieldPartialUpdateOpReq) ([]*schemapb.FieldPartialUpdateOp, error) {
if len(fieldOps) == 0 {
return nil, nil
}
ops := make([]*schemapb.FieldPartialUpdateOp, 0, len(fieldOps))
for _, fieldOp := range fieldOps {
op, err := parseFieldPartialUpdateOp(fieldOp.Op)
if err != nil {
return nil, err
}
ops = append(ops, &schemapb.FieldPartialUpdateOp{
FieldName: fieldOp.FieldName,
Op: op,
})
}
return ops, nil
}
func parseFieldPartialUpdateOp(op string) (schemapb.FieldPartialUpdateOp_OpType, error) {
switch strings.ToUpper(strings.TrimSpace(op)) {
case "REPLACE":
return schemapb.FieldPartialUpdateOp_REPLACE, nil
case "ARRAY_APPEND":
return schemapb.FieldPartialUpdateOp_ARRAY_APPEND, nil
case "ARRAY_REMOVE":
return schemapb.FieldPartialUpdateOp_ARRAY_REMOVE, nil
default:
return schemapb.FieldPartialUpdateOp_REPLACE,
merr.WrapErrParameterInvalidMsg("unsupported partial update op: " + op)
}
}
type SearchReqV2 struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Data []interface{} `json:"data"`
Ids []interface{} `json:"ids"`
AnnsField string `json:"annsField"`
PartitionNames []string `json:"partitionNames"`
Filter string `json:"filter"`
GroupByField string `json:"groupingField"`
GroupSize int32 `json:"groupSize"`
StrictGroupSize bool `json:"strictGroupSize"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
OutputFields []string `json:"outputFields"`
SearchParams map[string]interface{} `json:"searchParams"`
ConsistencyLevel string `json:"consistencyLevel"`
ExprParams map[string]interface{} `json:"exprParams"`
FunctionScore FunctionScore `json:"functionScore"`
FunctionChains []FunctionChainReq `json:"functionChains"`
SearchAggregation *SearchAggregationReq `json:"searchAggregation"`
// not use Params any more, just for compatibility
Params map[string]float64 `json:"params"`
}
func (req *SearchReqV2) GetDbName() string { return req.DbName }
func (req *SearchReqV2) GetCollectionName() string { return req.CollectionName }
type SearchAggregationReq struct {
Fields []string `json:"fields"`
Size int64 `json:"size"`
SearchSize int64 `json:"searchSize"`
Metrics map[string]MetricAggregationReq `json:"metrics"`
Order []AggregationOrderReq `json:"order"`
TopHits *TopHitsReq `json:"topHits"`
SubAggregation *SearchAggregationReq `json:"subAggregation"`
}
type MetricAggregationReq struct {
Op string `json:"op"`
FieldName string `json:"fieldName"`
}
type AggregationOrderReq struct {
Key string `json:"key"`
Direction string `json:"direction"`
}
type TopHitsReq struct {
Size int64 `json:"size"`
Sort []AggregationSortReq `json:"sort"`
}
type AggregationSortReq struct {
FieldName string `json:"fieldName"`
Direction string `json:"direction"`
}
type Rand struct {
Strategy string `json:"strategy"`
Params map[string]interface{} `json:"params"`
}
type SubSearchReq struct {
Data []interface{} `json:"data" binding:"required"`
AnnsField string `json:"annsField"`
Filter string `json:"filter"`
GroupByField string `json:"groupingField"`
MetricType string `json:"metricType"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
SearchParams map[string]interface{} `json:"params"`
ExprParams map[string]interface{} `json:"exprParams"`
SearchAggregation *SearchAggregationReq `json:"searchAggregation"`
}
type HybridSearchReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionNames []string `json:"partitionNames"`
Search []SubSearchReq `json:"search"`
Rerank Rand `json:"rerank"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
GroupByField string `json:"groupingField"`
GroupSize int32 `json:"groupSize"`
StrictGroupSize bool `json:"strictGroupSize"`
OutputFields []string `json:"outputFields"`
ConsistencyLevel string `json:"consistencyLevel"`
FunctionScore FunctionScore `json:"functionScore"`
FunctionChains []FunctionChainReq `json:"functionChains"`
SearchAggregation *SearchAggregationReq `json:"searchAggregation"`
}
func (req *HybridSearchReq) GetDbName() string { return req.DbName }
func (req *HybridSearchReq) GetCollectionName() string { return req.CollectionName }
type ReturnErrMsg struct {
Code int32 `json:"code"`
Message string `json:"message"`
}
type PartitionsReq struct {
// CollectionNameReq
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
PartitionNames []string `json:"partitionNames" binding:"required"`
}
func (req *PartitionsReq) GetDbName() string { return req.DbName }
func (req *PartitionsReq) GetCollectionName() string { return req.CollectionName }
type UserReq struct {
UserName string `json:"userName" binding:"required"`
}
func (req *UserReq) GetUserName() string { return req.UserName }
type BaseGetter interface {
GetBase() *commonpb.MsgBase
}
type UserNameGetter interface {
GetUserName() string
}
type RoleNameGetter interface {
GetRoleName() string
}
type IndexNameGetter interface {
GetIndexName() string
}
type AliasNameGetter interface {
GetAliasName() string
}
type FilesGetter interface {
GetFiles() [][]string
}
type OptionsGetter interface {
GetOptions() map[string]string
}
type JobIDGetter interface {
GetJobID() string
}
type TimestampGetter interface {
GetTimestamp() uint64
}
type PasswordReq struct {
UserName string `json:"userName" binding:"required"`
Password string `json:"password" binding:"required"`
Description *string `json:"description"`
}
type NewPasswordReq struct {
UserName string `json:"userName" binding:"required"`
Password string `json:"password"`
NewPassword string `json:"newPassword"`
Description *string `json:"description"`
}
type UserRoleReq struct {
UserName string `json:"userName" binding:"required"`
RoleName string `json:"roleName" binding:"required"`
}
type RoleReq struct {
DbName string `json:"dbName"`
RoleName string `json:"roleName" binding:"required"`
Description string `json:"description"`
}
func (req *RoleReq) GetDbName() string { return req.DbName }
func (req *RoleReq) GetRoleName() string {
return req.RoleName
}
func (req *RoleReq) GetDescription() string {
return req.Description
}
type PrivilegeGroupReq struct {
PrivilegeGroupName string `json:"privilegeGroupName" binding:"required"`
Privileges []string `json:"privileges"`
}
type GrantV2Req struct {
RoleName string `json:"roleName" binding:"required"`
DbName string `json:"dbName"`
CollectionName string `json:"collectionName"`
Privilege string `json:"privilege" binding:"required"`
}
func (req *GrantV2Req) GetDbName() string { return req.DbName }
func (req *GrantV2Req) GetCollectionName() string { return req.CollectionName }
type GrantReq struct {
RoleName string `json:"roleName" binding:"required"`
ObjectType string `json:"objectType" binding:"required"`
ObjectName string `json:"objectName" binding:"required"`
Privilege string `json:"privilege" binding:"required"`
DbName string `json:"dbName"`
}
func (req *GrantReq) GetDbName() string { return req.DbName }
type IndexParam struct {
FieldName string `json:"fieldName" binding:"required"`
IndexName string `json:"indexName"`
MetricType string `json:"metricType"`
IndexType string `json:"indexType"`
Params map[string]interface{} `json:"params"`
}
type IndexParamReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IndexParams []IndexParam `json:"indexParams" binding:"required"`
}
func (req *IndexParamReq) GetDbName() string { return req.DbName }
func (req *IndexParamReq) GetCollectionName() string { return req.CollectionName }
type IndexReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IndexName string `json:"indexName" binding:"required"`
Timestamp uint64 `json:"timestamp"`
}
func (req *IndexReq) GetDbName() string { return req.DbName }
func (req *IndexReq) GetCollectionName() string {
return req.CollectionName
}
func (req *IndexReq) GetIndexName() string {
return req.IndexName
}
func (req *IndexReq) GetTimestamp() uint64 {
return req.Timestamp
}
type IndexReqWithProperties struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IndexName string `json:"indexName" binding:"required"`
Properties map[string]interface{} `json:"properties"`
}
func (req *IndexReqWithProperties) GetDbName() string { return req.DbName }
func (req *IndexReqWithProperties) GetCollectionName() string {
return req.CollectionName
}
func (req *IndexReqWithProperties) GetIndexName() string {
return req.IndexName
}
type DropIndexPropertiesReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IndexName string `json:"indexName" binding:"required"`
PropertyKeys []string `json:"propertyKeys"`
}
func (req *DropIndexPropertiesReq) GetDbName() string { return req.DbName }
func (req *DropIndexPropertiesReq) GetCollectionName() string {
return req.CollectionName
}
func (req *DropIndexPropertiesReq) GetIndexName() string {
return req.IndexName
}
type FieldSchema struct {
FieldName string `json:"fieldName" binding:"required"`
Description string `json:"description"`
DataType string `json:"dataType" binding:"required"`
ElementDataType string `json:"elementDataType"`
ExternalField string `json:"externalField"`
IsPrimary bool `json:"isPrimary"`
IsPartitionKey bool `json:"isPartitionKey"`
IsClusteringKey bool `json:"isClusteringKey"`
ElementTypeParams map[string]interface{} `json:"elementTypeParams"`
TypeParams map[string]interface{} `json:"typeParams"`
Fields []FieldSchema `json:"fields"`
Nullable bool `json:"nullable"`
DefaultValue interface{} `json:"defaultValue"`
}
func (field *FieldSchema) GetProto(ctx context.Context) (*schemapb.FieldSchema, error) {
fieldDataType, ok := schemapb.DataType_value[field.DataType]
if !ok {
mlog.Warn(ctx, "field's data type is invalid(case sensitive).", mlog.Any("fieldDataType", field.DataType), mlog.Any("field", field))
return nil, merr.WrapErrParameterInvalidMsg("data type %s is invalid(case sensitive)", field.DataType)
}
dataType := schemapb.DataType(fieldDataType)
fieldSchema := &schemapb.FieldSchema{
Name: field.FieldName,
Description: field.Description,
IsPrimaryKey: field.IsPrimary,
IsPartitionKey: field.IsPartitionKey,
IsClusteringKey: field.IsClusteringKey,
DataType: dataType,
TypeParams: []*commonpb.KeyValuePair{},
Nullable: field.Nullable,
ExternalField: field.ExternalField,
}
var err error
fieldSchema.DefaultValue, err = convertDefaultValue(field.DefaultValue, dataType)
if err != nil {
mlog.Warn(ctx, "convert defaultValue fail", mlog.Any("defaultValue", field.DefaultValue), mlog.Err(err))
return nil, merr.WrapErrParameterInvalidMsg("convert defaultValue fail, err: %s", err.Error())
}
if dataType == schemapb.DataType_Array || dataType == schemapb.DataType_ArrayOfVector {
if _, ok := schemapb.DataType_value[field.ElementDataType]; !ok {
mlog.Warn(ctx, "element's data type is invalid(case sensitive).", mlog.Any("elementDataType", field.ElementDataType), mlog.Any("field", field))
return nil, merr.WrapErrParameterInvalidMsg("element data type %s is invalid(case sensitive)", field.ElementDataType)
}
fieldSchema.ElementType = schemapb.DataType(schemapb.DataType_value[field.ElementDataType])
}
for key, fieldParam := range field.ElementTypeParams {
value, err := getElementTypeParams(fieldParam)
if err != nil {
return nil, err
}
fieldSchema.TypeParams = append(fieldSchema.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
}
return fieldSchema, nil
}
func (field *FieldSchema) IsStructArrayField() bool {
if field == nil {
return false
}
return field.DataType == schemapb.DataType_ArrayOfStruct.String() ||
field.DataType == schemapb.DataType_Array.String() && field.ElementDataType == schemapb.DataType_Struct.String()
}
func (field *FieldSchema) GetStructArrayProto(ctx context.Context) (*schemapb.StructArrayFieldSchema, error) {
if field == nil {
return nil, merr.WrapErrParameterInvalidMsg("StructArray field schema is required")
}
if !field.IsStructArrayField() {
return nil, merr.WrapErrParameterInvalidMsg(
"StructArray field must use ArrayOfStruct or Array with Struct element, got dataType %s and elementDataType %s",
field.DataType, field.ElementDataType)
}
typeParams := make(map[string]interface{}, len(field.TypeParams)+len(field.ElementTypeParams))
for key, param := range field.ElementTypeParams {
typeParams[key] = param
}
for key, param := range field.TypeParams {
typeParams[key] = param
}
return (&StructArrayFieldSchema{
FieldName: field.FieldName,
Description: field.Description,
Fields: field.Fields,
TypeParams: typeParams,
Nullable: field.Nullable,
}).GetProto(ctx)
}
// StructArrayFieldSchema describes a struct array field in RESTful v2 API.
// Each struct array field contains multiple sub-fields; every sub-field must
// be declared as either Array (scalar element) or ArrayOfVector (vector element).
type StructArrayFieldSchema struct {
FieldName string `json:"fieldName" binding:"required"`
Description string `json:"description"`
Fields []FieldSchema `json:"fields" binding:"required"`
TypeParams map[string]interface{} `json:"typeParams"`
Nullable bool `json:"nullable"`
}
// GetProto converts the RESTful StructArrayFieldSchema to its proto counterpart.
// Only Array / ArrayOfVector data types are allowed for sub-fields.
func (sf *StructArrayFieldSchema) GetProto(ctx context.Context) (*schemapb.StructArrayFieldSchema, error) {
if len(sf.Fields) == 0 {
return nil, merr.WrapErrParameterInvalidMsg("struct field %s must contain at least one sub-field", sf.FieldName)
}
proto := &schemapb.StructArrayFieldSchema{
Name: sf.FieldName,
Description: sf.Description,
TypeParams: []*commonpb.KeyValuePair{},
Nullable: sf.Nullable,
}
parentTypeParams := make(map[string]string, len(sf.TypeParams))
for key, param := range sf.TypeParams {
value, err := getElementTypeParams(param)
if err != nil {
return nil, err
}
parentTypeParams[key] = value
proto.TypeParams = append(proto.TypeParams, &commonpb.KeyValuePair{Key: key, Value: value})
}
subNames := map[string]struct{}{}
for i := range sf.Fields {
sub := sf.Fields[i]
subProto, err := sub.GetProto(ctx)
if err != nil {
return nil, err
}
if subProto.DataType != schemapb.DataType_Array && subProto.DataType != schemapb.DataType_ArrayOfVector {
return nil, merr.WrapErrParameterInvalidMsg(
"sub-field %s of struct %s must be Array or ArrayOfVector, got %s",
sub.FieldName, sf.FieldName, sub.DataType)
}
if subProto.IsPrimaryKey || subProto.IsPartitionKey || subProto.IsClusteringKey {
return nil, merr.WrapErrParameterInvalidMsg(
"sub-field %s of struct %s cannot be primary / partition / clustering key",
sub.FieldName, sf.FieldName)
}
if subProto.Nullable {
return nil, merr.WrapErrParameterInvalidMsg(
"sub-field %s of struct %s cannot be nullable",
sub.FieldName, sf.FieldName)
}
if subProto.DefaultValue != nil {
return nil, merr.WrapErrParameterInvalidMsg(
"sub-field %s of struct %s cannot set defaultValue",
sub.FieldName, sf.FieldName)
}
if _, dup := subNames[subProto.Name]; dup {
return nil, merr.WrapErrParameterInvalidMsg(
"duplicated sub-field name %s in struct %s", subProto.Name, sf.FieldName)
}
subNames[subProto.Name] = struct{}{}
if maxCapacity, ok := parentTypeParams[common.MaxCapacityKey]; ok && !hasTypeParam(subProto.TypeParams, common.MaxCapacityKey) {
subProto.TypeParams = append(subProto.TypeParams, &commonpb.KeyValuePair{Key: common.MaxCapacityKey, Value: maxCapacity})
}
proto.Fields = append(proto.Fields, subProto)
}
return proto, nil
}
func hasTypeParam(typeParams []*commonpb.KeyValuePair, key string) bool {
for _, typeParam := range typeParams {
if typeParam.GetKey() == key {
return true
}
}
return false
}
type FunctionScore struct {
Functions []FunctionSchema `json:"functions"`
Params map[string]interface{} `json:"params"`
}
type FunctionChainReq struct {
Name string `json:"name"`
Stage string `json:"stage"`
Ops []FunctionChainOpReq `json:"ops"`
}
type FunctionChainOpReq struct {
Op string `json:"op"`
Expr *FunctionChainExprReq `json:"expr"`
Inputs []string `json:"inputs"`
Outputs []string `json:"outputs"`
Params map[string]interface{} `json:"params"`
}
type FunctionChainExprReq struct {
Name string `json:"name"`
Args []FunctionChainExprArgReq `json:"args"`
Params map[string]interface{} `json:"params"`
}
type FunctionChainExprArgReq struct {
Column *string `json:"column"`
Literal interface{} `json:"literal"`
}
type FunctionSchema struct {
FunctionName string `json:"name" binding:"required"`
Description string `json:"description"`
FunctionType string `json:"type" binding:"required"`
InputFieldNames []string `json:"inputFieldNames" binding:"required"`
OutputFieldNames []string `json:"outputFieldNames" binding:"required"`
Params map[string]interface{} `json:"params"`
}
type CollectionSchema struct {
Fields []FieldSchema `json:"fields"`
StructFields []StructArrayFieldSchema `json:"structFields"`
Functions []FunctionSchema `json:"functions"`
AutoId bool `json:"autoID"`
EnableDynamicField bool `json:"enableDynamicField"`
ExternalSource string `json:"externalSource"`
ExternalSpec string `json:"externalSpec"`
}
type CollectionReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Dimension int32 `json:"dimension"`
IDType string `json:"idType"`
AutoID bool `json:"autoID"`
MetricType string `json:"metricType"`
VectorFieldType string `json:"vectorFieldType"`
ConsistencyLevel string `json:"consistencyLevel"`
PrimaryFieldName string `json:"primaryFieldName"`
VectorFieldName string `json:"vectorFieldName"`
Schema CollectionSchema `json:"schema"`
IndexParams []IndexParam `json:"indexParams"`
Params map[string]interface{} `json:"params"`
Properties map[string]interface{} `json:"properties"`
Description string `json:"description"`
// Top-level external config is accepted only for explicit rejection.
// Create external collection must use schema.externalSource/schema.externalSpec.
TopLevelExternalSource string `json:"externalSource"`
TopLevelExternalSpec string `json:"externalSpec"`
}
func (req *CollectionReq) GetDbName() string { return req.DbName }
func (req *CollectionReq) GetCollectionName() string { return req.CollectionName }
func (req *CollectionReq) GetExternalSource() string {
return req.Schema.ExternalSource
}
func (req *CollectionReq) GetExternalSpec() string {
return req.Schema.ExternalSpec
}
func (req *CollectionReq) HasTopLevelExternalConfig() bool {
return req.TopLevelExternalSource != "" || req.TopLevelExternalSpec != ""
}
type RefreshExternalCollectionReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
ExternalSource string `json:"externalSource"`
ExternalSpec string `json:"externalSpec"`
}
func (req *RefreshExternalCollectionReq) GetDbName() string { return req.DbName }
func (req *RefreshExternalCollectionReq) GetCollectionName() string {
return req.CollectionName
}
type RefreshExternalCollectionProgressReq struct {
JobID int64 `json:"jobId" binding:"required"`
}
func (req *RefreshExternalCollectionProgressReq) GetJobID() int64 { return req.JobID }
type AliasReq struct {
DbName string `json:"dbName"`
AliasName string `json:"aliasName" binding:"required"`
}
func (req *AliasReq) GetDbName() string { return req.DbName }
func (req *AliasReq) GetAliasName() string {
return req.AliasName
}
type AliasCollectionReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
AliasName string `json:"aliasName" binding:"required"`
}
func (req *AliasCollectionReq) GetDbName() string { return req.DbName }
func (req *AliasCollectionReq) GetCollectionName() string {
return req.CollectionName
}
func (req *AliasCollectionReq) GetAliasName() string {
return req.AliasName
}
func wrapperReturnHas(has bool) gin.H {
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: gin.H{HTTPReturnHas: has}}
}
func wrapperReturnList(names []string) gin.H {
if names == nil {
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: []string{}}
}
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: names}
}
func wrapperReturnRowCount(pairs []*commonpb.KeyValuePair) gin.H {
rowCountValue := "0"
for _, keyValue := range pairs {
if keyValue.Key == "row_count" {
rowCountValue = keyValue.GetValue()
}
}
rowCount, err := strconv.ParseInt(rowCountValue, 10, 64)
if err != nil {
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: gin.H{HTTPReturnRowCount: rowCountValue}}
}
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: gin.H{HTTPReturnRowCount: rowCount}}
}
func wrapperReturnDefault() gin.H {
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: gin.H{}}
}
func wrapperReturnDefaultWithCost(cost int) gin.H {
return gin.H{HTTPReturnCode: merr.Code(nil), HTTPReturnData: gin.H{}, HTTPReturnCost: cost}
}
type ResourceGroupNodeFilter struct {
NodeLabels map[string]string `json:"node_labels" binding:"required"`
}
func (req *ResourceGroupNodeFilter) GetNodeLabels() map[string]string {
return req.NodeLabels
}
type ResourceGroupLimit struct {
NodeNum int32 `json:"node_num" binding:"required"`
}
func (req *ResourceGroupLimit) GetNodeNum() int32 {
return req.NodeNum
}
type ResourceGroupTransfer struct {
ResourceGroup string `json:"resource_group" binding:"required"`
}
func (req *ResourceGroupTransfer) GetResourceGroup() string {
return req.ResourceGroup
}
type ResourceGroupConfig struct {
Requests *ResourceGroupLimit `json:"requests" binding:"required"`
Limits *ResourceGroupLimit `json:"limits" binding:"required"`
TransferFrom []*ResourceGroupTransfer `json:"transfer_from"`
TransferTo []*ResourceGroupTransfer `json:"transfer_to"`
NodeFilter *ResourceGroupNodeFilter `json:"node_filter"`
}
func (req *ResourceGroupConfig) GetRequests() *ResourceGroupLimit {
return req.Requests
}
func (req *ResourceGroupConfig) GetLimits() *ResourceGroupLimit {
return req.Limits
}
func (req *ResourceGroupConfig) GetTransferFrom() []*ResourceGroupTransfer {
return req.TransferFrom
}
func (req *ResourceGroupConfig) GetTransferTo() []*ResourceGroupTransfer {
return req.TransferTo
}
func (req *ResourceGroupConfig) GetNodeFilter() *ResourceGroupNodeFilter {
return req.NodeFilter
}
type ResourceGroupReq struct {
Name string `json:"name" binding:"required"`
Config *ResourceGroupConfig `json:"config"`
}
func (req *ResourceGroupReq) GetName() string {
return req.Name
}
func (req *ResourceGroupReq) GetConfig() *ResourceGroupConfig {
return req.Config
}
type UpdateResourceGroupReq struct {
ResourceGroups map[string]*ResourceGroupConfig `json:"resource_groups" binding:"required"`
}
func (req *UpdateResourceGroupReq) GetResourceGroups() map[string]*ResourceGroupConfig {
return req.ResourceGroups
}
type TransferReplicaReq struct {
SourceRgName string `json:"sourceRgName" binding:"required"`
TargetRgName string `json:"targetRgName" binding:"required"`
CollectionName string `json:"collectionName" binding:"required"`
ReplicaNum int64 `json:"replicaNum" binding:"required"`
}
func (req *TransferReplicaReq) GetSourceRgName() string {
return req.SourceRgName
}
func (req *TransferReplicaReq) GetTargetRgName() string {
return req.TargetRgName
}
func (req *TransferReplicaReq) GetCollectionName() string {
return req.CollectionName
}
func (req *TransferReplicaReq) GetReplicaNum() int64 {
return req.ReplicaNum
}
type GetSegmentsInfoReq struct {
DbName string `json:"dbName"`
CollectionID int64 `json:"collectionID"`
SegmentIDs []int64 `json:"segmentIDs"`
}
func (req *GetSegmentsInfoReq) GetDbName() string {
return req.DbName
}
func (req *GetSegmentsInfoReq) GetCollectionID() int64 {
return req.CollectionID
}
func (req *GetSegmentsInfoReq) GetSegmentIDs() []int64 {
return req.SegmentIDs
}
type GetQuotaMetricsReq struct{}
type RunAnalyzerReq struct {
DbName string `json:"dbName"`
AnalyzerParams string `json:"analyzerParams"`
Text []string `json:"text" binding:"required"`
WithDetail bool `json:"withDetail"`
WithHash bool `json:"withHash"`
CollectionName string `json:"collectionName"`
FieldName string `json:"fieldName"`
AnalyzerNames []string `json:"analyzerNames"`
}
func (req *RunAnalyzerReq) GetDbName() string { return req.DbName }
func (req *RunAnalyzerReq) GetCollectionName() string { return req.CollectionName }