wehub-resource-sync
498b235461
Build and test / Build and test AMD64 Ubuntu 22.04 (push) Failing after 0s
Publish Builder / amazonlinux2023 (push) Failing after 1s
Build and test / UT for Go (push) Has been skipped
Publish KRTE Images / KRTE (push) Failing after 1s
Build and test / Integration Test (push) Has been skipped
Build and test / Upload Code Coverage (push) Has been skipped
Publish Builder / rockylinux9 (push) Failing after 1s
Publish Builder / ubuntu22.04 (push) Failing after 0s
Publish Builder / ubuntu24.04 (push) Failing after 0s
Publish Gpu Builder / publish-gpu-builder (push) Failing after 1s
Publish Test Images / PyTest (push) Failing after 0s
Build and test / UT for Cpp (push) Has been cancelled
103 lines
3.6 KiB
Go
103 lines
3.6 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package proxy
|
|
|
|
import (
|
|
"context"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v3/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v3/msgpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v3/schemapb"
|
|
"github.com/milvus-io/milvus/pkg/v3/mq/msgstream"
|
|
"github.com/milvus-io/milvus/pkg/v3/util/commonpbutil"
|
|
"github.com/milvus-io/milvus/pkg/v3/util/typeutil"
|
|
)
|
|
|
|
func genInsertMsgsByPartition(ctx context.Context,
|
|
segmentID UniqueID,
|
|
partitionID UniqueID,
|
|
partitionName string,
|
|
rowOffsets []int,
|
|
channelName string,
|
|
insertMsg *msgstream.InsertMsg,
|
|
) ([]msgstream.TsMsg, error) {
|
|
threshold := Params.PulsarCfg.MaxMessageSize.GetAsInt()
|
|
|
|
// create empty insert message
|
|
createInsertMsg := func(segmentID UniqueID, channelName string) *msgstream.InsertMsg {
|
|
insertReq := &msgpb.InsertRequest{
|
|
Base: commonpbutil.NewMsgBase(
|
|
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
|
|
commonpbutil.WithTimeStamp(insertMsg.BeginTimestamp), // entity's timestamp was set to equal it.BeginTimestamp in preExecute()
|
|
commonpbutil.WithSourceID(insertMsg.Base.SourceID),
|
|
),
|
|
CollectionID: insertMsg.CollectionID,
|
|
PartitionID: partitionID,
|
|
DbName: insertMsg.DbName,
|
|
CollectionName: insertMsg.CollectionName,
|
|
PartitionName: partitionName,
|
|
SegmentID: segmentID,
|
|
ShardName: channelName,
|
|
Version: msgpb.InsertDataVersion_ColumnBased,
|
|
FieldsData: make([]*schemapb.FieldData, len(insertMsg.GetFieldsData())),
|
|
}
|
|
msg := &msgstream.InsertMsg{
|
|
BaseMsg: msgstream.BaseMsg{
|
|
Ctx: ctx,
|
|
},
|
|
InsertRequest: insertReq,
|
|
}
|
|
|
|
return msg
|
|
}
|
|
|
|
fieldsData := insertMsg.GetFieldsData()
|
|
idxComputer := typeutil.NewFieldDataIdxComputer(fieldsData)
|
|
|
|
repackedMsgs := make([]msgstream.TsMsg, 0)
|
|
requestSize := 0
|
|
msg := createInsertMsg(segmentID, channelName)
|
|
for _, offset := range rowOffsets {
|
|
fieldIdxs := idxComputer.Compute(int64(offset))
|
|
curRowMessageSize, err := typeutil.EstimateEntitySize(fieldsData, offset, fieldIdxs...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// If the insert message size exceeds the threshold, flush the current
|
|
// message first. A single row can be larger than the threshold, so do
|
|
// not emit an empty message before adding that row.
|
|
if msg.NumRows > 0 && requestSize+curRowMessageSize >= threshold {
|
|
repackedMsgs = append(repackedMsgs, msg)
|
|
msg = createInsertMsg(segmentID, channelName)
|
|
requestSize = 0
|
|
}
|
|
|
|
typeutil.AppendFieldData(msg.FieldsData, fieldsData, int64(offset), fieldIdxs...)
|
|
msg.HashValues = append(msg.HashValues, insertMsg.HashValues[offset])
|
|
msg.Timestamps = append(msg.Timestamps, insertMsg.Timestamps[offset])
|
|
msg.RowIDs = append(msg.RowIDs, insertMsg.RowIDs[offset])
|
|
msg.NumRows++
|
|
requestSize += curRowMessageSize
|
|
}
|
|
if msg.NumRows > 0 {
|
|
repackedMsgs = append(repackedMsgs, msg)
|
|
}
|
|
|
|
return repackedMsgs, nil
|
|
}
|