chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/bytedance/sonic"
|
||||
|
||||
knowledge "github.com/coze-dev/coze-studio/backend/crossdomain/knowledge/model"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/internal/consts"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/internal/convert"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/internal/dal/model"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/internal/events"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/repository"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/document"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/document/parser"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/eventbus"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/idgen"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/rdb"
|
||||
rdbEntity "github.com/coze-dev/coze-studio/backend/infra/rdb/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/storage"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/errorx"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
"github.com/coze-dev/coze-studio/backend/types/errno"
|
||||
)
|
||||
|
||||
type baseDocProcessor struct {
|
||||
ctx context.Context
|
||||
UserID int64
|
||||
SpaceID int64
|
||||
Documents []*entity.Document
|
||||
documentSource *entity.DocumentSource
|
||||
|
||||
// Drop DB model
|
||||
TableName string
|
||||
docModels []*model.KnowledgeDocument
|
||||
imageSlices []*model.KnowledgeDocumentSlice
|
||||
|
||||
storage storage.Storage
|
||||
knowledgeRepo repository.KnowledgeRepo
|
||||
documentRepo repository.KnowledgeDocumentRepo
|
||||
sliceRepo repository.KnowledgeDocumentSliceRepo
|
||||
idgen idgen.IDGenerator
|
||||
rdb rdb.RDB
|
||||
producer eventbus.Producer
|
||||
parseManager parser.Manager
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) BeforeCreate() error {
|
||||
// Pull data from a data source
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) BuildDBModel() error {
|
||||
p.docModels = make([]*model.KnowledgeDocument, 0, len(p.Documents))
|
||||
for i := range p.Documents {
|
||||
id, err := p.idgen.GenID(p.ctx)
|
||||
if err != nil {
|
||||
logs.CtxErrorf(p.ctx, "gen id failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeIDGenCode)
|
||||
}
|
||||
docModel := &model.KnowledgeDocument{
|
||||
ID: id,
|
||||
KnowledgeID: p.Documents[i].KnowledgeID,
|
||||
Name: p.Documents[i].Name,
|
||||
FileExtension: string(p.Documents[i].FileExtension),
|
||||
URI: p.Documents[i].URI,
|
||||
DocumentType: int32(p.Documents[i].Type),
|
||||
CreatorID: p.UserID,
|
||||
SpaceID: p.SpaceID,
|
||||
SourceType: int32(p.Documents[i].Source),
|
||||
Status: int32(knowledge.KnowledgeStatusInit),
|
||||
ParseRule: &model.DocumentParseRule{
|
||||
ParsingStrategy: p.Documents[i].ParsingStrategy,
|
||||
ChunkingStrategy: p.Documents[i].ChunkingStrategy,
|
||||
},
|
||||
CreatedAt: time.Now().UnixMilli(),
|
||||
UpdatedAt: time.Now().UnixMilli(),
|
||||
}
|
||||
p.Documents[i].ID = docModel.ID
|
||||
p.docModels = append(p.docModels, docModel)
|
||||
if p.Documents[i].Type == knowledge.DocumentTypeImage {
|
||||
id, err := p.idgen.GenID(p.ctx)
|
||||
if err != nil {
|
||||
logs.CtxErrorf(p.ctx, "gen id failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeIDGenCode)
|
||||
}
|
||||
p.imageSlices = append(p.imageSlices, &model.KnowledgeDocumentSlice{
|
||||
ID: id,
|
||||
KnowledgeID: p.Documents[i].KnowledgeID,
|
||||
DocumentID: p.Documents[i].ID,
|
||||
CreatedAt: time.Now().UnixMilli(),
|
||||
UpdatedAt: time.Now().UnixMilli(),
|
||||
CreatorID: p.UserID,
|
||||
SpaceID: p.SpaceID,
|
||||
Status: int32(knowledge.SliceStatusInit),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) InsertDBModel() (err error) {
|
||||
ctx := p.ctx
|
||||
|
||||
if !isTableAppend(p.Documents) {
|
||||
err = p.createTable()
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "create table failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeCrossDomainCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
}
|
||||
|
||||
tx, err := p.knowledgeRepo.InitTx()
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "init tx failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
defer func() {
|
||||
if e := recover(); e != nil {
|
||||
logs.CtxErrorf(ctx, "panic: %v", e)
|
||||
err = errorx.New(errno.ErrKnowledgeSystemCode, errorx.KVf("msg", "panic: %v", e))
|
||||
tx.Rollback()
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "InsertDBModel err: %v", err)
|
||||
tx.Rollback()
|
||||
if p.TableName != "" {
|
||||
deleteErr := p.deleteTable()
|
||||
if deleteErr != nil {
|
||||
logs.CtxErrorf(ctx, "delete table failed, err: %v", deleteErr)
|
||||
return
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tx.Commit()
|
||||
}
|
||||
}()
|
||||
err = p.documentRepo.CreateWithTx(ctx, tx, p.docModels)
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "create document failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
err = p.sliceRepo.BatchCreateWithTX(ctx, tx, p.imageSlices)
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "update knowledge failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
err = p.knowledgeRepo.UpdateWithTx(ctx, tx, p.Documents[0].KnowledgeID, map[string]interface{}{
|
||||
"updated_at": time.Now().UnixMilli(),
|
||||
})
|
||||
if err != nil {
|
||||
logs.CtxErrorf(ctx, "update knowledge failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) createTable() error {
|
||||
if len(p.Documents) == 1 && p.Documents[0].Type == knowledge.DocumentTypeTable {
|
||||
// Tabular knowledge base, creating tables
|
||||
rdbColumns := []*rdbEntity.Column{}
|
||||
tableColumns := p.Documents[0].TableInfo.Columns
|
||||
columnIDs, err := p.idgen.GenMultiIDs(p.ctx, len(tableColumns)+1)
|
||||
if err != nil {
|
||||
logs.CtxErrorf(p.ctx, "gen ids failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeIDGenCode)
|
||||
}
|
||||
for i := range tableColumns {
|
||||
tableColumns[i].ID = columnIDs[i]
|
||||
rdbColumns = append(rdbColumns, &rdbEntity.Column{
|
||||
Name: convert.ColumnIDToRDBField(columnIDs[i]),
|
||||
DataType: convert.ConvertColumnType(tableColumns[i].Type),
|
||||
NotNull: tableColumns[i].Indexing,
|
||||
})
|
||||
}
|
||||
p.Documents[0].TableInfo.Columns = append(p.Documents[0].TableInfo.Columns, &entity.TableColumn{
|
||||
ID: columnIDs[len(columnIDs)-1],
|
||||
Name: consts.RDBFieldID,
|
||||
Type: document.TableColumnTypeInteger,
|
||||
Description: "主键ID",
|
||||
Indexing: false,
|
||||
Sequence: -1,
|
||||
})
|
||||
// Add a primary key ID to each table
|
||||
rdbColumns = append(rdbColumns, &rdbEntity.Column{
|
||||
Name: consts.RDBFieldID,
|
||||
DataType: rdbEntity.TypeBigInt,
|
||||
NotNull: true,
|
||||
})
|
||||
// Create a data table
|
||||
resp, err := p.rdb.CreateTable(p.ctx, &rdb.CreateTableRequest{
|
||||
Table: &rdbEntity.Table{
|
||||
Columns: rdbColumns,
|
||||
Indexes: []*rdbEntity.Index{
|
||||
{
|
||||
Name: "pk",
|
||||
Type: rdbEntity.PrimaryKey,
|
||||
Columns: []string{consts.RDBFieldID},
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logs.CtxErrorf(p.ctx, "create table failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeCrossDomainCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
p.TableName = resp.Table.Name
|
||||
p.Documents[0].TableInfo.PhysicalTableName = p.TableName
|
||||
p.docModels[0].TableInfo = &entity.TableInfo{
|
||||
VirtualTableName: p.Documents[0].Name,
|
||||
PhysicalTableName: p.TableName,
|
||||
TableDesc: p.Documents[0].Description,
|
||||
Columns: p.Documents[0].TableInfo.Columns,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) deleteTable() error {
|
||||
if len(p.Documents) == 1 && p.Documents[0].Type == knowledge.DocumentTypeTable {
|
||||
_, err := p.rdb.DropTable(p.ctx, &rdb.DropTableRequest{
|
||||
TableName: p.TableName,
|
||||
IfExists: false,
|
||||
})
|
||||
if err != nil {
|
||||
logs.CtxErrorf(p.ctx, "[deleteTable] drop table failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeCrossDomainCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) Indexing() error {
|
||||
event := events.NewIndexDocumentsEvent(p.Documents[0].KnowledgeID, p.Documents)
|
||||
body, err := sonic.Marshal(event)
|
||||
if err != nil {
|
||||
return errorx.New(errno.ErrKnowledgeParseJSONCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
|
||||
if err = p.producer.Send(p.ctx, body); err != nil {
|
||||
logs.CtxErrorf(p.ctx, "send message failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeMQSendFailCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *baseDocProcessor) GetResp() []*entity.Document {
|
||||
return p.Documents
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import "github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
|
||||
// Create a document after the user enters custom content
|
||||
type customDocProcessor struct {
|
||||
baseDocProcessor
|
||||
}
|
||||
|
||||
func (c *customDocProcessor) BeforeCreate() error {
|
||||
for i := range c.Documents {
|
||||
if c.Documents[i].RawContent != "" {
|
||||
c.Documents[i].FileExtension = getFormatType(c.Documents[i].Type)
|
||||
uri := getTosUri(c.UserID, string(c.Documents[i].FileExtension))
|
||||
err := c.storage.PutObject(c.ctx, uri, []byte(c.Documents[i].RawContent))
|
||||
if err != nil {
|
||||
logs.CtxErrorf(c.ctx, "put object failed, err: %v", err)
|
||||
return err
|
||||
}
|
||||
c.Documents[i].URI = uri
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import (
|
||||
model "github.com/coze-dev/coze-studio/backend/crossdomain/knowledge/model"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/errorx"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
"github.com/coze-dev/coze-studio/backend/types/errno"
|
||||
)
|
||||
|
||||
// User-defined form creation document
|
||||
type customTableProcessor struct {
|
||||
baseDocProcessor
|
||||
}
|
||||
|
||||
func (c *customTableProcessor) BeforeCreate() error {
|
||||
if isTableAppend(c.Documents) {
|
||||
tableDoc, _, err := c.documentRepo.FindDocumentByCondition(c.ctx, &entity.WhereDocumentOpt{KnowledgeIDs: []int64{c.Documents[0].KnowledgeID}, SelectAll: true})
|
||||
if err != nil {
|
||||
logs.CtxErrorf(c.ctx, "find document failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
if len(tableDoc) == 0 {
|
||||
logs.CtxErrorf(c.ctx, "table doc not found")
|
||||
return errorx.New(errno.ErrKnowledgeDocumentNotExistCode, errorx.KV("msg", "doc not found"))
|
||||
}
|
||||
c.Documents[0].ID = tableDoc[0].ID
|
||||
if tableDoc[0].TableInfo == nil {
|
||||
logs.CtxErrorf(c.ctx, "table info not found")
|
||||
return errorx.New(errno.ErrKnowledgeTableInfoNotExistCode, errorx.KVf("msg", "table info not found, doc_id: %d", tableDoc[0].ID))
|
||||
}
|
||||
c.Documents[0].TableInfo = *tableDoc[0].TableInfo
|
||||
// append scene
|
||||
if c.Documents[0].RawContent != "" {
|
||||
c.Documents[0].FileExtension = getFormatType(c.Documents[0].Type)
|
||||
uri := getTosUri(c.UserID, string(c.Documents[0].FileExtension))
|
||||
err := c.storage.PutObject(c.ctx, uri, []byte(c.Documents[0].RawContent))
|
||||
if err != nil {
|
||||
logs.CtxErrorf(c.ctx, "put object failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgePutObjectFailCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
c.Documents[0].URI = uri
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *customTableProcessor) BuildDBModel() error {
|
||||
if len(c.Documents) > 0 &&
|
||||
c.Documents[0].Type == model.DocumentTypeTable {
|
||||
if c.Documents[0].IsAppend {
|
||||
// Append the scene, no need to create a table
|
||||
// First, the user customizes some data, and second, uploads another form and appends the data in the form to the form
|
||||
} else {
|
||||
err := c.baseDocProcessor.BuildDBModel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Since this method of creation does not carry any data, the state is set to available directly
|
||||
for i := range c.docModels {
|
||||
c.docModels[i].DocumentType = 1
|
||||
c.docModels[i].Status = int32(entity.DocumentStatusInit)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *customTableProcessor) InsertDBModel() error {
|
||||
if isTableAppend(c.Documents) {
|
||||
// Append the scene and set the document to the processing state
|
||||
err := c.documentRepo.SetStatus(c.ctx, c.Documents[0].ID, int32(entity.DocumentStatusUploading), "")
|
||||
if err != nil {
|
||||
logs.CtxErrorf(c.ctx, "document set status err:%v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return c.baseDocProcessor.InsertDBModel()
|
||||
}
|
||||
|
||||
func (c *customTableProcessor) Indexing() error {
|
||||
// c.baseDocProcessor.Indexing()
|
||||
if isTableAppend(c.Documents) {
|
||||
err := c.baseDocProcessor.Indexing()
|
||||
if err != nil {
|
||||
logs.CtxErrorf(c.ctx, "document indexing err:%v", err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
knowledge "github.com/coze-dev/coze-studio/backend/crossdomain/knowledge/model"
|
||||
model "github.com/coze-dev/coze-studio/backend/crossdomain/knowledge/model"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/processor"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/repository"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/document/parser"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/eventbus"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/idgen"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/rdb"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/storage"
|
||||
)
|
||||
|
||||
type DocProcessorConfig struct {
|
||||
UserID int64
|
||||
SpaceID int64
|
||||
DocumentSource entity.DocumentSource
|
||||
Documents []*entity.Document
|
||||
|
||||
KnowledgeRepo repository.KnowledgeRepo
|
||||
DocumentRepo repository.KnowledgeDocumentRepo
|
||||
SliceRepo repository.KnowledgeDocumentSliceRepo
|
||||
Idgen idgen.IDGenerator
|
||||
Storage storage.Storage
|
||||
Rdb rdb.RDB
|
||||
Producer eventbus.Producer
|
||||
ParseManager parser.Manager
|
||||
}
|
||||
|
||||
func NewDocProcessor(ctx context.Context, config *DocProcessorConfig) (p processor.DocProcessor) {
|
||||
base := &baseDocProcessor{
|
||||
ctx: ctx,
|
||||
UserID: config.UserID,
|
||||
SpaceID: config.SpaceID,
|
||||
Documents: config.Documents,
|
||||
documentSource: &config.DocumentSource,
|
||||
knowledgeRepo: config.KnowledgeRepo,
|
||||
documentRepo: config.DocumentRepo,
|
||||
sliceRepo: config.SliceRepo,
|
||||
storage: config.Storage,
|
||||
idgen: config.Idgen,
|
||||
rdb: config.Rdb,
|
||||
producer: config.Producer,
|
||||
parseManager: config.ParseManager,
|
||||
}
|
||||
|
||||
switch config.DocumentSource {
|
||||
case entity.DocumentSourceCustom:
|
||||
p = &customDocProcessor{
|
||||
baseDocProcessor: *base,
|
||||
}
|
||||
if config.Documents[0].Type == model.DocumentTypeTable {
|
||||
p = &customTableProcessor{
|
||||
baseDocProcessor: *base,
|
||||
}
|
||||
}
|
||||
return p
|
||||
case entity.DocumentSourceLocal:
|
||||
if config.Documents[0].Type == knowledge.DocumentTypeTable {
|
||||
return &localTableProcessor{
|
||||
baseDocProcessor: *base,
|
||||
}
|
||||
}
|
||||
return base
|
||||
default:
|
||||
return base
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import (
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/errorx"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/lang/ptr"
|
||||
"github.com/coze-dev/coze-studio/backend/pkg/logs"
|
||||
"github.com/coze-dev/coze-studio/backend/types/errno"
|
||||
)
|
||||
|
||||
type localTableProcessor struct {
|
||||
baseDocProcessor
|
||||
}
|
||||
|
||||
func (l *localTableProcessor) BeforeCreate() error {
|
||||
if isTableAppend(l.Documents) {
|
||||
tableDoc, _, err := l.documentRepo.FindDocumentByCondition(l.ctx, &entity.WhereDocumentOpt{
|
||||
KnowledgeIDs: []int64{l.Documents[0].KnowledgeID},
|
||||
SelectAll: true,
|
||||
})
|
||||
if err != nil {
|
||||
logs.CtxErrorf(l.ctx, "find document failed, err: %v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
|
||||
if len(tableDoc) == 0 {
|
||||
logs.CtxErrorf(l.ctx, "table doc not found")
|
||||
return errorx.New(errno.ErrKnowledgeDocumentNotExistCode, errorx.KV("msg", "doc not found"))
|
||||
}
|
||||
|
||||
l.Documents[0].ID = tableDoc[0].ID
|
||||
|
||||
if tableDoc[0].TableInfo == nil {
|
||||
logs.CtxErrorf(l.ctx, "table info not found")
|
||||
return errorx.New(errno.ErrKnowledgeTableInfoNotExistCode, errorx.KVf("msg", "table info not found, doc_id: %d", tableDoc[0].ID))
|
||||
}
|
||||
l.Documents[0].TableInfo = ptr.From(tableDoc[0].TableInfo)
|
||||
return nil
|
||||
}
|
||||
return l.baseDocProcessor.BeforeCreate()
|
||||
}
|
||||
|
||||
func (l *localTableProcessor) BuildDBModel() error {
|
||||
if isTableAppend(l.Documents) {
|
||||
return nil
|
||||
}
|
||||
return l.baseDocProcessor.BuildDBModel()
|
||||
}
|
||||
|
||||
func (l *localTableProcessor) InsertDBModel() error {
|
||||
if isTableAppend(l.Documents) {
|
||||
// Append the scene and set the document to the processing state
|
||||
err := l.documentRepo.SetStatus(l.ctx, l.Documents[0].ID, int32(entity.DocumentStatusUploading), "")
|
||||
if err != nil {
|
||||
logs.CtxErrorf(l.ctx, "document set status err:%v", err)
|
||||
return errorx.New(errno.ErrKnowledgeDBCode, errorx.KV("msg", err.Error()))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return l.baseDocProcessor.InsertDBModel()
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/*
|
||||
* Copyright 2025 coze-dev Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package impl
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
knowledge "github.com/coze-dev/coze-studio/backend/crossdomain/knowledge/model"
|
||||
"github.com/coze-dev/coze-studio/backend/domain/knowledge/entity"
|
||||
"github.com/coze-dev/coze-studio/backend/infra/document/parser"
|
||||
)
|
||||
|
||||
func getFormatType(tp knowledge.DocumentType) parser.FileExtension {
|
||||
docType := parser.FileExtensionTXT
|
||||
if tp == knowledge.DocumentTypeTable {
|
||||
docType = parser.FileExtensionJSON
|
||||
}
|
||||
return docType
|
||||
}
|
||||
|
||||
func getTosUri(userID int64, fileType string) string {
|
||||
fileName := fmt.Sprintf("FileBizType.Knowledge/%d_%d.%s", userID, time.Now().UnixNano(), fileType)
|
||||
return fileName
|
||||
}
|
||||
|
||||
func isTableAppend(docs []*entity.Document) bool {
|
||||
return len(docs) > 0 &&
|
||||
docs[0].Type == knowledge.DocumentTypeTable &&
|
||||
docs[0].IsAppend
|
||||
}
|
||||
Reference in New Issue
Block a user