wehub-resource-sync
241 lines
7.9 KiB
Go
241 lines
7.9 KiB
Go
/*
|
|
* Licensed to the Apache Software Foundation (ASF) under one
|
|
* or more contributor license agreements. See the NOTICE file
|
|
* distributed with this work for additional information
|
|
* regarding copyright ownership. The ASF licenses this file
|
|
* to you under the Apache License, Version 2.0 (the
|
|
* "License"); you may not use this file except in compliance
|
|
* with the License. You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing,
|
|
* software distributed under the License is distributed on an
|
|
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
* KIND, either express or implied. See the License for the
|
|
* specific language governing permissions and limitations
|
|
* under the License.
|
|
*/
|
|
|
|
package vector_search_sync
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/apache/answer/internal/base/data"
|
|
"github.com/apache/answer/internal/entity"
|
|
"github.com/apache/answer/pkg/uid"
|
|
"github.com/apache/answer/plugin"
|
|
"github.com/segmentfault/pacman/log"
|
|
)
|
|
|
|
// NewPluginSyncer creates a new VectorSearchSyncer that reads from the database.
|
|
func NewPluginSyncer(data *data.Data) plugin.VectorSearchSyncer {
|
|
return &PluginSyncer{data: data}
|
|
}
|
|
|
|
// PluginSyncer implements plugin.VectorSearchSyncer.
|
|
// It aggregates question/answer text with comments for vector embedding.
|
|
type PluginSyncer struct {
|
|
data *data.Data
|
|
}
|
|
|
|
// GetQuestionsPage returns a page of questions with aggregated text
|
|
// (question title + body + all answers + all comments).
|
|
func (p *PluginSyncer) GetQuestionsPage(ctx context.Context, page, pageSize int) (
|
|
[]*plugin.VectorSearchContent, error) {
|
|
questions := make([]*entity.Question, 0)
|
|
startNum := (page - 1) * pageSize
|
|
err := p.data.DB.Context(ctx).Limit(pageSize, startNum).Find(&questions)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return p.buildQuestionContents(ctx, questions)
|
|
}
|
|
|
|
// GetAnswersPage returns a page of answers with aggregated text
|
|
// (parent question title + answer body + answer comments).
|
|
func (p *PluginSyncer) GetAnswersPage(ctx context.Context, page, pageSize int) (
|
|
[]*plugin.VectorSearchContent, error) {
|
|
answers := make([]*entity.Answer, 0)
|
|
startNum := (page - 1) * pageSize
|
|
err := p.data.DB.Context(ctx).Limit(pageSize, startNum).Find(&answers)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return p.buildAnswerContents(ctx, answers)
|
|
}
|
|
|
|
// buildQuestionContents aggregates each question with its answers and comments.
|
|
func (p *PluginSyncer) buildQuestionContents(ctx context.Context, questions []*entity.Question) (
|
|
[]*plugin.VectorSearchContent, error) {
|
|
result := make([]*plugin.VectorSearchContent, 0, len(questions))
|
|
for _, q := range questions {
|
|
meta := plugin.VectorSearchMetadata{
|
|
QuestionID: uid.DeShortID(q.ID),
|
|
}
|
|
|
|
var parts []string
|
|
parts = append(parts, fmt.Sprintf("Question: %s\n%s", q.Title, q.OriginalText))
|
|
|
|
// Get answers for this question
|
|
answers := make([]*entity.Answer, 0)
|
|
err := p.data.DB.Context(ctx).Where("question_id = ?", q.ID).Find(&answers)
|
|
if err != nil {
|
|
log.Warnf("get answers for question %s failed: %v", q.ID, err)
|
|
} else {
|
|
for _, a := range answers {
|
|
parts = append(parts, fmt.Sprintf("Answer: %s", a.OriginalText))
|
|
answerMeta := plugin.VectorSearchMetadataAnswer{
|
|
AnswerID: uid.DeShortID(a.ID),
|
|
}
|
|
|
|
// Get comments on this answer
|
|
answerComments := make([]*entity.Comment, 0)
|
|
err := p.data.DB.Context(ctx).Where("object_id = ?", a.ID).
|
|
OrderBy("created_at ASC").Limit(50).Find(&answerComments)
|
|
if err != nil {
|
|
log.Warnf("get comments for answer %s failed: %v", a.ID, err)
|
|
} else {
|
|
for _, c := range answerComments {
|
|
parts = append(parts, fmt.Sprintf("Comment on answer: %s", c.OriginalText))
|
|
answerMeta.Comments = append(answerMeta.Comments, plugin.VectorSearchMetadataComment{
|
|
CommentID: uid.DeShortID(c.ID),
|
|
})
|
|
}
|
|
}
|
|
meta.Answers = append(meta.Answers, answerMeta)
|
|
}
|
|
}
|
|
|
|
// Get comments on the question
|
|
questionComments := make([]*entity.Comment, 0)
|
|
err = p.data.DB.Context(ctx).Where("object_id = ?", q.ID).
|
|
OrderBy("created_at ASC").Limit(50).Find(&questionComments)
|
|
if err != nil {
|
|
log.Warnf("get comments for question %s failed: %v", q.ID, err)
|
|
} else {
|
|
for _, c := range questionComments {
|
|
parts = append(parts, fmt.Sprintf("Comment: %s", c.OriginalText))
|
|
meta.Comments = append(meta.Comments, plugin.VectorSearchMetadataComment{
|
|
CommentID: uid.DeShortID(c.ID),
|
|
})
|
|
}
|
|
}
|
|
|
|
metaJSON, _ := json.Marshal(meta)
|
|
result = append(result, &plugin.VectorSearchContent{
|
|
ObjectID: uid.DeShortID(q.ID),
|
|
ObjectType: "question",
|
|
Title: q.Title,
|
|
Content: strings.Join(parts, "\n\n"),
|
|
Metadata: string(metaJSON),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// buildAnswerContents aggregates each answer with its parent question title and comments.
|
|
func (p *PluginSyncer) buildAnswerContents(ctx context.Context, answers []*entity.Answer) (
|
|
[]*plugin.VectorSearchContent, error) {
|
|
result := make([]*plugin.VectorSearchContent, 0, len(answers))
|
|
for _, a := range answers {
|
|
// Get parent question for title
|
|
question := &entity.Question{}
|
|
exist, err := p.data.DB.Context(ctx).Where("id = ?", a.QuestionID).Get(question)
|
|
if err != nil {
|
|
log.Errorf("get question %s failed: %v", a.QuestionID, err)
|
|
continue
|
|
}
|
|
if !exist {
|
|
continue
|
|
}
|
|
|
|
meta := plugin.VectorSearchMetadata{
|
|
QuestionID: uid.DeShortID(a.QuestionID),
|
|
AnswerID: uid.DeShortID(a.ID),
|
|
}
|
|
|
|
var parts []string
|
|
parts = append(parts, fmt.Sprintf("Question: %s", question.Title))
|
|
parts = append(parts, fmt.Sprintf("Answer: %s", a.OriginalText))
|
|
|
|
answerMeta := plugin.VectorSearchMetadataAnswer{
|
|
AnswerID: uid.DeShortID(a.ID),
|
|
}
|
|
|
|
// Get comments on this answer
|
|
answerComments := make([]*entity.Comment, 0)
|
|
err = p.data.DB.Context(ctx).Where("object_id = ?", a.ID).
|
|
OrderBy("created_at ASC").Limit(50).Find(&answerComments)
|
|
if err != nil {
|
|
log.Warnf("get comments for answer %s failed: %v", a.ID, err)
|
|
} else {
|
|
for _, c := range answerComments {
|
|
parts = append(parts, fmt.Sprintf("Comment: %s", c.OriginalText))
|
|
answerMeta.Comments = append(answerMeta.Comments, plugin.VectorSearchMetadataComment{
|
|
CommentID: uid.DeShortID(c.ID),
|
|
})
|
|
}
|
|
}
|
|
meta.Answers = append(meta.Answers, answerMeta)
|
|
|
|
metaJSON, _ := json.Marshal(meta)
|
|
result = append(result, &plugin.VectorSearchContent{
|
|
ObjectID: uid.DeShortID(a.ID),
|
|
ObjectType: "answer",
|
|
Title: question.Title,
|
|
Content: strings.Join(parts, "\n\n"),
|
|
Metadata: string(metaJSON),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
// BuildQuestionContentByID builds vector content for one question using the same
|
|
// aggregation semantics as bulk vector sync.
|
|
func BuildQuestionContentByID(ctx context.Context, data *data.Data, questionID string) (*plugin.VectorSearchContent, error) {
|
|
question := &entity.Question{}
|
|
exist, err := data.DB.Context(ctx).Where("id = ?", uid.DeShortID(questionID)).Get(question)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !exist {
|
|
return nil, nil
|
|
}
|
|
syncer := &PluginSyncer{data: data}
|
|
contents, err := syncer.buildQuestionContents(ctx, []*entity.Question{question})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(contents) == 0 {
|
|
return nil, nil
|
|
}
|
|
return contents[0], nil
|
|
}
|
|
|
|
// BuildAnswerContentByID builds vector content for one answer using the same
|
|
// aggregation semantics as bulk vector sync.
|
|
func BuildAnswerContentByID(ctx context.Context, data *data.Data, answerID string) (*plugin.VectorSearchContent, error) {
|
|
answer := &entity.Answer{}
|
|
exist, err := data.DB.Context(ctx).Where("id = ?", uid.DeShortID(answerID)).Get(answer)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if !exist {
|
|
return nil, nil
|
|
}
|
|
syncer := &PluginSyncer{data: data}
|
|
contents, err := syncer.buildAnswerContents(ctx, []*entity.Answer{answer})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(contents) == 0 {
|
|
return nil, nil
|
|
}
|
|
return contents[0], nil
|
|
}
|