chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"github.com/cloudwego/eino/callbacks"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// LoaderCallbackInput is the input for the loader callback.
|
||||
type LoaderCallbackInput struct {
|
||||
// Source is the source of the documents.
|
||||
Source Source
|
||||
|
||||
// Extra is the extra information for the callback.
|
||||
Extra map[string]any
|
||||
}
|
||||
|
||||
// LoaderCallbackOutput is the output for the loader callback.
|
||||
type LoaderCallbackOutput struct {
|
||||
// Source is the source of the documents.
|
||||
Source Source
|
||||
|
||||
// Docs is the documents to be loaded.
|
||||
Docs []*schema.Document
|
||||
|
||||
// Extra is the extra information for the callback.
|
||||
Extra map[string]any
|
||||
}
|
||||
|
||||
// ConvLoaderCallbackInput converts the callback input to the loader callback input.
|
||||
func ConvLoaderCallbackInput(src callbacks.CallbackInput) *LoaderCallbackInput {
|
||||
switch t := src.(type) {
|
||||
case *LoaderCallbackInput:
|
||||
return t
|
||||
case Source:
|
||||
return &LoaderCallbackInput{
|
||||
Source: t,
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ConvLoaderCallbackOutput converts the callback output to the loader callback output.
|
||||
func ConvLoaderCallbackOutput(src callbacks.CallbackOutput) *LoaderCallbackOutput {
|
||||
switch t := src.(type) {
|
||||
case *LoaderCallbackOutput:
|
||||
return t
|
||||
case []*schema.Document:
|
||||
return &LoaderCallbackOutput{
|
||||
Docs: t,
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"github.com/cloudwego/eino/callbacks"
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// TransformerCallbackInput is the input for the transformer callback.
|
||||
type TransformerCallbackInput struct {
|
||||
// Input is the input documents.
|
||||
Input []*schema.Document
|
||||
|
||||
// Extra is the extra information for the callback.
|
||||
Extra map[string]any
|
||||
}
|
||||
|
||||
// TransformerCallbackOutput is the output for the transformer callback.
|
||||
type TransformerCallbackOutput struct {
|
||||
// Output is the output documents.
|
||||
Output []*schema.Document
|
||||
|
||||
// Extra is the extra information for the callback.
|
||||
Extra map[string]any
|
||||
}
|
||||
|
||||
// ConvTransformerCallbackInput converts the callback input to the transformer callback input.
|
||||
func ConvTransformerCallbackInput(src callbacks.CallbackInput) *TransformerCallbackInput {
|
||||
switch t := src.(type) {
|
||||
case *TransformerCallbackInput:
|
||||
return t
|
||||
case []*schema.Document:
|
||||
return &TransformerCallbackInput{
|
||||
Input: t,
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// ConvTransformerCallbackOutput converts the callback output to the transformer callback output.
|
||||
func ConvTransformerCallbackOutput(src callbacks.CallbackOutput) *TransformerCallbackOutput {
|
||||
switch t := src.(type) {
|
||||
case *TransformerCallbackOutput:
|
||||
return t
|
||||
case []*schema.Document:
|
||||
return &TransformerCallbackOutput{
|
||||
Output: t,
|
||||
}
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Package document defines the Loader and Transformer component interfaces
|
||||
// for ingesting and processing documents in an eino pipeline.
|
||||
//
|
||||
// # Components
|
||||
//
|
||||
// - [Loader]: reads raw content from an external source (file, URL, S3, …)
|
||||
// and returns [schema.Document] values. Parsing is typically delegated to
|
||||
// a [parser.Parser] configured on the loader.
|
||||
// - [Transformer]: takes a slice of [schema.Document] values and transforms
|
||||
// them — splitting, filtering, merging, re-ranking, etc.
|
||||
//
|
||||
// Concrete implementations live in eino-ext:
|
||||
//
|
||||
// github.com/cloudwego/eino-ext/components/document/
|
||||
//
|
||||
// # Document Metadata
|
||||
//
|
||||
// [schema.Document].MetaData is the primary mechanism for carrying contextual
|
||||
// information (source URI, scores, chunk indices, embeddings) through the
|
||||
// pipeline. Transformers should preserve existing metadata and merge rather
|
||||
// than replace when adding their own keys.
|
||||
//
|
||||
// See https://www.cloudwego.io/docs/eino/core_modules/components/document_loader_guide/
|
||||
// See https://www.cloudwego.io/docs/eino/core_modules/components/document_transformer_guide/
|
||||
package document
|
||||
@@ -0,0 +1,55 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// Source identifies the external location of a document.
|
||||
// URI can be a local file path or a remote URL reachable by the loader.
|
||||
type Source struct {
|
||||
URI string
|
||||
}
|
||||
|
||||
//go:generate mockgen -destination ../../internal/mock/components/document/document_mock.go --package document -source interface.go
|
||||
|
||||
// Loader reads raw content from an external source and returns it as a slice
|
||||
// of [schema.Document] values.
|
||||
//
|
||||
// The Source.URI may be a local file path or a remote URL. The loader is
|
||||
// responsible for fetching the raw bytes; actual format parsing is typically
|
||||
// delegated to a [parser.Parser] configured on the loader via
|
||||
// [WithParserOptions].
|
||||
//
|
||||
// Document metadata ([schema.Document].MetaData) should be populated with at
|
||||
// least the source URI so that downstream nodes can trace document provenance.
|
||||
type Loader interface {
|
||||
Load(ctx context.Context, src Source, opts ...LoaderOption) ([]*schema.Document, error)
|
||||
}
|
||||
|
||||
// Transformer converts a slice of [schema.Document] values into another slice,
|
||||
// applying operations such as splitting, filtering, merging, or re-ranking.
|
||||
//
|
||||
// Implementations should preserve existing MetaData keys and merge rather than
|
||||
// replace when adding their own metadata. Downstream nodes (e.g. Indexer,
|
||||
// Retriever) may depend on metadata set by earlier pipeline stages.
|
||||
type Transformer interface {
|
||||
Transform(ctx context.Context, src []*schema.Document, opts ...TransformerOption) ([]*schema.Document, error)
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package document
|
||||
|
||||
import "github.com/cloudwego/eino/components/document/parser"
|
||||
|
||||
// LoaderOptions configures document loaders, including parser options.
|
||||
type LoaderOptions struct {
|
||||
ParserOptions []parser.Option
|
||||
}
|
||||
|
||||
// LoaderOption defines call option for Loader component, which is part of the component interface signature.
|
||||
// Each Loader implementation could define its own options struct and option funcs within its own package,
|
||||
// then wrap the impl specific option funcs into this type, before passing to Load.
|
||||
type LoaderOption struct {
|
||||
apply func(opts *LoaderOptions)
|
||||
|
||||
implSpecificOptFn any
|
||||
}
|
||||
|
||||
// WrapLoaderImplSpecificOptFn wraps the impl specific option functions into LoaderOption type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// Loader implementations are required to use this function to convert its own option functions into the unified LoaderOption type.
|
||||
// For example, if the Loader impl defines its own options struct:
|
||||
//
|
||||
// type customOptions struct {
|
||||
// conf string
|
||||
// }
|
||||
//
|
||||
// Then the impl needs to provide an option function as such:
|
||||
//
|
||||
// func WithConf(conf string) Option {
|
||||
// return WrapLoaderImplSpecificOptFn(func(o *customOptions) {
|
||||
// o.conf = conf
|
||||
// }
|
||||
// }
|
||||
func WrapLoaderImplSpecificOptFn[T any](optFn func(*T)) LoaderOption {
|
||||
return LoaderOption{
|
||||
implSpecificOptFn: optFn,
|
||||
}
|
||||
}
|
||||
|
||||
// GetLoaderImplSpecificOptions provides Loader author the ability to extract their own custom options from the unified LoaderOption type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// This function should be used within the Loader implementation's Load function.
|
||||
// It is recommended to provide a base T as the first argument, within which the Loader author can provide default values for the impl specific options.
|
||||
// eg.
|
||||
//
|
||||
// myOption := &MyOption{
|
||||
// Field1: "default_value",
|
||||
// }
|
||||
// myOption := loader.GetLoaderImplSpecificOptions(myOption, opts...)
|
||||
func GetLoaderImplSpecificOptions[T any](base *T, opts ...LoaderOption) *T {
|
||||
if base == nil {
|
||||
base = new(T)
|
||||
}
|
||||
|
||||
for i := range opts {
|
||||
opt := opts[i]
|
||||
if opt.implSpecificOptFn != nil {
|
||||
s, ok := opt.implSpecificOptFn.(func(*T))
|
||||
if ok {
|
||||
s(base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
// GetLoaderCommonOptions extract loader Options from Option list, optionally providing a base Options with default values.
|
||||
func GetLoaderCommonOptions(base *LoaderOptions, opts ...LoaderOption) *LoaderOptions {
|
||||
if base == nil {
|
||||
base = &LoaderOptions{}
|
||||
}
|
||||
|
||||
for i := range opts {
|
||||
opt := opts[i]
|
||||
if opt.apply != nil {
|
||||
opt.apply(base)
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
// WithParserOptions attaches parser options to a loader request.
|
||||
func WithParserOptions(opts ...parser.Option) LoaderOption {
|
||||
return LoaderOption{
|
||||
apply: func(o *LoaderOptions) {
|
||||
o.ParserOptions = opts
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// TransformerOption defines call option for Transformer component, which is part of the component interface signature.
|
||||
// Each Transformer implementation could define its own options struct and option funcs within its own package,
|
||||
// then wrap the impl specific option funcs into this type, before passing to Transform.
|
||||
type TransformerOption struct {
|
||||
implSpecificOptFn any
|
||||
}
|
||||
|
||||
// WrapTransformerImplSpecificOptFn wraps the impl specific option functions into TransformerOption type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// Transformer implementations are required to use this function to convert its own option functions into the unified TransformerOption type.
|
||||
// For example, if the Transformer impl defines its own options struct:
|
||||
//
|
||||
// type customOptions struct {
|
||||
// conf string
|
||||
// }
|
||||
//
|
||||
// Then the impl needs to provide an option function as such:
|
||||
//
|
||||
// func WithConf(conf string) TransformerOption {
|
||||
// return WrapTransformerImplSpecificOptFn(func(o *customOptions) {
|
||||
// o.conf = conf
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// .
|
||||
func WrapTransformerImplSpecificOptFn[T any](optFn func(*T)) TransformerOption {
|
||||
return TransformerOption{
|
||||
implSpecificOptFn: optFn,
|
||||
}
|
||||
}
|
||||
|
||||
// GetTransformerImplSpecificOptions provides Transformer author the ability to extract their own custom options from the unified TransformerOption type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// This function should be used within the Transformer implementation's Transform function.
|
||||
// It is recommended to provide a base T as the first argument, within which the Transformer author can provide default values for the impl specific options.
|
||||
// eg.
|
||||
//
|
||||
// myOption := &MyOption{
|
||||
// Field1: "default_value",
|
||||
// }
|
||||
// myOption := transformer.GetTransformerImplSpecificOptions(myOption, opts...)
|
||||
func GetTransformerImplSpecificOptions[T any](base *T, opts ...TransformerOption) *T {
|
||||
if base == nil {
|
||||
base = new(T)
|
||||
}
|
||||
|
||||
for i := range opts {
|
||||
opt := opts[i]
|
||||
if opt.implSpecificOptFn != nil {
|
||||
s, ok := opt.implSpecificOptFn.(func(*T))
|
||||
if ok {
|
||||
s(base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package document
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/smartystreets/goconvey/convey"
|
||||
|
||||
"github.com/cloudwego/eino/components/document/parser"
|
||||
)
|
||||
|
||||
func TestImplSpecificOpts(t *testing.T) {
|
||||
type implSpecificOptions struct {
|
||||
conf string
|
||||
index int
|
||||
}
|
||||
|
||||
withConf := func(conf string) func(o *implSpecificOptions) {
|
||||
return func(o *implSpecificOptions) {
|
||||
o.conf = conf
|
||||
}
|
||||
}
|
||||
|
||||
withIndex := func(index int) func(o *implSpecificOptions) {
|
||||
return func(o *implSpecificOptions) {
|
||||
o.index = index
|
||||
}
|
||||
}
|
||||
|
||||
convey.Convey("TestLoaderImplSpecificOpts", t, func() {
|
||||
documentOption1 := WrapLoaderImplSpecificOptFn(withConf("test_conf"))
|
||||
documentOption2 := WrapLoaderImplSpecificOptFn(withIndex(1))
|
||||
|
||||
implSpecificOpts := GetLoaderImplSpecificOptions(&implSpecificOptions{}, documentOption1, documentOption2)
|
||||
|
||||
convey.So(implSpecificOpts, convey.ShouldResemble, &implSpecificOptions{
|
||||
conf: "test_conf",
|
||||
index: 1,
|
||||
})
|
||||
})
|
||||
convey.Convey("TestTransformerImplSpecificOpts", t, func() {
|
||||
documentOption1 := WrapTransformerImplSpecificOptFn(withConf("test_conf"))
|
||||
documentOption2 := WrapTransformerImplSpecificOptFn(withIndex(1))
|
||||
|
||||
implSpecificOpts := GetTransformerImplSpecificOptions(&implSpecificOptions{}, documentOption1, documentOption2)
|
||||
|
||||
convey.So(implSpecificOpts, convey.ShouldResemble, &implSpecificOptions{
|
||||
conf: "test_conf",
|
||||
index: 1,
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestCommonOptions(t *testing.T) {
|
||||
convey.Convey("TestCommonOptions", t, func() {
|
||||
o := &LoaderOptions{ParserOptions: []parser.Option{{}}}
|
||||
o1 := GetLoaderCommonOptions(o)
|
||||
convey.So(len(o1.ParserOptions), convey.ShouldEqual, 1)
|
||||
|
||||
o2 := GetLoaderCommonOptions(o, WithParserOptions(parser.Option{}, parser.Option{}))
|
||||
convey.So(len(o2.ParserOptions), convey.ShouldEqual, 2)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
// Package parser defines the Parser interface for converting raw byte streams
|
||||
// into [schema.Document] values.
|
||||
//
|
||||
// # Overview
|
||||
//
|
||||
// A Parser is not a standalone pipeline component — it is used inside a
|
||||
// [document.Loader] to handle format-specific decoding. The loader fetches
|
||||
// raw bytes; the parser converts them into documents.
|
||||
//
|
||||
// # Built-in Implementations
|
||||
//
|
||||
// - TextParser: treats the entire reader as plain text, one document per call
|
||||
// - ExtParser: selects a parser by file extension (from [Options.URI]), with
|
||||
// a configurable fallback for unknown extensions
|
||||
//
|
||||
// Use ExtParser when you want format-agnostic loading: pass the source URI
|
||||
// via [WithURI] and ExtParser picks the right sub-parser automatically.
|
||||
//
|
||||
// # Reader Contract
|
||||
//
|
||||
// The [io.Reader] passed to [Parser.Parse] is consumed during the call —
|
||||
// it cannot be read again. Loaders must not reuse the same reader across
|
||||
// multiple Parse calls.
|
||||
//
|
||||
// # Metadata Propagation
|
||||
//
|
||||
// Use [WithExtraMeta] to attach key-value pairs that are merged into every
|
||||
// document's MetaData. This is the standard way to tag documents with source
|
||||
// information (URI, content type, etc.) at parse time.
|
||||
//
|
||||
// See https://www.cloudwego.io/docs/eino/core_modules/components/document_loader_guide/document_parser_interface_guide/
|
||||
package parser
|
||||
@@ -0,0 +1,130 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// ExtParserConfig defines the configuration for the ExtParser.
|
||||
type ExtParserConfig struct {
|
||||
// ext -> parser.
|
||||
// eg: map[string]Parser{
|
||||
// ".pdf": &PDFParser{},
|
||||
// ".md": &MarkdownParser{},
|
||||
// }
|
||||
Parsers map[string]Parser
|
||||
|
||||
// Fallback parser to use when no other parser is found.
|
||||
// Default is TextParser if not set.
|
||||
FallbackParser Parser
|
||||
}
|
||||
|
||||
// ExtParser is a parser that uses the file extension to determine which parser to use.
|
||||
// You can register your own parsers by calling RegisterParser.
|
||||
// Default parser is TextParser.
|
||||
// Note:
|
||||
//
|
||||
// parse 时，是通过 filepath.Ext(uri) 的方式找到对应的 parser，因此使用时需要：
|
||||
// ① 必须使用 parser.WithURI 在请求时传入 URI
|
||||
// ② URI 必须能通过 filepath.Ext 来解析出符合预期的 ext
|
||||
//
|
||||
// eg:
|
||||
//
|
||||
// pdf, _ := os.Open("./testdata/test.pdf")
|
||||
// docs, err := ExtParser.Parse(ctx, pdf, parser.WithURI("./testdata/test.pdf"))
|
||||
type ExtParser struct {
|
||||
parsers map[string]Parser
|
||||
|
||||
fallbackParser Parser
|
||||
}
|
||||
|
||||
// NewExtParser creates a new ExtParser.
|
||||
func NewExtParser(ctx context.Context, conf *ExtParserConfig) (*ExtParser, error) {
|
||||
if conf == nil {
|
||||
conf = &ExtParserConfig{}
|
||||
}
|
||||
|
||||
p := &ExtParser{
|
||||
parsers: conf.Parsers,
|
||||
fallbackParser: conf.FallbackParser,
|
||||
}
|
||||
|
||||
if p.fallbackParser == nil {
|
||||
p.fallbackParser = TextParser{}
|
||||
}
|
||||
|
||||
if p.parsers == nil {
|
||||
p.parsers = make(map[string]Parser)
|
||||
}
|
||||
|
||||
return p, nil
|
||||
}
|
||||
|
||||
// GetParsers returns a copy of the registered parsers.
|
||||
// It is safe to modify the returned parsers.
|
||||
func (p *ExtParser) GetParsers() map[string]Parser {
|
||||
res := make(map[string]Parser, len(p.parsers))
|
||||
for k, v := range p.parsers {
|
||||
res[k] = v
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// Parse parses the given reader and returns a list of documents.
|
||||
func (p *ExtParser) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
|
||||
opt := GetCommonOptions(&Options{}, opts...)
|
||||
|
||||
ext := filepath.Ext(opt.URI)
|
||||
|
||||
parser, ok := p.parsers[ext]
|
||||
|
||||
if !ok {
|
||||
parser = p.fallbackParser
|
||||
}
|
||||
|
||||
if parser == nil {
|
||||
return nil, errors.New("no parser found for extension " + ext)
|
||||
}
|
||||
|
||||
docs, err := parser.Parse(ctx, reader, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, doc := range docs {
|
||||
if doc == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if doc.MetaData == nil {
|
||||
doc.MetaData = make(map[string]any)
|
||||
}
|
||||
|
||||
for k, v := range opt.ExtraMeta {
|
||||
doc.MetaData[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return docs, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
// Parser converts raw content from an [io.Reader] into [schema.Document] values.
|
||||
//
|
||||
// Parse may return multiple documents from a single reader (e.g. a PDF with
|
||||
// per-page splitting). The reader is consumed during Parse and must not be
|
||||
// reused.
|
||||
//
|
||||
// Parsers are typically not called directly — they are configured on a
|
||||
// [document.Loader] and invoked via [document.WithParserOptions].
|
||||
type Parser interface {
|
||||
Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error)
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
// Options configures the document parser with source URI and extra metadata.
|
||||
type Options struct {
|
||||
// uri of source.
|
||||
URI string
|
||||
|
||||
// extra metadata will merge to each document.
|
||||
ExtraMeta map[string]any
|
||||
}
|
||||
|
||||
// Option defines call option for Parser component, which is part of the component interface signature.
|
||||
// Each Parser implementation could define its own options struct and option funcs within its own package,
|
||||
// then wrap the impl specific option funcs into this type, before passing to Transform.
|
||||
type Option struct {
|
||||
apply func(opts *Options)
|
||||
|
||||
implSpecificOptFn any
|
||||
}
|
||||
|
||||
// WithURI specifies the source URI of the document.
|
||||
// It will be used as to select parser in ExtParser.
|
||||
func WithURI(uri string) Option {
|
||||
return Option{
|
||||
apply: func(opts *Options) {
|
||||
opts.URI = uri
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// WithExtraMeta attaches extra metadata to the parsed document.
|
||||
func WithExtraMeta(meta map[string]any) Option {
|
||||
return Option{
|
||||
apply: func(opts *Options) {
|
||||
opts.ExtraMeta = meta
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetCommonOptions extract parser Options from Option list, optionally providing a base Options with default values.
|
||||
func GetCommonOptions(base *Options, opts ...Option) *Options {
|
||||
if base == nil {
|
||||
base = &Options{}
|
||||
}
|
||||
|
||||
for i := range opts {
|
||||
opt := opts[i]
|
||||
if opt.apply != nil {
|
||||
opt.apply(base)
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
|
||||
// WrapImplSpecificOptFn wraps the impl specific option functions into Option type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// Parser implementations are required to use this function to convert its own option functions into the unified Option type.
|
||||
// For example, if the Parser impl defines its own options struct:
|
||||
//
|
||||
// type customOptions struct {
|
||||
// conf string
|
||||
// }
|
||||
//
|
||||
// Then the impl needs to provide an option function as such:
|
||||
//
|
||||
// func WithConf(conf string) Option {
|
||||
// return WrapImplSpecificOptFn(func(o *customOptions) {
|
||||
// o.conf = conf
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// .
|
||||
func WrapImplSpecificOptFn[T any](optFn func(*T)) Option {
|
||||
return Option{
|
||||
implSpecificOptFn: optFn,
|
||||
}
|
||||
}
|
||||
|
||||
// GetImplSpecificOptions provides Parser author the ability to extract their own custom options from the unified Option type.
|
||||
// T: the type of the impl specific options struct.
|
||||
// This function should be used within the Parser implementation's Transform function.
|
||||
// It is recommended to provide a base T as the first argument, within which the Parser author can provide default values for the impl specific options.
|
||||
func GetImplSpecificOptions[T any](base *T, opts ...Option) *T {
|
||||
if base == nil {
|
||||
base = new(T)
|
||||
}
|
||||
|
||||
for i := range opts {
|
||||
opt := opts[i]
|
||||
if opt.implSpecificOptFn != nil {
|
||||
s, ok := opt.implSpecificOptFn.(func(*T))
|
||||
if ok {
|
||||
s(base)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return base
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestImplSpecificOpts(t *testing.T) {
|
||||
type implSpecificOptions struct {
|
||||
conf string
|
||||
index int
|
||||
}
|
||||
|
||||
withConf := func(conf string) func(o *implSpecificOptions) {
|
||||
return func(o *implSpecificOptions) {
|
||||
o.conf = conf
|
||||
}
|
||||
}
|
||||
|
||||
withIndex := func(index int) func(o *implSpecificOptions) {
|
||||
return func(o *implSpecificOptions) {
|
||||
o.index = index
|
||||
}
|
||||
}
|
||||
|
||||
convey.Convey("TestImplSpecificOpts", t, func() {
|
||||
parserOption1 := WrapImplSpecificOptFn(withConf("test_conf"))
|
||||
parserOption2 := WrapImplSpecificOptFn(withIndex(1))
|
||||
|
||||
implSpecificOpts := GetImplSpecificOptions(&implSpecificOptions{}, parserOption1, parserOption2)
|
||||
|
||||
convey.So(implSpecificOpts, convey.ShouldResemble, &implSpecificOptions{
|
||||
conf: "test_conf",
|
||||
index: 1,
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
type ParserForTest struct {
|
||||
mock func() ([]*schema.Document, error)
|
||||
}
|
||||
|
||||
func (p *ParserForTest) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
|
||||
return p.mock()
|
||||
}
|
||||
|
||||
func TestParser(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
t.Run("Test default parser", func(t *testing.T) {
|
||||
conf := &ExtParserConfig{}
|
||||
|
||||
p, err := NewExtParser(ctx, conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f, err := os.Open("testdata/test.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
docs, err := p.Parse(ctx, f, WithURI("testdata/test.md"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, len(docs))
|
||||
assert.Equal(t, "# Title\nhello world", docs[0].Content)
|
||||
})
|
||||
|
||||
t.Run("test types", func(t *testing.T) {
|
||||
mockParser := &ParserForTest{
|
||||
mock: func() ([]*schema.Document, error) {
|
||||
return []*schema.Document{
|
||||
{
|
||||
Content: "hello world",
|
||||
MetaData: map[string]any{
|
||||
"type": "text",
|
||||
},
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
conf := &ExtParserConfig{
|
||||
Parsers: map[string]Parser{
|
||||
".md": mockParser,
|
||||
},
|
||||
}
|
||||
|
||||
p, err := NewExtParser(ctx, conf)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f, err := os.Open("testdata/test.md")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
docs, err := p.Parse(ctx, f, WithURI("x/test.md"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
assert.Equal(t, 1, len(docs))
|
||||
assert.Equal(t, "hello world", docs[0].Content)
|
||||
assert.Equal(t, "text", docs[0].MetaData["type"])
|
||||
})
|
||||
|
||||
t.Run("test get parsers", func(t *testing.T) {
|
||||
p, err := NewExtParser(ctx, &ExtParserConfig{
|
||||
Parsers: map[string]Parser{
|
||||
".md": &TextParser{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
ps := p.GetParsers()
|
||||
assert.Equal(t, 1, len(ps))
|
||||
})
|
||||
}
|
||||
+2
@@ -0,0 +1,2 @@
|
||||
# Title
|
||||
hello world
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2024 CloudWeGo Authors
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package parser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
|
||||
"github.com/cloudwego/eino/schema"
|
||||
)
|
||||
|
||||
const (
|
||||
// MetaKeySource is the metadata key storing the document's source URI.
|
||||
MetaKeySource = "_source"
|
||||
)
|
||||
|
||||
// TextParser is a simple parser that reads the text from a reader and returns a single document.
|
||||
// eg:
|
||||
//
|
||||
// docs, err := TextParser.Parse(ctx, strings.NewReader("hello world"))
|
||||
// fmt.Println(docs[0].Content) // "hello world"
|
||||
type TextParser struct{}
|
||||
|
||||
// Parse reads the text from a reader and returns a single document.
|
||||
func (dp TextParser) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
opt := GetCommonOptions(&Options{}, opts...)
|
||||
|
||||
meta := make(map[string]any)
|
||||
meta[MetaKeySource] = opt.URI
|
||||
|
||||
for k, v := range opt.ExtraMeta {
|
||||
meta[k] = v
|
||||
}
|
||||
|
||||
doc := &schema.Document{
|
||||
Content: string(data),
|
||||
MetaData: meta,
|
||||
}
|
||||
|
||||
return []*schema.Document{doc}, nil
|
||||
}
|
||||
Reference in New Issue
Block a user