chore: import upstream snapshot with attribution

components/document/parser/doc.go

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package parser defines the Parser interface for converting raw byte streams
+// into [schema.Document] values.
+//
+// # Overview
+//
+// A Parser is not a standalone pipeline component — it is used inside a
+// [document.Loader] to handle format-specific decoding. The loader fetches
+// raw bytes; the parser converts them into documents.
+//
+// # Built-in Implementations
+//
+//   - TextParser: treats the entire reader as plain text, one document per call
+//   - ExtParser: selects a parser by file extension (from [Options.URI]), with
+//     a configurable fallback for unknown extensions
+//
+// Use ExtParser when you want format-agnostic loading: pass the source URI
+// via [WithURI] and ExtParser picks the right sub-parser automatically.
+//
+// # Reader Contract
+//
+// The [io.Reader] passed to [Parser.Parse] is consumed during the call —
+// it cannot be read again. Loaders must not reuse the same reader across
+// multiple Parse calls.
+//
+// # Metadata Propagation
+//
+// Use [WithExtraMeta] to attach key-value pairs that are merged into every
+// document's MetaData. This is the standard way to tag documents with source
+// information (URI, content type, etc.) at parse time.
+//
+// See https://www.cloudwego.io/docs/eino/core_modules/components/document_loader_guide/document_parser_interface_guide/
+package parser

components/document/parser/ext_parser.go

@@ -0,0 +1,130 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"context"
+	"errors"
+	"io"
+	"path/filepath"
+
+	"github.com/cloudwego/eino/schema"
+)
+
+// ExtParserConfig defines the configuration for the ExtParser.
+type ExtParserConfig struct {
+	// ext -> parser.
+	// eg: map[string]Parser{
+	// 	".pdf": &PDFParser{},
+	// 	".md": &MarkdownParser{},
+	// }
+	Parsers map[string]Parser
+
+	// Fallback parser to use when no other parser is found.
+	// Default is TextParser if not set.
+	FallbackParser Parser
+}
+
+// ExtParser is a parser that uses the file extension to determine which parser to use.
+// You can register your own parsers by calling RegisterParser.
+// Default parser is TextParser.
+// Note:
+//
+//	parse 时，是通过 filepath.Ext(uri) 的方式找到对应的 parser，因此使用时需要：
+//	 	① 必须使用 parser.WithURI 在请求时传入 URI
+//	 	② URI 必须能通过 filepath.Ext 来解析出符合预期的 ext
+//
+// eg:
+//
+//	pdf, _ := os.Open("./testdata/test.pdf")
+//	docs, err := ExtParser.Parse(ctx, pdf, parser.WithURI("./testdata/test.pdf"))
+type ExtParser struct {
+	parsers map[string]Parser
+
+	fallbackParser Parser
+}
+
+// NewExtParser creates a new ExtParser.
+func NewExtParser(ctx context.Context, conf *ExtParserConfig) (*ExtParser, error) {
+	if conf == nil {
+		conf = &ExtParserConfig{}
+	}
+
+	p := &ExtParser{
+		parsers:        conf.Parsers,
+		fallbackParser: conf.FallbackParser,
+	}
+
+	if p.fallbackParser == nil {
+		p.fallbackParser = TextParser{}
+	}
+
+	if p.parsers == nil {
+		p.parsers = make(map[string]Parser)
+	}
+
+	return p, nil
+}
+
+// GetParsers returns a copy of the registered parsers.
+// It is safe to modify the returned parsers.
+func (p *ExtParser) GetParsers() map[string]Parser {
+	res := make(map[string]Parser, len(p.parsers))
+	for k, v := range p.parsers {
+		res[k] = v
+	}
+
+	return res
+}
+
+// Parse parses the given reader and returns a list of documents.
+func (p *ExtParser) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
+	opt := GetCommonOptions(&Options{}, opts...)
+
+	ext := filepath.Ext(opt.URI)
+
+	parser, ok := p.parsers[ext]
+
+	if !ok {
+		parser = p.fallbackParser
+	}
+
+	if parser == nil {
+		return nil, errors.New("no parser found for extension " + ext)
+	}
+
+	docs, err := parser.Parse(ctx, reader, opts...)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, doc := range docs {
+		if doc == nil {
+			continue
+		}
+
+		if doc.MetaData == nil {
+			doc.MetaData = make(map[string]any)
+		}
+
+		for k, v := range opt.ExtraMeta {
+			doc.MetaData[k] = v
+		}
+	}
+
+	return docs, nil
+}

components/document/parser/interface.go

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"context"
+	"io"
+
+	"github.com/cloudwego/eino/schema"
+)
+
+// Parser converts raw content from an [io.Reader] into [schema.Document] values.
+//
+// Parse may return multiple documents from a single reader (e.g. a PDF with
+// per-page splitting). The reader is consumed during Parse and must not be
+// reused.
+//
+// Parsers are typically not called directly — they are configured on a
+// [document.Loader] and invoked via [document.WithParserOptions].
+type Parser interface {
+	Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error)
+}

components/document/parser/option.go

@@ -0,0 +1,116 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+// Options configures the document parser with source URI and extra metadata.
+type Options struct {
+	// uri of source.
+	URI string
+
+	// extra metadata will merge to each document.
+	ExtraMeta map[string]any
+}
+
+// Option defines call option for Parser component, which is part of the component interface signature.
+// Each Parser implementation could define its own options struct and option funcs within its own package,
+// then wrap the impl specific option funcs into this type, before passing to Transform.
+type Option struct {
+	apply func(opts *Options)
+
+	implSpecificOptFn any
+}
+
+// WithURI specifies the source URI of the document.
+// It will be used as to select parser in ExtParser.
+func WithURI(uri string) Option {
+	return Option{
+		apply: func(opts *Options) {
+			opts.URI = uri
+		},
+	}
+}
+
+// WithExtraMeta attaches extra metadata to the parsed document.
+func WithExtraMeta(meta map[string]any) Option {
+	return Option{
+		apply: func(opts *Options) {
+			opts.ExtraMeta = meta
+		},
+	}
+}
+
+// GetCommonOptions extract parser Options from Option list, optionally providing a base Options with default values.
+func GetCommonOptions(base *Options, opts ...Option) *Options {
+	if base == nil {
+		base = &Options{}
+	}
+
+	for i := range opts {
+		opt := opts[i]
+		if opt.apply != nil {
+			opt.apply(base)
+		}
+	}
+
+	return base
+}
+
+// WrapImplSpecificOptFn wraps the impl specific option functions into Option type.
+// T: the type of the impl specific options struct.
+// Parser implementations are required to use this function to convert its own option functions into the unified Option type.
+// For example, if the Parser impl defines its own options struct:
+//
+//	type customOptions struct {
+//	    conf string
+//	}
+//
+// Then the impl needs to provide an option function as such:
+//
+//	func WithConf(conf string) Option {
+//	    return WrapImplSpecificOptFn(func(o *customOptions) {
+//			o.conf = conf
+//		}
+//	}
+//
+// .
+func WrapImplSpecificOptFn[T any](optFn func(*T)) Option {
+	return Option{
+		implSpecificOptFn: optFn,
+	}
+}
+
+// GetImplSpecificOptions provides Parser author the ability to extract their own custom options from the unified Option type.
+// T: the type of the impl specific options struct.
+// This function should be used within the Parser implementation's Transform function.
+// It is recommended to provide a base T as the first argument, within which the Parser author can provide default values for the impl specific options.
+func GetImplSpecificOptions[T any](base *T, opts ...Option) *T {
+	if base == nil {
+		base = new(T)
+	}
+
+	for i := range opts {
+		opt := opts[i]
+		if opt.implSpecificOptFn != nil {
+			s, ok := opt.implSpecificOptFn.(func(*T))
+			if ok {
+				s(base)
+			}
+		}
+	}
+
+	return base
+}

components/document/parser/option_test.go

@@ -0,0 +1,54 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestImplSpecificOpts(t *testing.T) {
+	type implSpecificOptions struct {
+		conf  string
+		index int
+	}
+
+	withConf := func(conf string) func(o *implSpecificOptions) {
+		return func(o *implSpecificOptions) {
+			o.conf = conf
+		}
+	}
+
+	withIndex := func(index int) func(o *implSpecificOptions) {
+		return func(o *implSpecificOptions) {
+			o.index = index
+		}
+	}
+
+	convey.Convey("TestImplSpecificOpts", t, func() {
+		parserOption1 := WrapImplSpecificOptFn(withConf("test_conf"))
+		parserOption2 := WrapImplSpecificOptFn(withIndex(1))
+
+		implSpecificOpts := GetImplSpecificOptions(&implSpecificOptions{}, parserOption1, parserOption2)
+
+		convey.So(implSpecificOpts, convey.ShouldResemble, &implSpecificOptions{
+			conf:  "test_conf",
+			index: 1,
+		})
+	})
+}

components/document/parser/parser_test.go

@@ -0,0 +1,118 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"context"
+	"io"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudwego/eino/schema"
+)
+
+type ParserForTest struct {
+	mock func() ([]*schema.Document, error)
+}
+
+func (p *ParserForTest) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
+	return p.mock()
+}
+
+func TestParser(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("Test default parser", func(t *testing.T) {
+		conf := &ExtParserConfig{}
+
+		p, err := NewExtParser(ctx, conf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		f, err := os.Open("testdata/test.md")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer f.Close()
+
+		docs, err := p.Parse(ctx, f, WithURI("testdata/test.md"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, 1, len(docs))
+		assert.Equal(t, "# Title\nhello world", docs[0].Content)
+	})
+
+	t.Run("test types", func(t *testing.T) {
+		mockParser := &ParserForTest{
+			mock: func() ([]*schema.Document, error) {
+				return []*schema.Document{
+					{
+						Content: "hello world",
+						MetaData: map[string]any{
+							"type": "text",
+						},
+					},
+				}, nil
+			},
+		}
+
+		conf := &ExtParserConfig{
+			Parsers: map[string]Parser{
+				".md": mockParser,
+			},
+		}
+
+		p, err := NewExtParser(ctx, conf)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		f, err := os.Open("testdata/test.md")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer f.Close()
+
+		docs, err := p.Parse(ctx, f, WithURI("x/test.md"))
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		assert.Equal(t, 1, len(docs))
+		assert.Equal(t, "hello world", docs[0].Content)
+		assert.Equal(t, "text", docs[0].MetaData["type"])
+	})
+
+	t.Run("test get parsers", func(t *testing.T) {
+		p, err := NewExtParser(ctx, &ExtParserConfig{
+			Parsers: map[string]Parser{
+				".md": &TextParser{},
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		ps := p.GetParsers()
+		assert.Equal(t, 1, len(ps))
+	})
+}

components/document/parser/testdata/test.md

@@ -0,0 +1,2 @@
+# Title
+hello world

components/document/parser/text_parser.go

@@ -0,0 +1,60 @@
+/*
+ * Copyright 2024 CloudWeGo Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parser
+
+import (
+	"context"
+	"io"
+
+	"github.com/cloudwego/eino/schema"
+)
+
+const (
+	// MetaKeySource is the metadata key storing the document's source URI.
+	MetaKeySource = "_source"
+)
+
+// TextParser is a simple parser that reads the text from a reader and returns a single document.
+// eg:
+//
+//	docs, err := TextParser.Parse(ctx, strings.NewReader("hello world"))
+//	fmt.Println(docs[0].Content) // "hello world"
+type TextParser struct{}
+
+// Parse reads the text from a reader and returns a single document.
+func (dp TextParser) Parse(ctx context.Context, reader io.Reader, opts ...Option) ([]*schema.Document, error) {
+	data, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	opt := GetCommonOptions(&Options{}, opts...)
+
+	meta := make(map[string]any)
+	meta[MetaKeySource] = opt.URI
+
+	for k, v := range opt.ExtraMeta {
+		meta[k] = v
+	}
+
+	doc := &schema.Document{
+		Content:  string(data),
+		MetaData: meta,
+	}
+
+	return []*schema.Document{doc}, nil
+}