infiniflow--ragflow/internal/parser/chunk/split_test.go

//
// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package chunk

import (
	"strings"
	"testing"
	"unicode/utf8"
)

// runLength builds a SplitOperator for the "length" strategy from a DSL-style
// config (numbers as float64, as JSON decoding produces) and runs it.
func runLength(t *testing.T, text string, chunkSize, overlap float64) []ChunkData {
	t.Helper()
	op, err := NewSplitOperator(map[string]interface{}{
		"strategy": "length",
		"params": map[string]interface{}{
			"chunk_size": chunkSize,
			"overlap":    overlap,
		},
	})
	if err != nil {
		t.Fatalf("NewSplitOperator returned error: %v", err)
	}
	ctx := &ChunkContext{TextAfterPreprocess: text}
	if err := op.Execute(ctx); err != nil {
		t.Fatalf("Execute returned error: %v", err)
	}
	return ctx.SplitChunks
}

func TestSplitByLengthBasicWindowing(t *testing.T) {
	chunks := runLength(t, "abcdefghij", 4, 0) // 10 runes, size 4, no overlap

	want := []string{"abcd", "efgh", "ij"}
	if len(chunks) != len(want) {
		t.Fatalf("got %d chunks, want %d: %#v", len(chunks), len(want), chunks)
	}
	for i, w := range want {
		if chunks[i].Content != w {
			t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
		}
		if chunks[i].Index != i {
			t.Errorf("chunk %d Index = %d, want %d", i, chunks[i].Index, i)
		}
		if chunks[i].Size != utf8.RuneCountInString(w) {
			t.Errorf("chunk %d Size = %d, want %d", i, chunks[i].Size, utf8.RuneCountInString(w))
		}
	}
}

func TestSplitByLengthWithOverlap(t *testing.T) {
	chunks := runLength(t, "abcdefg", 4, 2) // size 4, overlap 2 -> step 2

	want := []string{"abcd", "cdef", "efg"}
	got := make([]string, len(chunks))
	for i, c := range chunks {
		got[i] = c.Content
	}
	if strings.Join(got, "|") != strings.Join(want, "|") {
		t.Fatalf("got %v, want %v", got, want)
	}
}

func TestSplitByLengthOverlapClampedTerminates(t *testing.T) {
	// overlap >= chunk_size must be clamped so the window still advances and
	// the call terminates instead of looping forever.
	chunks := runLength(t, "abcde", 3, 9)
	if len(chunks) == 0 {
		t.Fatal("expected chunks, got none")
	}
	// step = chunkSize-(chunkSize-1) = 1, so windows advance one rune at a time.
	if chunks[0].Content != "abc" {
		t.Errorf("first chunk = %q, want %q", chunks[0].Content, "abc")
	}
	last := chunks[len(chunks)-1]
	if !strings.HasSuffix("abcde", last.Content) {
		t.Errorf("last chunk %q is not a suffix of the input", last.Content)
	}
}

func TestSplitByLengthDefaultSize(t *testing.T) {
	// chunk_size <= 0 falls back to the default window; short text => one chunk.
	chunks := runLength(t, "hello world", 0, 0)
	if len(chunks) != 1 {
		t.Fatalf("got %d chunks, want 1", len(chunks))
	}
	if chunks[0].Content != "hello world" {
		t.Errorf("content = %q, want %q", chunks[0].Content, "hello world")
	}
}

func TestSplitByLengthRuneAware(t *testing.T) {
	// Multi-byte runes must be windowed by rune, never split mid-rune.
	text := "你好世界朋友" // 6 CJK runes, 3 bytes each
	chunks := runLength(t, text, 2, 0)

	want := []string{"你好", "世界", "朋友"}
	if len(chunks) != len(want) {
		t.Fatalf("got %d chunks, want %d", len(chunks), len(want))
	}
	for i, w := range want {
		if chunks[i].Content != w {
			t.Errorf("chunk %d = %q, want %q", i, chunks[i].Content, w)
		}
		if !utf8.ValidString(chunks[i].Content) {
			t.Errorf("chunk %d %q is not valid UTF-8 (rune was split)", i, chunks[i].Content)
		}
	}
}

func TestSplitByLengthNoOverlapReconstructs(t *testing.T) {
	// With no overlap, concatenating the chunks must reproduce the input exactly.
	text := "The quick brown fox jumps over the lazy dog."
	chunks := runLength(t, text, 7, 0)

	var sb strings.Builder
	for _, c := range chunks {
		sb.WriteString(c.Content)
	}
	if sb.String() != text {
		t.Errorf("reconstructed %q, want %q", sb.String(), text)
	}
}

func TestSplitByLengthEmptyInput(t *testing.T) {
	if chunks := runLength(t, "", 4, 0); len(chunks) != 0 {
		t.Errorf("empty input produced %d chunks, want 0", len(chunks))
	}
}

func TestNewSplitOperatorParsesLengthParams(t *testing.T) {
	op, err := NewSplitOperator(map[string]interface{}{
		"strategy": "length",
		"params": map[string]interface{}{
			"chunk_size": float64(128),
			"overlap":    float64(16),
		},
	})
	if err != nil {
		t.Fatalf("unexpected error: %v", err)
	}
	if op.chunkSize != 128 {
		t.Errorf("chunkSize = %d, want 128", op.chunkSize)
	}
	if op.overlap != 16 {
		t.Errorf("overlap = %d, want 16", op.overlap)
	}
}