wehub-resource-sync
42 lines
964 B
TypeScript
42 lines
964 B
TypeScript
const asciiTokenPattern = /[a-z0-9_'-]+/gi;
|
|
const hanTokenPattern = /[\u3400-\u9fff]+/g;
|
|
|
|
function unique(tokens: string[]) {
|
|
return Array.from(new Set(tokens.filter(Boolean)));
|
|
}
|
|
|
|
function tokenizeCjkText(input: string) {
|
|
const source = input.toLowerCase();
|
|
const tokens: string[] = [];
|
|
|
|
for (const match of source.matchAll(asciiTokenPattern)) {
|
|
tokens.push(match[0]);
|
|
}
|
|
|
|
for (const match of source.matchAll(hanTokenPattern)) {
|
|
const text = match[0];
|
|
tokens.push(text);
|
|
|
|
for (const char of text) {
|
|
tokens.push(char);
|
|
}
|
|
|
|
for (let index = 0; index < text.length - 1; index += 1) {
|
|
tokens.push(text.slice(index, index + 2));
|
|
}
|
|
}
|
|
|
|
return unique(tokens);
|
|
}
|
|
|
|
export function createCjkSearchTokenizer() {
|
|
return {
|
|
language: "cjk",
|
|
normalizationCache: new Map<string, string>(),
|
|
tokenize(raw: string) {
|
|
if (typeof raw !== "string") return [raw];
|
|
return tokenizeCjkText(raw);
|
|
},
|
|
};
|
|
}
|