wehub-resource-sync
1166 lines
35 KiB
C
Generated
1166 lines
35 KiB
C
Generated
#include "tree_sitter/parser.h"
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
#include <wctype.h>
|
|
|
|
// Custom punctuation check for WASM compatibility
|
|
// ispunct() is not in tree-sitter's WASM allowed functions list
|
|
// https://github.com/tree-sitter/tree-sitter/blob/master/lib/src/wasm/stdlib-symbols.txt
|
|
static inline bool is_punct_char(const uint32_t chr)
|
|
{
|
|
if (chr >= 0x80) {
|
|
return false;
|
|
}
|
|
|
|
return (chr >= 33 && chr <= 47) || // !"#$%&'()*+,-./
|
|
(chr >= 58 && chr <= 64) || // :;<=>?@
|
|
(chr >= 91 && chr <= 96) || // [\]^_`
|
|
(chr >= 123 && chr <= 126); // {|}~
|
|
}
|
|
|
|
enum TokenType {
|
|
COMMENT,
|
|
LINE_CONTINUATION,
|
|
COMMAND_NAME,
|
|
COMMAND_ARGUMENT,
|
|
SINGLE_QUOTE_STRING_START,
|
|
SINGLE_QUOTE_STRING_END,
|
|
DOUBLE_QUOTE_STRING_START,
|
|
DOUBLE_QUOTE_STRING_END,
|
|
FORMATTING_SEQUENCE,
|
|
ESCAPE_SEQUENCE,
|
|
STRING_CONTENT,
|
|
ENTRY_DELIMITER,
|
|
MULTIOUTPUT_VARIABLE_START,
|
|
IDENTIFIER,
|
|
CATCH_IDENTIFIER,
|
|
TRANSPOSE,
|
|
CTRANSPOSE,
|
|
ERROR_SENTINEL,
|
|
};
|
|
|
|
typedef struct
|
|
{
|
|
bool is_inside_command;
|
|
bool line_continuation;
|
|
bool is_shell_scape;
|
|
char string_delimiter;
|
|
bool generate_entry_delimiter;
|
|
} Scanner;
|
|
|
|
static const char* const keywords[] = {
|
|
"arguments", "break", "case", "catch", "classdef", "continue", "else", "elseif",
|
|
"end", "enumeration", "events", "for", "function", "global", "if", "methods",
|
|
"otherwise", "parfor", "persistent", "return", "spmd", "switch", "try", "while",
|
|
};
|
|
static const size_t keywords_size = sizeof(keywords) / sizeof(keywords[0]);
|
|
|
|
static inline void advance(TSLexer* lexer)
|
|
{
|
|
lexer->advance(lexer, false);
|
|
}
|
|
|
|
static inline void skip(TSLexer* lexer)
|
|
{
|
|
lexer->advance(lexer, true);
|
|
}
|
|
|
|
static inline bool consume_char(char chr, TSLexer* lexer)
|
|
{
|
|
if (lexer->lookahead != chr) {
|
|
return false;
|
|
}
|
|
advance(lexer);
|
|
return true;
|
|
}
|
|
|
|
static inline bool is_eol(const uint32_t chr)
|
|
{
|
|
return chr == '\n' || chr == '\r' || chr == ',' || chr == ';';
|
|
}
|
|
|
|
static inline bool iswspace_matlab(const uint32_t chr)
|
|
{
|
|
return iswspace(chr) && chr != '\n' && chr != '\r';
|
|
}
|
|
|
|
static inline bool is_identifier(const uint32_t chr, const bool start)
|
|
{
|
|
// isalpha or isdigit is SIGSEGVing os some UTF-8 chars, like U+10C6BD
|
|
// (0xF48C9ABD), a file with just those bytes shows the problem.
|
|
if (chr >= 0x80) {
|
|
return false;
|
|
}
|
|
|
|
const bool alpha = iswalpha(chr);
|
|
const bool numeric = !start && iswdigit(chr);
|
|
const bool special = chr == '_';
|
|
|
|
return alpha || numeric || special;
|
|
}
|
|
|
|
static inline void consume_identifier(TSLexer* lexer, char* buffer)
|
|
{
|
|
size_t size = 0;
|
|
if (is_identifier(lexer->lookahead, true)) {
|
|
buffer[size] = (char) lexer->lookahead;
|
|
advance(lexer);
|
|
while (is_identifier(lexer->lookahead, false)) {
|
|
if (size == 255) {
|
|
buffer[0] = 0;
|
|
return;
|
|
}
|
|
buffer[++size] = (char) lexer->lookahead;
|
|
advance(lexer);
|
|
}
|
|
return;
|
|
}
|
|
buffer[0] = 0;
|
|
}
|
|
|
|
static inline int skip_whitespaces(TSLexer* lexer)
|
|
{
|
|
// 0b001 -> something skipped
|
|
// 0b010 -> newline skipped
|
|
// 0b100 -> newline was at the end of skipped sequence
|
|
int skipped = 0;
|
|
while (!lexer->eof(lexer) && iswspace(lexer->lookahead)) {
|
|
skipped &= 0b011;
|
|
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
skipped |= 0b111;
|
|
} else {
|
|
skipped |= 0b001;
|
|
}
|
|
skip(lexer);
|
|
}
|
|
return skipped;
|
|
}
|
|
|
|
static inline int consume_whitespaces(TSLexer* lexer)
|
|
{
|
|
int skipped = 0;
|
|
while (iswspace(lexer->lookahead)) {
|
|
skipped &= 0b011;
|
|
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
skipped |= 0b111;
|
|
} else {
|
|
skipped |= 0b001;
|
|
}
|
|
advance(lexer);
|
|
}
|
|
return skipped;
|
|
}
|
|
|
|
static inline void consume_whitespaces_once(TSLexer* lexer)
|
|
{
|
|
while (!lexer->eof(lexer) && iswspace(lexer->lookahead)) {
|
|
if (lexer->lookahead == '\n' || lexer->lookahead == '\r') {
|
|
advance(lexer);
|
|
break;
|
|
}
|
|
advance(lexer);
|
|
}
|
|
}
|
|
|
|
void* tree_sitter_matlab_external_scanner_create()
|
|
{
|
|
Scanner* scanner = calloc(1, sizeof(Scanner));
|
|
return scanner;
|
|
}
|
|
|
|
void tree_sitter_matlab_external_scanner_destroy(void* payload)
|
|
{
|
|
if (payload != NULL) {
|
|
free(payload);
|
|
}
|
|
}
|
|
|
|
unsigned tree_sitter_matlab_external_scanner_serialize(void* payload, char* buffer)
|
|
{
|
|
Scanner* scanner = (Scanner*) payload;
|
|
buffer[0] = (char) scanner->is_inside_command;
|
|
buffer[1] = (char) scanner->line_continuation;
|
|
buffer[2] = (char) scanner->is_shell_scape;
|
|
buffer[3] = scanner->string_delimiter;
|
|
buffer[4] = (char) scanner->generate_entry_delimiter;
|
|
return 5;
|
|
}
|
|
|
|
void tree_sitter_matlab_external_scanner_deserialize(
|
|
void* payload,
|
|
const char* buffer,
|
|
unsigned length)
|
|
{
|
|
Scanner* scanner = (Scanner*) payload;
|
|
if (length == 5) {
|
|
scanner->is_inside_command = buffer[0];
|
|
scanner->line_continuation = buffer[1];
|
|
scanner->is_shell_scape = buffer[2];
|
|
scanner->string_delimiter = buffer[3];
|
|
scanner->generate_entry_delimiter = buffer[4];
|
|
}
|
|
}
|
|
|
|
static inline void consume_comment_line(TSLexer* lexer)
|
|
{
|
|
while (lexer->lookahead != '\n' && lexer->lookahead != '\r' && !lexer->eof(lexer)) {
|
|
advance(lexer);
|
|
}
|
|
}
|
|
|
|
// NOLINTNEXTLINE(*misc-no-recursion)
|
|
static bool scan_comment(
|
|
Scanner* scanner,
|
|
TSLexer* lexer,
|
|
bool entry_delimiter,
|
|
bool ctranspose,
|
|
int skipped)
|
|
{
|
|
lexer->mark_end(lexer);
|
|
|
|
const bool percent = lexer->lookahead == '%';
|
|
const bool line_continuation = lexer->lookahead == '.' && consume_char('.', lexer)
|
|
&& consume_char('.', lexer) && consume_char('.', lexer);
|
|
const bool block = percent && consume_char('%', lexer) && consume_char('{', lexer);
|
|
|
|
// Since we cannot look multiple chars ahead in the main function, this
|
|
// ended up being handled here. It allows the correct detection of numbers
|
|
// like .5 inside matrices/cells: [0 .5].
|
|
if (entry_delimiter && !percent && !line_continuation) {
|
|
if (iswdigit(lexer->lookahead)) {
|
|
lexer->result_symbol = ENTRY_DELIMITER;
|
|
return true;
|
|
}
|
|
if (lexer->lookahead == '\'') {
|
|
advance(lexer);
|
|
lexer->result_symbol = CTRANSPOSE;
|
|
lexer->mark_end(lexer);
|
|
return skipped == 0;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
// We are inside a matrix/cell row and there is a line continuation, like this:
|
|
// a = { 1 ...
|
|
// 2 ...
|
|
// }
|
|
if (entry_delimiter && line_continuation) {
|
|
consume_comment_line(lexer);
|
|
consume_whitespaces(lexer);
|
|
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = LINE_CONTINUATION;
|
|
|
|
const bool is_alpha = iswalpha(lexer->lookahead);
|
|
const bool is_digit = iswdigit(lexer->lookahead);
|
|
const bool is_meta = lexer->lookahead == '?' || lexer->lookahead == '@';
|
|
const bool is_quote = lexer->lookahead == '\'' || lexer->lookahead == '"';
|
|
const bool is_container = lexer->lookahead == '{' || lexer->lookahead == '['
|
|
|| lexer->lookahead == '(';
|
|
|
|
if (lexer->lookahead == '~') {
|
|
advance(lexer);
|
|
scanner->generate_entry_delimiter = lexer->lookahead != '=';
|
|
} else if (lexer->lookahead == '+' || lexer->lookahead == '-') {
|
|
advance(lexer);
|
|
scanner->generate_entry_delimiter = lexer->lookahead != ' ';
|
|
} else if (lexer->lookahead == '.') {
|
|
advance(lexer);
|
|
scanner->generate_entry_delimiter = is_digit;
|
|
} else if (is_alpha || is_digit || is_quote || is_container || is_meta) {
|
|
scanner->generate_entry_delimiter = true;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
if (block) {
|
|
if (skipped & 2) {
|
|
return false;
|
|
}
|
|
|
|
// If it has things on the same line, it's not a block, just a comment
|
|
while (!lexer->eof(lexer) && iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
if (!consume_char('\n', lexer) && !consume_char('\r', lexer)) {
|
|
consume_comment_line(lexer);
|
|
lexer->result_symbol = COMMENT;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
// Otherwise, find the matching closing block
|
|
int level = 1;
|
|
while (!lexer->eof(lexer)) {
|
|
consume_whitespaces(lexer);
|
|
if (consume_char('%', lexer)) {
|
|
if (consume_char('{', lexer) && (consume_whitespaces(lexer) & 2)) {
|
|
level++;
|
|
} else if (consume_char('}', lexer)) {
|
|
lexer->mark_end(lexer);
|
|
if (consume_whitespaces(lexer) & 2) {
|
|
level--;
|
|
}
|
|
}
|
|
if (level == 0) {
|
|
break;
|
|
}
|
|
continue;
|
|
}
|
|
consume_comment_line(lexer);
|
|
lexer->mark_end(lexer);
|
|
}
|
|
|
|
lexer->result_symbol = COMMENT;
|
|
return true;
|
|
}
|
|
|
|
if (percent || line_continuation) {
|
|
consume_comment_line(lexer);
|
|
lexer->mark_end(lexer);
|
|
|
|
if (!line_continuation) {
|
|
lexer->result_symbol = COMMENT;
|
|
advance(lexer);
|
|
} else {
|
|
lexer->result_symbol = LINE_CONTINUATION;
|
|
consume_whitespaces_once(lexer);
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
// Merges consecutive comments into one token, unless they are
|
|
// separated by a newline.
|
|
while (!lexer->eof(lexer) && (lexer->lookahead == ' ' || lexer->lookahead == '\t')) {
|
|
advance(lexer);
|
|
}
|
|
|
|
if (lexer->lookahead == '%') {
|
|
return scan_comment(scanner, lexer, false, false, 0);
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
if (ctranspose && lexer->lookahead == '\'') {
|
|
advance(lexer);
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = CTRANSPOSE;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool scan_command(Scanner* scanner, TSLexer* lexer, const bool* valid_symbols)
|
|
{
|
|
// Special case: shell escape
|
|
if (lexer->lookahead == '!') {
|
|
advance(lexer);
|
|
while (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
while (lexer->lookahead != ' ' && lexer->lookahead != '\n' && !lexer->eof(lexer)) {
|
|
advance(lexer);
|
|
}
|
|
lexer->result_symbol = COMMAND_NAME;
|
|
lexer->mark_end(lexer);
|
|
while (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
scanner->is_inside_command = lexer->lookahead != '\n';
|
|
scanner->is_shell_scape = scanner->is_inside_command;
|
|
return true;
|
|
}
|
|
|
|
if (!is_identifier(lexer->lookahead, true)) {
|
|
return false;
|
|
}
|
|
|
|
char buffer[256] = {0};
|
|
consume_identifier(lexer, buffer);
|
|
lexer->mark_end(lexer);
|
|
const char* allowed_commands[] = {"methods", "arguments", "enumeration", "events"};
|
|
if (buffer[0] != 0) {
|
|
if (lexer->lookahead == '.') {
|
|
// Since it is not followed by a space, it cannot be a command.
|
|
if ((strcmp("get", buffer) == 0 || strcmp("set", buffer) == 0)) {
|
|
return false;
|
|
}
|
|
// so it is ok to consume to identify a line continuation
|
|
// NOLINTNEXTLINE(*misc-redundant-expression)
|
|
if (consume_char('.', lexer) && consume_char('.', lexer) && consume_char('.', lexer)) {
|
|
// If it is a keyword, yield to the internal scanner
|
|
for (size_t i = 0; i < keywords_size; i++) {
|
|
if (strcmp(keywords[i], buffer) == 0) {
|
|
if (strcmp("enumeration", buffer) == 0) {
|
|
goto check_enumeration;
|
|
}
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
// The following keywords are allowed as commands if they get 1 argument
|
|
for (unsigned i = 0; i < sizeof(allowed_commands) / sizeof(allowed_commands[0]); i++) {
|
|
if (strcmp(allowed_commands[i], buffer) == 0) {
|
|
goto check_command_for_argument;
|
|
}
|
|
}
|
|
for (unsigned i = 0; i < keywords_size; i++) {
|
|
if (strcmp(keywords[i], buffer) == 0) {
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
goto skip_command_check;
|
|
|
|
check_enumeration: {
|
|
const int skipped = consume_whitespaces(lexer);
|
|
if (skipped & 2) {
|
|
// enumeration can be a function
|
|
if (lexer->lookahead == '(') {
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
check_command_for_argument:
|
|
// If this is a keyword-command, check if it has an argument.
|
|
// If it has no arguments, this is a keyword, not a command.
|
|
lexer->result_symbol = COMMAND_NAME;
|
|
while (!lexer->eof(lexer) && iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
if (is_identifier(lexer->lookahead, true)) {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
return false;
|
|
|
|
skip_command_check:
|
|
// First case: found an end-of-line already, so this is a command for sure.
|
|
// example:
|
|
// pwd
|
|
// pwd;
|
|
// pwd,
|
|
if (is_eol(lexer->lookahead)) {
|
|
lexer->result_symbol = valid_symbols[CATCH_IDENTIFIER] ? CATCH_IDENTIFIER : COMMAND_NAME;
|
|
return true;
|
|
}
|
|
|
|
// If it's not followed by a space, it may be something else, like A' for
|
|
// example. Or A+2.
|
|
if (lexer->lookahead != ' ') {
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
|
|
// If followed by a line continuation, look after it
|
|
const int skipped = consume_whitespaces(lexer);
|
|
if (skipped & 2) {
|
|
// `catch e `
|
|
if (valid_symbols[CATCH_IDENTIFIER]) {
|
|
lexer->result_symbol = CATCH_IDENTIFIER;
|
|
return true;
|
|
}
|
|
// Command followed by spaces then newline
|
|
scanner->is_inside_command = false;
|
|
lexer->result_symbol = COMMAND_NAME;
|
|
return true;
|
|
}
|
|
if (lexer->lookahead == '.' && consume_char('.', lexer) && consume_char('.', lexer)
|
|
&& consume_char('.', lexer)) {
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
|
|
// If it is followed by a space, it doesn't mean it's a command yet.
|
|
// It could be A + 2 or A = 2. Let's check what is the first char after
|
|
// all whitespaces. We mark it already as this is the right place, and we
|
|
// only need to make sure this is a command and not something else from
|
|
// this point on.
|
|
lexer->result_symbol = COMMAND_NAME;
|
|
while (!lexer->eof(lexer) && iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
|
|
// Check for end-of-line again, since it may be that the user just put a
|
|
// space at the end, like `pwd ;`
|
|
if (is_eol(lexer->lookahead) || lexer->lookahead == '%') {
|
|
if (valid_symbols[CATCH_IDENTIFIER] && (skipped & 4) == 0) {
|
|
lexer->result_symbol = CATCH_IDENTIFIER;
|
|
return true;
|
|
}
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
// The first char of the first argument cannot be /=()/
|
|
if (lexer->lookahead == '=' || lexer->lookahead == '(' || lexer->lookahead == ')') {
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
|
|
// If it is a single quote, it is a command.
|
|
if (lexer->lookahead == '\'') {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
// If it is an identifier char, then it's a command
|
|
if (is_identifier(lexer->lookahead, false)) {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
// If it is a char greater than 0xC0, then assume it's a valid UTF-8
|
|
// char, and that this is a command.
|
|
if (lexer->lookahead >= 0xC0) {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
// Let's now consider punctuation marks.
|
|
if (is_punct_char(lexer->lookahead)) {
|
|
// In this case, we advance and look at what comes next too.
|
|
const uint32_t first = lexer->lookahead;
|
|
advance(lexer);
|
|
const uint32_t second = lexer->lookahead;
|
|
|
|
// If it's the end-of-line, then it's a command.
|
|
if (is_eol(second)) {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
if (iswspace_matlab(second)) {
|
|
// If it is a space, then it depends on what we have, since
|
|
// `disp + ;` is a valid command but `disp + 2;` isn't.
|
|
const char operators[] = {
|
|
'!',
|
|
'&',
|
|
'*',
|
|
'+',
|
|
'-',
|
|
'/',
|
|
'<',
|
|
'>',
|
|
'@',
|
|
'\\',
|
|
'^',
|
|
'|',
|
|
};
|
|
bool is_invalid = false;
|
|
for (size_t i = 0; i < sizeof(operators); i++) {
|
|
if (first == (uint32_t) operators[i]) {
|
|
is_invalid = true;
|
|
break;
|
|
}
|
|
}
|
|
// If it is an operator, this can only be a command if there
|
|
// are no further arguments.
|
|
if (is_invalid) {
|
|
advance(lexer);
|
|
while (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
scanner->is_inside_command = is_eol(lexer->lookahead);
|
|
lexer->result_symbol = scanner->is_inside_command ? COMMAND_NAME : IDENTIFIER;
|
|
return true;
|
|
}
|
|
|
|
// If it's not an operator, then this is a command.
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
// Now we check for the rest of the operators.
|
|
// Since they have 2 digits, it matters if the next is a space.
|
|
advance(lexer);
|
|
|
|
if (lexer->lookahead != ' ') {
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
const char operators[][2] = {
|
|
{'&', '&'},
|
|
{'|', '|'},
|
|
{'=', '='},
|
|
{'~', '='},
|
|
{'<', '='},
|
|
{'>', '='},
|
|
{'.', '+'},
|
|
{'.', '-'},
|
|
{'.', '*'},
|
|
{'.', '/'},
|
|
{'.', '\\'},
|
|
{'.', '^'},
|
|
};
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
if ((uint32_t) operators[i][0] == first && (uint32_t) operators[i][1] == second) {
|
|
lexer->result_symbol = IDENTIFIER;
|
|
return true;
|
|
}
|
|
}
|
|
|
|
scanner->is_inside_command = true;
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool scan_command_argument(Scanner* scanner, TSLexer* lexer)
|
|
{
|
|
// If this is a shell escape command, we just break arguments in spaces
|
|
// since we don't know what shell it is.
|
|
if (scanner->is_shell_scape) {
|
|
if (lexer->eof(lexer)) {
|
|
return false;
|
|
}
|
|
|
|
while (lexer->lookahead != ' ' && lexer->lookahead != '\n' && !lexer->eof(lexer)) {
|
|
advance(lexer);
|
|
}
|
|
lexer->result_symbol = COMMAND_ARGUMENT;
|
|
lexer->mark_end(lexer);
|
|
while (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
if (lexer->lookahead == '\n') {
|
|
scanner->is_inside_command = false;
|
|
scanner->is_shell_scape = false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Avoids infinite loop when the argument is right before the eof.
|
|
if (lexer->eof(lexer)) {
|
|
return false;
|
|
}
|
|
|
|
bool quote = false;
|
|
int32_t parens = 0;
|
|
bool consumed = false;
|
|
|
|
while (!lexer->eof(lexer)) {
|
|
// No matter what, found new line
|
|
const bool cond1 = lexer->lookahead == '\n' || lexer->lookahead == '\r';
|
|
// No quotes, no parens, found $._end_of_line or space
|
|
const bool cond2 = !quote && parens == 0
|
|
&& (is_eol(lexer->lookahead) || iswspace_matlab(lexer->lookahead));
|
|
// Inside parens, no quotes, found ;
|
|
const bool cond3 = !quote && parens != 0 && lexer->lookahead == ';';
|
|
if (cond1 || cond2 || cond3) {
|
|
lexer->result_symbol = COMMAND_ARGUMENT;
|
|
lexer->mark_end(lexer);
|
|
|
|
while (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
}
|
|
|
|
if (is_eol(lexer->lookahead) || cond1) {
|
|
scanner->line_continuation = false;
|
|
scanner->is_inside_command = false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
// Line comment, finish.
|
|
if ((!quote || (quote && parens != 0)) && lexer->lookahead == '%') {
|
|
scanner->is_inside_command = false;
|
|
if (consumed) {
|
|
lexer->result_symbol = COMMAND_ARGUMENT;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
return scan_comment(scanner, lexer, false, false, 0);
|
|
}
|
|
|
|
// Line continuation
|
|
if ((!quote || (quote && parens != 0)) && lexer->lookahead == '.') {
|
|
lexer->result_symbol = COMMAND_ARGUMENT;
|
|
lexer->mark_end(lexer);
|
|
advance(lexer);
|
|
if (lexer->lookahead == '.') {
|
|
advance(lexer);
|
|
if (lexer->lookahead == '.') {
|
|
if (consumed) {
|
|
scanner->line_continuation = true;
|
|
} else {
|
|
consume_comment_line(lexer);
|
|
lexer->result_symbol = LINE_CONTINUATION;
|
|
lexer->mark_end(lexer);
|
|
}
|
|
return true;
|
|
}
|
|
consumed = true;
|
|
continue;
|
|
}
|
|
consumed = true;
|
|
continue;
|
|
}
|
|
|
|
if ((lexer->lookahead == '(' || lexer->lookahead == '[' || lexer->lookahead == '{')
|
|
&& (!quote || (quote && parens != 0))) {
|
|
parens++;
|
|
}
|
|
|
|
if ((lexer->lookahead == ')' || lexer->lookahead == ']' || lexer->lookahead == '}')
|
|
&& (!quote || (quote && parens != 0))) {
|
|
parens--;
|
|
}
|
|
|
|
if (lexer->lookahead == '\'') {
|
|
quote = !quote;
|
|
}
|
|
|
|
advance(lexer);
|
|
consumed = true;
|
|
}
|
|
|
|
// Mark as argument so the scanner doesnt get called again in an infinite
|
|
// loop.
|
|
if (lexer->eof(lexer)) {
|
|
lexer->result_symbol = COMMAND_ARGUMENT;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool scan_string_open(Scanner* scanner, TSLexer* lexer)
|
|
{
|
|
switch (lexer->lookahead) {
|
|
case '"':
|
|
scanner->string_delimiter = '"';
|
|
advance(lexer);
|
|
lexer->result_symbol = DOUBLE_QUOTE_STRING_START;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
case '\'':
|
|
scanner->string_delimiter = '\'';
|
|
advance(lexer);
|
|
lexer->result_symbol = SINGLE_QUOTE_STRING_START;
|
|
lexer->mark_end(lexer);
|
|
// A single quote string has to be ended in the same line.
|
|
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
|
if (lexer->lookahead == '\'') {
|
|
return true;
|
|
}
|
|
advance(lexer);
|
|
}
|
|
return false;
|
|
default:
|
|
return false;
|
|
}
|
|
}
|
|
|
|
static bool scan_string_close(Scanner* scanner, TSLexer* lexer)
|
|
{
|
|
if (lexer->lookahead == scanner->string_delimiter) {
|
|
advance(lexer);
|
|
if (lexer->lookahead == scanner->string_delimiter) {
|
|
advance(lexer);
|
|
lexer->result_symbol = STRING_CONTENT;
|
|
goto content;
|
|
}
|
|
lexer->result_symbol = scanner->string_delimiter == '"' ? DOUBLE_QUOTE_STRING_END
|
|
: SINGLE_QUOTE_STRING_END;
|
|
lexer->mark_end(lexer);
|
|
scanner->string_delimiter = 0;
|
|
return true;
|
|
}
|
|
|
|
// This means this string is not properly terminated.
|
|
if (lexer->lookahead == '\n' || lexer->lookahead == '\r' || lexer->eof(lexer)) {
|
|
scanner->string_delimiter = 0;
|
|
return false;
|
|
}
|
|
|
|
if (lexer->lookahead == '%') {
|
|
advance(lexer);
|
|
|
|
if (lexer->lookahead == '%') {
|
|
advance(lexer);
|
|
lexer->result_symbol = FORMATTING_SEQUENCE;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
const char* valid_tokens = "1234567890.-+ #btcdeEfgGosuxX";
|
|
const char* end_tokens = "cdeEfgGosuxX";
|
|
while (!lexer->eof(lexer) && lexer->lookahead != '\n' && lexer->lookahead != '\r') {
|
|
bool is_valid = false;
|
|
for (size_t i = 0; i < strlen(valid_tokens); i++) {
|
|
if ((int32_t) valid_tokens[i] == lexer->lookahead) {
|
|
is_valid = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!is_valid) {
|
|
lexer->result_symbol = STRING_CONTENT;
|
|
goto content;
|
|
}
|
|
|
|
for (int i = 0; i < 12; i++) {
|
|
if (end_tokens[i] == lexer->lookahead) {
|
|
advance(lexer);
|
|
lexer->result_symbol = FORMATTING_SEQUENCE;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
advance(lexer);
|
|
}
|
|
|
|
scanner->string_delimiter = 0;
|
|
return false;
|
|
}
|
|
|
|
if (lexer->lookahead == '\\') {
|
|
advance(lexer);
|
|
|
|
if (lexer->lookahead == 'x') {
|
|
advance(lexer);
|
|
while (!lexer->eof(lexer)) {
|
|
const char* hexa_chars = "1234567890abcdefABCDEF";
|
|
bool is_valid = false;
|
|
for (int i = 0; i < 22; i++) {
|
|
if (hexa_chars[i] == lexer->lookahead) {
|
|
is_valid = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!is_valid) {
|
|
lexer->result_symbol = ESCAPE_SEQUENCE;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
advance(lexer);
|
|
}
|
|
}
|
|
|
|
if (lexer->lookahead >= '0' && lexer->lookahead <= '7') {
|
|
while (lexer->lookahead >= '0' && lexer->lookahead <= '7' && !lexer->eof(lexer)) {
|
|
advance(lexer);
|
|
}
|
|
|
|
lexer->result_symbol = ESCAPE_SEQUENCE;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
const char* escapes = "abfnrtv\\";
|
|
bool is_valid = false;
|
|
for (int i = 0; i < 8; i++) {
|
|
if (escapes[i] == lexer->lookahead) {
|
|
is_valid = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (is_valid) {
|
|
advance(lexer);
|
|
lexer->result_symbol = ESCAPE_SEQUENCE;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
content:
|
|
while (lexer->lookahead != '\n' && lexer->lookahead != '\r' && !lexer->eof(lexer)) {
|
|
// In MATLAB '' and "" are valid inside their own kind: 'It''s ok' "He said ""it's ok"""
|
|
if (lexer->lookahead == scanner->string_delimiter) {
|
|
lexer->result_symbol = STRING_CONTENT;
|
|
lexer->mark_end(lexer);
|
|
advance(lexer);
|
|
if (lexer->lookahead != scanner->string_delimiter) {
|
|
return true;
|
|
}
|
|
advance(lexer);
|
|
continue;
|
|
}
|
|
|
|
// The scanner will be called again, and this time we will match in the if
|
|
// before this while.
|
|
if (lexer->lookahead == '%' || lexer->lookahead == '\\') {
|
|
lexer->result_symbol = STRING_CONTENT;
|
|
lexer->mark_end(lexer);
|
|
advance(lexer);
|
|
if (lexer->lookahead == scanner->string_delimiter || iswspace_matlab(lexer->lookahead)) {
|
|
goto content;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
advance(lexer);
|
|
}
|
|
|
|
// Mark end of content here and end of string on next call. This is an
|
|
// unterminated string and it's better to wrongly finish it here, otherwise
|
|
// the error will appear god knows how many lines after this and it will be
|
|
// hard for the user to understand what went wrong.
|
|
if (lexer->lookahead == '\n' || lexer->lookahead == '\r' || lexer->eof(lexer)) {
|
|
lexer->result_symbol = STRING_CONTENT;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
|
|
scanner->string_delimiter = 0;
|
|
return false;
|
|
}
|
|
|
|
static inline bool scan_multioutput_var_start(TSLexer* lexer)
|
|
{
|
|
advance(lexer);
|
|
lexer->result_symbol = MULTIOUTPUT_VARIABLE_START;
|
|
lexer->mark_end(lexer);
|
|
|
|
// We can have arrays inside function calls inside the multi-output variable, so we have to keep
|
|
// track.
|
|
unsigned sb_count = 0;
|
|
|
|
while (!lexer->eof(lexer)) {
|
|
// NOLINTNEXTLINE(*misc-redundant-expression)
|
|
if (consume_char('.', lexer) && consume_char('.', lexer) && consume_char('.', lexer)) {
|
|
consume_comment_line(lexer);
|
|
advance(lexer);
|
|
}
|
|
|
|
if (lexer->lookahead == '[') {
|
|
sb_count++;
|
|
advance(lexer);
|
|
}
|
|
|
|
if (lexer->lookahead != ']') {
|
|
advance(lexer);
|
|
} else if (sb_count > 0) {
|
|
sb_count--;
|
|
advance(lexer);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (lexer->lookahead != ']') {
|
|
return false;
|
|
}
|
|
|
|
advance(lexer);
|
|
|
|
while (!lexer->eof(lexer)) {
|
|
// NOLINTNEXTLINE(*misc-redundant-expression)
|
|
if (consume_char('.', lexer) && consume_char('.', lexer) && consume_char('.', lexer)) {
|
|
consume_comment_line(lexer);
|
|
advance(lexer);
|
|
} else if (iswspace_matlab(lexer->lookahead)) {
|
|
advance(lexer);
|
|
} else {
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (lexer->lookahead == '=') {
|
|
advance(lexer);
|
|
if (lexer->lookahead != '=') {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool scan_identifier(TSLexer* lexer);
|
|
static bool scan_entry_delimiter(TSLexer* lexer, int skipped)
|
|
{
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = ENTRY_DELIMITER;
|
|
|
|
if (skipped & 2) {
|
|
return false;
|
|
}
|
|
|
|
if (lexer->lookahead == ',') {
|
|
advance(lexer);
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = ENTRY_DELIMITER;
|
|
return true;
|
|
}
|
|
|
|
if (lexer->lookahead == '.') {
|
|
advance(lexer);
|
|
advance(lexer);
|
|
return iswdigit(lexer->lookahead);
|
|
}
|
|
|
|
if (lexer->lookahead == '{' || lexer->lookahead == '(' || lexer->lookahead == '\'') {
|
|
return skipped != 0;
|
|
}
|
|
|
|
if (lexer->lookahead == '[') {
|
|
return true;
|
|
}
|
|
|
|
// These chars mean we cannot end the cell here, as the expression will
|
|
// surely continue OR we need to just leave the char there and the internal
|
|
// parser will do the rest.
|
|
const char no_end[] = {']', '}', '&', '|', '=', '<', '>', '*', '/', '\\', '^', ';', ':'};
|
|
for (size_t i = 0; i < sizeof(no_end); i++) {
|
|
if ((int32_t) no_end[i] == lexer->lookahead) {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if (lexer->lookahead == '~') {
|
|
advance(lexer);
|
|
return lexer->lookahead != '=';
|
|
}
|
|
|
|
const char maybe_end[] = {'+', '-'};
|
|
for (size_t i = 0; i < sizeof(maybe_end); i++) {
|
|
if ((int32_t) maybe_end[i] == lexer->lookahead) {
|
|
advance(lexer);
|
|
if (lexer->lookahead == ' ') {
|
|
return false;
|
|
}
|
|
return skipped != 0;
|
|
}
|
|
}
|
|
|
|
if (skipped != 0) {
|
|
return true;
|
|
}
|
|
|
|
if (is_identifier(lexer->lookahead, true)) {
|
|
return scan_identifier(lexer);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
static bool scan_identifier(TSLexer* lexer)
|
|
{
|
|
char buffer[256] = {0};
|
|
consume_identifier(lexer, buffer);
|
|
if (buffer[0] != 0) {
|
|
if (lexer->lookahead == '.') {
|
|
if ((strcmp("get", buffer) == 0 || strcmp("set", buffer) == 0)) {
|
|
return false;
|
|
}
|
|
lexer->result_symbol = IDENTIFIER;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
for (size_t i = 0; i < keywords_size; i++) {
|
|
if (strcmp(keywords[i], buffer) == 0) {
|
|
return false;
|
|
}
|
|
}
|
|
lexer->result_symbol = IDENTIFIER;
|
|
lexer->mark_end(lexer);
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static bool scan_transpose(TSLexer* lexer)
|
|
{
|
|
if (lexer->lookahead == '\'') {
|
|
advance(lexer);
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = TRANSPOSE;
|
|
return true;
|
|
}
|
|
if (lexer->lookahead == '.' && consume_char('\'', lexer)) {
|
|
advance(lexer);
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = CTRANSPOSE;
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
bool tree_sitter_matlab_external_scanner_scan(void* payload, TSLexer* lexer, const bool* valid_symbols)
|
|
{
|
|
Scanner* scanner = (Scanner*) payload;
|
|
|
|
if (scanner->generate_entry_delimiter) {
|
|
scanner->generate_entry_delimiter = false;
|
|
lexer->mark_end(lexer);
|
|
lexer->result_symbol = ENTRY_DELIMITER;
|
|
return true;
|
|
}
|
|
|
|
if (scanner->string_delimiter == 0) {
|
|
int skipped = skip_whitespaces(lexer);
|
|
|
|
if ((scanner->line_continuation || !scanner->is_inside_command) && valid_symbols[COMMENT]
|
|
&& (lexer->lookahead == '%' || ((skipped & 2) == 0 && lexer->lookahead == '.'))) {
|
|
return scan_comment(
|
|
scanner, lexer, valid_symbols[ENTRY_DELIMITER], valid_symbols[CTRANSPOSE], skipped);
|
|
}
|
|
|
|
if (!scanner->is_inside_command) {
|
|
if (skipped == 0 && valid_symbols[TRANSPOSE]) {
|
|
if (scan_transpose(lexer)) {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
if ((valid_symbols[SINGLE_QUOTE_STRING_START] && lexer->lookahead == '\'')
|
|
|| (valid_symbols[DOUBLE_QUOTE_STRING_START] && lexer->lookahead == '"')) {
|
|
return scan_string_open(scanner, lexer);
|
|
}
|
|
|
|
if (!scanner->line_continuation) {
|
|
if (valid_symbols[MULTIOUTPUT_VARIABLE_START] && lexer->lookahead == '[') {
|
|
return scan_multioutput_var_start(lexer);
|
|
}
|
|
|
|
if (valid_symbols[ENTRY_DELIMITER]) {
|
|
return scan_entry_delimiter(lexer, skipped);
|
|
}
|
|
}
|
|
|
|
if (valid_symbols[COMMAND_NAME]) {
|
|
scanner->is_inside_command = false;
|
|
scanner->is_shell_scape = false;
|
|
return scan_command(scanner, lexer, valid_symbols);
|
|
}
|
|
|
|
if (valid_symbols[IDENTIFIER] && (skipped & 2) == 0) {
|
|
scanner->is_inside_command = false;
|
|
scanner->is_shell_scape = false;
|
|
return scan_identifier(lexer);
|
|
}
|
|
} else {
|
|
if (valid_symbols[COMMAND_ARGUMENT]) {
|
|
return scan_command_argument(scanner, lexer);
|
|
}
|
|
}
|
|
} else {
|
|
if (valid_symbols[DOUBLE_QUOTE_STRING_END] || valid_symbols[SINGLE_QUOTE_STRING_END]
|
|
|| valid_symbols[FORMATTING_SEQUENCE]) {
|
|
return scan_string_close(scanner, lexer);
|
|
}
|
|
}
|
|
|
|
return false;
|
|
} |