From b13fbf71625a03fa13a2f3f7cb45e1f29f450ce0 Mon Sep 17 00:00:00 2001 From: Alexey Romanoff Date: Mon, 7 Nov 2022 10:22:38 +0300 Subject: [PATCH 1/2] test/lexers.lua: add the "Keywords II" test --- test/lexers.lua | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/test/lexers.lua b/test/lexers.lua index 8f7550e..c341ebd 100644 --- a/test/lexers.lua +++ b/test/lexers.lua @@ -235,6 +235,67 @@ check_tokens(lxsh.lexers.lua.gmatch(keywords), { { 'keyword', 'while' }, }) +-- Keywords II. {{{2 +local code_with_keywords = [[if true and not false or nil then + repeat + break + until true +elseif 1 then + local a = 1 +else + for k, v in pairs(_G) do + local f = function() end + end +end + +while true do end + +return 1]] + +check_tokens(lxsh.lexers.lua.gmatch(code_with_keywords), { + { 'keyword', 'if' }, { 'whitespace', ' ' }, + { 'constant', 'true' }, { 'whitespace', ' ' }, + { 'operator', 'and' }, { 'whitespace', ' ' }, + { 'operator', 'not' }, { 'whitespace', ' ' }, + { 'constant', 'false' }, { 'whitespace', ' ' }, + { 'operator', 'or' }, { 'whitespace', ' ' }, + { 'constant', 'nil' }, { 'whitespace', ' ' }, + { 'keyword', 'then' }, { 'whitespace', '\n ' }, + { 'keyword', 'repeat' }, { 'whitespace', '\n ' }, + { 'keyword', 'break' }, { 'whitespace', '\n ' }, + { 'keyword', 'until' }, { 'whitespace', ' ' }, + { 'constant', 'true' }, { 'whitespace', '\n' }, + { 'keyword', 'elseif' }, { 'whitespace', ' ' }, + { 'number', '1' }, { 'whitespace', ' ' }, + { 'keyword', 'then' }, { 'whitespace', '\n ' }, + { 'keyword', 'local' }, { 'whitespace', ' ' }, + { 'identifier', 'a' }, { 'whitespace', ' ' }, + { 'operator', '=' }, { 'whitespace', ' ' }, + { 'number', '1' }, { 'whitespace', '\n' }, + { 'keyword', 'else' }, { 'whitespace', '\n ' }, + { 'keyword', 'for' }, { 'whitespace', ' ' }, + { 'identifier', 'k' }, { 'operator', ',' }, { 'whitespace', ' ' }, + { 'identifier', 'v' }, { 'whitespace', ' ' }, + { 'keyword', 'in' }, { 'whitespace', ' ' }, + { 'identifier', 'pairs' }, { 'operator', '(' }, { 'identifier', '_G' }, + { 'operator', ')' }, { 'whitespace', ' ' }, + { 'keyword', 'do' }, { 'whitespace', '\n ' }, + { 'keyword', 'local' }, { 'whitespace', ' ' }, + { 'identifier', 'f' }, { 'whitespace', ' ' }, + { 'operator', '=' }, { 'whitespace', ' ' }, + { 'keyword', 'function' }, { 'operator', '(' }, { 'operator', ')' }, + { 'whitespace', ' ' }, + { 'keyword', 'end' }, { 'whitespace', '\n ' }, + { 'keyword', 'end' }, { 'whitespace', '\n' }, + { 'keyword', 'end' }, { 'whitespace', '\n\n' }, + { 'keyword', 'while' }, { 'whitespace', ' ' }, + { 'constant', 'true' }, { 'whitespace', ' ' }, + { 'keyword', 'do' }, { 'whitespace', ' ' }, + { 'keyword', 'end' }, { 'whitespace', '\n\n' }, + { 'keyword', 'return' }, { 'whitespace', ' ' }, + { 'number', '1' }, +}) + -- Identifiers. {{{2 check_tokens(lxsh.lexers.lua.gmatch('io.write'), { { 'identifier', 'io' }, From b6c7f25d061a6618f780bb4ef9a3889df2f6fdda Mon Sep 17 00:00:00 2001 From: Alexey Romanoff Date: Mon, 7 Nov 2022 10:26:15 +0300 Subject: [PATCH 2/2] src/lexers/init.lua: compile_keywords: fix the resulting pattern --- src/lexers/init.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lexers/init.lua b/src/lexers/init.lua index bd7c5ee..0a56224 100644 --- a/src/lexers/init.lua +++ b/src/lexers/init.lua @@ -35,9 +35,8 @@ local function compile_keywords(context, keywords) pattern = pattern and (pattern + p) or p end local B = context.word_boundary or B - local BB = lpeg.B(B, 1) + SOS -- starting boundary local AB = B + EOS -- ending boundary - return BB * pattern * AB + return pattern * AB end -- Closure to define token type given name and LPeg pattern.