Skip to content

Commit f6bb943

Browse files
tlivelytlively
authored
[Parser][NFC] Remove Token from lexer interface (#6333)
Replace the general `peek` method that returned a `Token` with specific peek methods that look for (but do not consume) specific kinds of tokens. This change is a prerequisite for simplifying the lexer implementation by removing `Token` entirely.
1 parent f9a49fa commit f6bb943

File tree

2 files changed

+46
-44
lines changed

2 files changed

+46
-44
lines changed

‎src/parser/lexer.h‎

Lines changed: 38 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -168,42 +168,41 @@ struct Lexer {
168168
advance();
169169
}
170170

171-
std::optional<Token> peek() const { return curr; }
172-
173171
bool takeLParen() {
174-
auto t = peek();
175-
if (!t || !t->isLParen()) {
172+
if (!curr || !curr->isLParen()) {
176173
return false;
177174
}
178175
advance();
179176
return true;
180177
}
181178

179+
bool peekLParen() { return Lexer(*this).takeLParen(); }
180+
182181
bool takeRParen() {
183-
auto t = peek();
184-
if (!t || !t->isRParen()) {
182+
if (!curr || !curr->isRParen()) {
185183
return false;
186184
}
187185
advance();
188186
return true;
189187
}
190188

189+
bool peekRParen() { return Lexer(*this).takeRParen(); }
190+
191191
bool takeUntilParen() {
192192
while (true) {
193-
auto t = peek();
194-
if (!t) {
193+
if (!curr) {
195194
return false;
196195
}
197-
if (t->isLParen() || t->isRParen()) {
196+
if (curr->isLParen() || curr->isRParen()) {
198197
return true;
199198
}
200199
advance();
201200
}
202201
}
203202

204203
std::optional<Name> takeID() {
205-
if (auto t = peek()) {
206-
if (auto id = t->getID()) {
204+
if (curr) {
205+
if (auto id = curr->getID()) {
207206
advance();
208207
// See comment on takeName.
209208
return Name(std::string(*id));
@@ -213,18 +212,22 @@ struct Lexer {
213212
}
214213

215214
std::optional<std::string_view> takeKeyword() {
216-
if (auto t = peek()) {
217-
if (auto keyword = t->getKeyword()) {
215+
if (curr) {
216+
if (auto keyword = curr->getKeyword()) {
218217
advance();
219218
return *keyword;
220219
}
221220
}
222221
return {};
223222
}
224223

224+
std::optional<std::string_view> peekKeyword() {
225+
return Lexer(*this).takeKeyword();
226+
}
227+
225228
bool takeKeyword(std::string_view expected) {
226-
if (auto t = peek()) {
227-
if (auto keyword = t->getKeyword()) {
229+
if (curr) {
230+
if (auto keyword = curr->getKeyword()) {
228231
if (*keyword == expected) {
229232
advance();
230233
return true;
@@ -236,16 +239,16 @@ struct Lexer {
236239

237240
std::optional<uint64_t> takeOffset() {
238241
using namespace std::string_view_literals;
239-
if (auto t = peek()) {
240-
if (auto keyword = t->getKeyword()) {
242+
if (curr) {
243+
if (auto keyword = curr->getKeyword()) {
241244
if (keyword->substr(0, 7) != "offset="sv) {
242245
return {};
243246
}
244247
Lexer subLexer(keyword->substr(7));
245248
if (subLexer.empty()) {
246249
return {};
247250
}
248-
if (auto o = subLexer.peek()->getU<uint64_t>()) {
251+
if (auto o = subLexer.curr->getU<uint64_t>()) {
249252
subLexer.advance();
250253
if (subLexer.empty()) {
251254
advance();
@@ -259,16 +262,16 @@ struct Lexer {
259262

260263
std::optional<uint32_t> takeAlign() {
261264
using namespace std::string_view_literals;
262-
if (auto t = peek()) {
263-
if (auto keyword = t->getKeyword()) {
265+
if (curr) {
266+
if (auto keyword = curr->getKeyword()) {
264267
if (keyword->substr(0, 6) != "align="sv) {
265268
return {};
266269
}
267270
Lexer subLexer(keyword->substr(6));
268271
if (subLexer.empty()) {
269272
return {};
270273
}
271-
if (auto a = subLexer.peek()->getU<uint32_t>()) {
274+
if (auto a = subLexer.curr->getU<uint32_t>()) {
272275
subLexer.advance();
273276
if (subLexer.empty()) {
274277
advance();
@@ -281,8 +284,8 @@ struct Lexer {
281284
}
282285

283286
template<typename T> std::optional<T> takeU() {
284-
if (auto t = peek()) {
285-
if (auto n = t->getU<T>()) {
287+
if (curr) {
288+
if (auto n = curr->getU<T>()) {
286289
advance();
287290
return n;
288291
}
@@ -291,8 +294,8 @@ struct Lexer {
291294
}
292295

293296
template<typename T> std::optional<T> takeI() {
294-
if (auto t = peek()) {
295-
if (auto n = t->getI<T>()) {
297+
if (curr) {
298+
if (auto n = curr->getI<T>()) {
296299
advance();
297300
return n;
298301
}
@@ -315,8 +318,8 @@ struct Lexer {
315318
std::optional<uint8_t> takeI8() { return takeI<uint8_t>(); }
316319

317320
std::optional<double> takeF64() {
318-
if (auto t = peek()) {
319-
if (auto d = t->getF64()) {
321+
if (curr) {
322+
if (auto d = curr->getF64()) {
320323
advance();
321324
return d;
322325
}
@@ -325,8 +328,8 @@ struct Lexer {
325328
}
326329

327330
std::optional<float> takeF32() {
328-
if (auto t = peek()) {
329-
if (auto f = t->getF32()) {
331+
if (curr) {
332+
if (auto f = curr->getF32()) {
330333
advance();
331334
return f;
332335
}
@@ -335,10 +338,11 @@ struct Lexer {
335338
}
336339

337340
std::optional<std::string> takeString() {
338-
if (auto t = peek()) {
339-
if (auto s = t->getString()) {
341+
if (curr) {
342+
if (auto s = curr->getString()) {
343+
std::string ret(*s);
340344
advance();
341-
return std::string(*s);
345+
return ret;
342346
}
343347
}
344348
return {};
@@ -392,8 +396,8 @@ struct Lexer {
392396
TextPos position() const { return position(getPos()); }
393397

394398
size_t getPos() const {
395-
if (auto t = peek()) {
396-
return getIndex() - t->span.size();
399+
if (curr) {
400+
return getIndex() - curr->span.size();
397401
}
398402
return getIndex();
399403
}

‎src/parser/parsers.h‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -549,7 +549,7 @@ template<typename Ctx> Result<typename Ctx::FieldT> fieldtype(Ctx& ctx) {
549549
template<typename Ctx> Result<typename Ctx::FieldsT> fields(Ctx& ctx) {
550550
auto res = ctx.makeFields();
551551
while (true) {
552-
if (auto t = ctx.in.peek(); !t || t->isRParen()) {
552+
if (ctx.in.empty() || ctx.in.peekRParen()) {
553553
return res;
554554
}
555555
if (ctx.in.takeSExprStart("field")) {
@@ -754,13 +754,11 @@ template<typename Ctx> MaybeResult<> plaininstr(Ctx& ctx) {
754754
// instr ::= plaininstr | blockinstr
755755
template<typename Ctx> MaybeResult<> instr(Ctx& ctx) {
756756
// Check for valid strings that are not instructions.
757-
if (auto tok = ctx.in.peek()) {
758-
if (auto keyword = tok->getKeyword()) {
759-
if (keyword == "end"sv || keyword == "then"sv || keyword == "else"sv ||
760-
keyword == "catch"sv || keyword == "catch_all"sv ||
761-
keyword == "delegate"sv || keyword == "ref"sv) {
762-
return {};
763-
}
757+
if (auto keyword = ctx.in.peekKeyword()) {
758+
if (keyword == "end"sv || keyword == "then"sv || keyword == "else"sv ||
759+
keyword == "catch"sv || keyword == "catch_all"sv ||
760+
keyword == "delegate"sv || keyword == "ref"sv) {
761+
return {};
764762
}
765763
}
766764
if (auto inst = blockinstr(ctx)) {
@@ -775,7 +773,7 @@ template<typename Ctx> MaybeResult<> instr(Ctx& ctx) {
775773

776774
template<typename Ctx> MaybeResult<> foldedinstr(Ctx& ctx) {
777775
// We must have an '(' to start a folded instruction.
778-
if (auto tok = ctx.in.peek(); !tok || !tok->isLParen()) {
776+
if (!ctx.in.peekLParen()) {
779777
return {};
780778
}
781779

@@ -2965,7 +2963,7 @@ template<typename Ctx> MaybeResult<> tag(Ctx& ctx) {
29652963
// | data
29662964
// | tag
29672965
template<typename Ctx> MaybeResult<> modulefield(Ctx& ctx) {
2968-
if (auto t = ctx.in.peek(); !t || t->isRParen()) {
2966+
if (ctx.in.empty() || ctx.in.peekRParen()) {
29692967
return {};
29702968
}
29712969
if (auto res = deftype(ctx)) {

0 commit comments

Comments
 (0)