feat(linter/plugins): Token-related SourceCode APIs (TS ESLint implementation)

apps/oxlint/package.json

@@ -20,17 +20,18 @@
     "node": "^20.19.0 || >=22.12.0"
   },
   "devDependencies": {
-    "@typescript-eslint/scope-manager": "8.46.2",
     "@types/esquery": "^1.5.4",
     "@types/estree": "^1.0.8",
+    "@typescript-eslint/scope-manager": "8.46.2",
+    "@typescript-eslint/typescript-estree": "^8.47.0",
     "eslint": "^9.36.0",
     "esquery": "^1.6.0",
     "execa": "^9.6.0",
     "jiti": "^2.6.0",
     "rolldown": "catalog:",
     "tsdown": "catalog:",
     "type-fest": "^5.2.0",
-    "typescript": "catalog:",
+    "typescript": "5.9.3",
     "vitest": "catalog:"
   },
   "napi": {

apps/oxlint/src-js/index.ts

@@ -23,7 +23,28 @@ export type {
 } from './plugins/scope.ts';
 export type { Settings } from './plugins/settings.ts';
 export type { SourceCode } from './plugins/source_code.ts';
-export type { CountOptions, FilterFn, RangeOptions, SkipOptions } from './plugins/tokens.ts';
+export type {
+  CountOptions,
+  FilterFn,
+  RangeOptions,
+  SkipOptions,
+  Token,
+  BooleanToken,
+  CommentToken,
+  BlockCommentToken,
+  LineCommentToken,
+  IdentifierToken,
+  JSXIdentifierToken,
+  JSXTextToken,
+  KeywordToken,
+  NullToken,
+  NumericToken,
+  PrivateIdentifierToken,
+  PunctuatorToken,
+  RegularExpressionToken,
+  StringToken,
+  TemplateToken,
+} from './plugins/tokens.ts';
 export type {
   RuleMeta,
   RuleDocs,
@@ -33,16 +54,7 @@ export type {
   RuleReplacedByExternalSpecifier,
 } from './plugins/rule_meta.ts';
 export type { LineColumn, Location, Range, Ranged, Span } from './plugins/location.ts';
-export type {
-  AfterHook,
-  BeforeHook,
-  Comment,
-  Node,
-  NodeOrToken,
-  Token,
-  Visitor,
-  VisitorWithHooks,
-} from './plugins/types.ts';
+export type { AfterHook, BeforeHook, Comment, Node, NodeOrToken, Visitor, VisitorWithHooks } from './plugins/types.ts';
 
 const {
   defineProperty,

apps/oxlint/src-js/plugins/source_code.ts

@@ -17,6 +17,7 @@ import {
 } from './location.js';
 import { resetScopeManager, SCOPE_MANAGER } from './scope.js';
 import * as scopeMethods from './scope.js';
+import { resetTokens } from './tokens.js';
 import * as tokenMethods from './tokens.js';
 import { assertIsNonNull } from '../utils/asserts.js';
 
@@ -98,6 +99,7 @@ export function resetSourceAndAst(): void {
   resetBuffer();
   resetLines();
   resetScopeManager();
+  resetTokens();
 }
 
 // `SourceCode` object.

apps/oxlint/src-js/plugins/tokens.ts

@@ -2,10 +2,14 @@
  * `SourceCode` methods related to tokens.
  */
 
+import { parse } from '@typescript-eslint/typescript-estree';
 import { sourceText, initSourceText } from './source_code.js';
 import { assertIsNonNull } from '../utils/asserts.js';
 
-import type { Comment, Node, NodeOrToken, Token } from './types.ts';
+import type { Comment, Node, NodeOrToken } from './types.ts';
+import type { Span } from './location.ts';
+
+const { max } = Math;
 
 /**
  * Options for various `SourceCode` methods e.g. `getFirstToken`.
@@ -44,10 +48,124 @@ export interface RangeOptions {
  */
 export type FilterFn = (token: Token) => boolean;
 
+/**
+ * AST token type.
+ */
+export type Token =
+  | BooleanToken
+  | CommentToken
+  | IdentifierToken
+  | JSXIdentifierToken
+  | JSXTextToken
+  | KeywordToken
+  | NullToken
+  | NumericToken
+  | PrivateIdentifierToken
+  | PunctuatorToken
+  | RegularExpressionToken
+  | StringToken
+  | TemplateToken;
+
+interface BaseToken extends Omit<Span, 'start' | 'end'> {
+  type: Token['type'];
+  value: string;
+}
+
+export interface BooleanToken extends BaseToken {
+  type: 'Boolean';
+}
+
+export type CommentToken = BlockCommentToken | LineCommentToken;
+
+export interface BlockCommentToken extends BaseToken {
+  type: 'Block';
+}
+
+export interface LineCommentToken extends BaseToken {
+  type: 'Line';
+}
+
+export interface IdentifierToken extends BaseToken {
+  type: 'Identifier';
+}
+
+export interface JSXIdentifierToken extends BaseToken {
+  type: 'JSXIdentifier';
+}
+
+export interface JSXTextToken extends BaseToken {
+  type: 'JSXText';
+}
+
+export interface KeywordToken extends BaseToken {
+  type: 'Keyword';
+}
+
+export interface NullToken extends BaseToken {
+  type: 'Null';
+}
+
+export interface NumericToken extends BaseToken {
+  type: 'Numeric';
+}
+
+export interface PrivateIdentifierToken extends BaseToken {
+  type: 'PrivateIdentifier';
+}
+
+export interface PunctuatorToken extends BaseToken {
+  type: 'Punctuator';
+}
+
+export interface RegularExpressionToken extends BaseToken {
+  type: 'RegularExpression';
+  regex: {
+    flags: string;
+    pattern: string;
+  };
+}
+
+export interface StringToken extends BaseToken {
+  type: 'String';
+}
+
+export interface TemplateToken extends BaseToken {
+  type: 'Template';
+}
+
+// Tokens for the current file parsed by TS-ESLint.
+// Created lazily only when needed.
+let tokens: Token[] | null = null;
+let comments: CommentToken[] | null = null;
+let tokensWithComments: Token[] | null = null;
+
+/**
+ * Initialize TS-ESLint tokens for current file.
+ */
+function initTokens() {
+  assertIsNonNull(sourceText);
+  ({ tokens, comments } = parse(sourceText, {
+    sourceType: 'module',
+    tokens: true,
+    comment: true,
+    // TODO: Enable JSX only when needed
+    jsx: true,
+  }));
+}
+
+/**
+ * Discard TS-ESLint tokens to free memory.
+ */
+export function resetTokens() {
+  tokens = null;
+  comments = null;
+  tokensWithComments = null;
+}
+
 /**
  * Get all tokens that are related to the given node.
  * @param node - The AST node.
- * @param countOptions? - Options object. If this is a function then it's `options.filter`.
+ * @param countOptions? - Options object. If this is a function then it's `countOptions.filter`.
  * @returns Array of `Token`s.
  */
 /**
@@ -57,15 +175,92 @@ export type FilterFn = (token: Token) => boolean;
  * @param afterCount? - The number of tokens after the node to retrieve.
  * @returns Array of `Token`s.
  */
-/* oxlint-disable no-unused-vars */
 export function getTokens(
   node: Node,
   countOptions?: CountOptions | number | FilterFn | null,
   afterCount?: number | null,
 ): Token[] {
-  throw new Error('`sourceCode.getTokens` not implemented yet'); // TODO
+  if (tokens === null) initTokens();
+  assertIsNonNull(tokens);
+  assertIsNonNull(comments);
+
+  // Maximum number of tokens to return
+  const count = typeof countOptions === 'object' && countOptions !== null ? countOptions.count : null;
+
+  // Number of preceding tokens to additionally return
+  const beforeCount = typeof countOptions === 'number' ? countOptions : 0;
+
+  // Number of following tokens to additionally return
+  afterCount =
+    (typeof countOptions === 'number' || typeof countOptions === 'undefined') && typeof afterCount === 'number'
+      ? afterCount
+      : 0;
+
+  // Function to filter tokens
+  const filter =
+    typeof countOptions === 'function'
+      ? countOptions
+      : typeof countOptions === 'object' && countOptions !== null
+        ? countOptions.filter
+        : null;
+
+  // Whether to return comment tokens
+  const includeComments =
+    typeof countOptions === 'object' &&
+    countOptions !== null &&
+    'includeComments' in countOptions &&
+    countOptions.includeComments;
+
+  // Source array of tokens to search in
+  let nodeTokens: Token[] | null = null;
+  if (includeComments) {
+    if (tokensWithComments === null) {
+      tokensWithComments = [...tokens, ...comments].sort((a, b) => a.range[0] - b.range[0]);
+    }
+    nodeTokens = tokensWithComments;
+  } else {
+    nodeTokens = tokens;
+  }
+
+  let sliceStart = nodeTokens.length;
+  let sliceEnd: number | undefined = undefined;
+
+  const { range } = node,
+    rangeStart = range[0],
+    rangeEnd = range[1];
+
+  // Binary search for first token within `node`'s range
+  for (let lo = 0, hi = nodeTokens.length; lo < hi; ) {
+    const mid = (lo + hi) >> 1;
+    if (nodeTokens[mid].range[0] < rangeStart) {
+      lo = mid + 1;
+    } else {
+      sliceStart = hi = mid;
+    }
+  }
+
+  // Binary search for the first token outside `node`'s range
+  for (let lo = sliceStart, hi = nodeTokens.length; lo < hi; ) {
+    const mid = (lo + hi) >> 1;
+    if (nodeTokens[mid].range[0] < rangeEnd) {
+      lo = mid + 1;
+    } else {
+      sliceEnd = hi = mid;
+    }
+  }
+
+  sliceStart = max(0, sliceStart - beforeCount);
+  // `sliceEnd` would remain undefined here if the node contains the last token of the file
+  if (sliceEnd !== undefined) sliceEnd += afterCount;
+
+  nodeTokens = nodeTokens.slice(sliceStart, sliceEnd);
+
+  // Filter before limiting by `count`
+  if (filter) nodeTokens = nodeTokens.filter(filter);
+  if (typeof count === 'number' && count < nodeTokens.length) nodeTokens = nodeTokens.slice(0, count);
+
+  return nodeTokens;
 }
-/* oxlint-enable no-unused-vars */
 
 /**
  * Get the first token of the given node.

apps/oxlint/src-js/plugins/types.ts

@@ -7,6 +7,7 @@ export interface Visitor {
 */
 
 import type { Span } from './location.ts';
+import type { Token } from './tokens.ts';
 
 import type { VisitorObject as Visitor } from '../generated/visitor.d.ts';
 export type { Visitor };
@@ -30,12 +31,6 @@ export type VisitFn = (node: Node) => void;
 // AST node type.
 export interface Node extends Span {}
 
-// AST token type.
-export interface Token extends Span {
-  type: string;
-  value: string;
-}
-
 // Currently we only support `Node`s, but will add support for `Token`s later.
 export type NodeOrToken = Node | Token;
 

apps/oxlint/test/fixtures/sourceCode_token_methods/.oxlintrc.json

@@ -0,0 +1,9 @@
+{
+  "jsPlugins": ["./plugin.ts"],
+  "categories": {
+    "correctness": "off"
+  },
+  "rules": {
+    "token-plugin/token": "error"
+  }
+}

apps/oxlint/test/fixtures/sourceCode_token_methods/files/eslint-test-case.js

@@ -0,0 +1,3 @@
+/*A*/ var answer /*B*/ = /*C*/ a /*D*/ * b; /*E*/ //F
+call();
+/*Z*/