feat: basic symbol table

bin/nextls

@@ -1,4 +1,6 @@
-#!/usr/bin/env -S elixir --sname undefined
+#!/usr/bin/env elixir
+
+Node.start("next-ls-#{System.system_time()}", :shortnames)
 
 System.no_halt(true)
 

bin/start

@@ -4,4 +4,4 @@
 
 cd "$(dirname "$0")"/.. || exit 1
 
-elixir --sname undefined -S mix run --no-halt -e "Application.ensure_all_started(:next_ls)" -- "$@"
+elixir --sname "next-ls-$RANDOM" -S mix run --no-halt -e "Application.ensure_all_started(:next_ls)" -- "$@"

lib/next_ls.ex

@@ -40,6 +40,7 @@ defmodule NextLS do
 
   alias NextLS.Runtime
   alias NextLS.DiagnosticCache
+  alias NextLS.SymbolTable
 
   def start_link(args) do
     {args, opts} =
@@ -48,7 +49,8 @@ defmodule NextLS do
         :task_supervisor,
         :dynamic_supervisor,
         :extensions,
-        :extension_registry
+        :extension_registry,
+        :symbol_table
       ])
 
     GenLSP.start_link(__MODULE__, args, opts)
@@ -61,13 +63,15 @@ defmodule NextLS do
     extension_registry = Keyword.fetch!(args, :extension_registry)
     extensions = Keyword.get(args, :extensions, [NextLS.ElixirExtension])
     cache = Keyword.fetch!(args, :cache)
+    symbol_table = Keyword.fetch!(args, :symbol_table)
 
     {:ok,
      assign(lsp,
        exit_code: 1,
        documents: %{},
        refresh_refs: %{},
        cache: cache,
+       symbol_table: symbol_table,
        task_supervisor: task_supervisor,
        dynamic_supervisor: dynamic_supervisor,
        extension_registry: extension_registry,
@@ -268,6 +272,11 @@ defmodule NextLS do
     {:noreply, lsp}
   end
 
+  def handle_info({:tracer, payload}, lsp) do
+    SymbolTable.put_symbols(lsp.assigns.symbol_table, payload)
+    {:noreply, lsp}
+  end
+
   def handle_info(:publish, lsp) do
     all =
       for {_namespace, cache} <- DiagnosticCache.get(lsp.assigns.cache), {file, diagnostics} <- cache, reduce: %{} do
@@ -342,7 +351,7 @@ defmodule NextLS do
     {:noreply, lsp}
   end
 
-  def handle_info(_, lsp) do
+  def handle_info(_message, lsp) do
     {:noreply, lsp}
   end
 

lib/next_ls/lsp_supervisor.ex

@@ -36,10 +36,12 @@ defmodule NextLS.LSPSupervisor do
         {DynamicSupervisor, name: NextLS.DynamicSupervisor},
         {Task.Supervisor, name: NextLS.TaskSupervisor},
         {GenLSP.Buffer, buffer_opts},
-        {NextLS.DiagnosticCache, [name: :diagnostic_cache]},
+        {NextLS.DiagnosticCache, name: :diagnostic_cache},
+        {NextLS.SymbolTable, name: :symbol_table, path: Path.expand("~/.cache/nvim/elixir-tools.nvim")},
         {Registry, name: NextLS.ExtensionRegistry, keys: :duplicate},
         {NextLS,
          cache: :diagnostic_cache,
+         symbol_table: :symbol_table,
          task_supervisor: NextLS.TaskSupervisor,
          dynamic_supervisor: NextLS.DynamicSupervisor,
          extension_registry: NextLS.ExtensionRegistry}

lib/next_ls/runtime.ex

@@ -40,7 +40,7 @@ defmodule NextLS.Runtime do
 
   @impl GenServer
   def init(opts) do
-    sname = "nextls#{System.system_time()}"
+    sname = "nextls-runtime-#{System.system_time()}"
     working_dir = Keyword.fetch!(opts, :working_dir)
     parent = Keyword.fetch!(opts, :parent)
     extension_registry = Keyword.fetch!(opts, :extension_registry)
@@ -55,6 +55,7 @@ defmodule NextLS.Runtime do
           :stream,
           cd: working_dir,
           env: [
+            {'NEXTLS_PARENT_PID', :erlang.term_to_binary(parent) |> Base.encode64() |> String.to_charlist()},
             {'MIX_ENV', 'dev'},
             {'MIX_BUILD_ROOT', '.elixir-tools/_build'}
           ],
@@ -87,6 +88,8 @@ defmodule NextLS.Runtime do
         |> Path.join("monkey/_next_ls_private_compiler.ex")
         |> then(&:rpc.call(node, Code, :compile_file, [&1]))
 
+        :rpc.call(node, Code, :put_compiler_option, [:parser_options, [columns: true, token_metadata: true]])
+
         send(me, {:node, node})
       else
         _ -> send(me, :cancel)

lib/next_ls/symbol_table.ex

@@ -0,0 +1,71 @@
+defmodule NextLS.SymbolTable do
+  @moduledoc false
+  use GenServer
+
+  defmodule Symbol do
+    defstruct [:file, :module, :type, :name, :line, :col]
+
+    def new(args) do
+      struct(__MODULE__, args)
+    end
+  end
+
+  def start_link(args) do
+    GenServer.start_link(__MODULE__, Keyword.take(args, [:path]), Keyword.take(args, [:name]))
+  end
+
+  @spec put_symbols(pid() | atom(), list(tuple())) :: :ok
+  def put_symbols(server, symbols), do: GenServer.cast(server, {:put_symbols, symbols})
+  @spec symbols(pid() | atom()) :: list(struct())
+  def symbols(server), do: GenServer.call(server, :symbols)
+
+  def init(args) do
+    path = Keyword.fetch!(args, :path)
+
+    {:ok, name} =
+      :dets.open_file(:symbol_table,
+        file: Path.join(path, "symbol_table.dets") |> String.to_charlist(),
+        type: :duplicate_bag
+      )
+
+    {:ok, %{table: name}}
+  end
+
+  def handle_call(:symbols, _, state) do
+    symbols =
+      :dets.foldl(
+        fn {_key, symbol}, acc -> [symbol | acc] end,
+        [],
+        state.table
+      )
+
+    {:reply, symbols, state}
+  end
+
+  def handle_cast({:put_symbols, symbols}, state) do
+    %{
+      module: mod,
+      file: file,
+      defs: defs
+    } = symbols
+
+    :dets.delete(state.table, mod)
+
+    for {name, {:v1, type, _meta, clauses}} <- defs, {meta, _, _, _} <- clauses do
+      :dets.insert(
+        state.table,
+        {mod,
+         %Symbol{
+           module: mod,
+           file: file,
+           type: type,
+           name: name,
+           line: meta[:line],
+           col: meta[:column]
+         }}
+      )
+    end
+
+    {:noreply, state}
+  end
+end

priv/monkey/_next_ls_private_compiler.ex

@@ -1,3 +1,24 @@
+defmodule NextLSPrivate.Tracer do
+  def trace({:on_module, _, _}, env) do
+    parent = "NEXTLS_PARENT_PID" |> System.get_env() |> Base.decode64!() |> :erlang.binary_to_term()
+
+    defs = Module.definitions_in(env.module)
+
+    defs =
+      for {name, arity} = _def <- defs do
+        {name, Module.get_definition(env.module, {name, arity})}
+      end
+
+    Process.send(parent, {:tracer, %{file: env.file, module: env.module, defs: defs}}, [])
+
+    :ok
+  end
+
+  def trace(_event, _env) do
+    :ok
+  end
+end
+
 defmodule :_next_ls_private_compiler do
   @moduledoc false
 
@@ -15,7 +36,7 @@ defmodule :_next_ls_private_compiler do
     # --no-compile, so nothing was compiled, but the
     # task was not re-enabled it seems
     Mix.Task.rerun("deps.loadpaths")
-    Mix.Task.rerun("compile", ["--no-protocol-consolidation", "--return-errors"])
+    Mix.Task.rerun("compile", ["--no-protocol-consolidation", "--return-errors", "--tracer", "NextLSPrivate.Tracer"])
   rescue
     e -> {:error, e}
   end

test/next_ls/runtime_test.exs

@@ -66,12 +66,18 @@ defmodule NextLs.RuntimeTest do
                  severity: :warning,
                  message:
                    "variable \"arg1\" is unused (if the variable is not meant to be used, prefix it with an underscore)",
-                 position: 2,
+                 position: position,
                  compiler_name: "Elixir",
                  details: nil
                }
              ] = Runtime.compile(pid)
 
+      if Version.match?(System.version(), ">= 1.15.0") do
+        assert position == {2, 11}
+      else
+        assert position == 2
+      end
+
       File.write!(file, """
       defmodule Bar do
         def foo(arg1) do

test/next_ls/symbol_table_test.exs

@@ -0,0 +1,108 @@
+defmodule NextLS.SymbolTableTest do
+  use ExUnit.Case, async: true
+  @moduletag :tmp_dir
+
+  alias NextLS.SymbolTable
+
+  setup %{tmp_dir: dir} do
+    pid = start_supervised!({SymbolTable, [path: dir]})
+
+    Process.link(pid)
+    [pid: pid, dir: dir]
+  end
+
+  test "creates a dets table", %{dir: dir, pid: pid} do
+    assert File.exists?(Path.join([dir, "symbol_table.dets"]))
+    assert :sys.get_state(pid).table == :symbol_table
+  end
+
+  test "builds the symbol table", %{pid: pid} do
+    symbols = symbols()
+
+    SymbolTable.put_symbols(pid, symbols)
+
+    assert [
+             %SymbolTable.Symbol{
+               module: "NextLS",
+               file: "/Users/alice/next_ls/lib/next_ls.ex",
+               type: :def,
+               name: :start_link,
+               line: 45,
+               col: nil
+             },
+             %SymbolTable.Symbol{
+               module: "NextLS",
+               file: "/Users/alice/next_ls/lib/next_ls.ex",
+               type: :def,
+               name: :start_link,
+               line: 44,
+               col: nil
+             }
+           ] == SymbolTable.symbols(pid)
+  end
+
+  defp symbols() do
+    %{
+      file: "/Users/alice/next_ls/lib/next_ls.ex",
+      module: "NextLS",
+      defs: [
+        start_link:
+          {:v1, :def, [line: 44],
+           [
+             {[line: 44], [{:args, [version: 0, line: 44, column: 18], nil}], [],
+              {:__block__, [],
+               [
+                 {:=,
+                  [
+                    end_of_expression: [newlines: 2, line: 52, column: 9],
+                    line: 45,
+                    column: 18
+                  ],
+                  [
+                    {{:args, [version: 1, line: 45, column: 6], nil}, {:opts, [version: 2, line: 45, column: 12], nil}},
+                    {{:., [line: 46, column: 14], [Keyword, :split]},
+                     [closing: [line: 52, column: 8], line: 46, column: 15],
+                     [
+                       {:args, [version: 0, line: 46, column: 21], nil},
+                       [:cache, :task_supervisor, :dynamic_supervisor, :extensions, :extension_registry]
+                     ]}
+                  ]},
+                 {{:., [line: 54, column: 11], [GenLSP, :start_link]},
+                  [closing: [line: 54, column: 45], line: 54, column: 12],
+                  [
+                    NextLS,
+                    {:args, [version: 1, line: 54, column: 35], nil},
+                    {:opts, [version: 2, line: 54, column: 41], nil}
+                  ]}
+               ]}},
+             {[line: 45], [{:args, [version: 0, line: 45, column: 18], nil}], [],
+              {:__block__, [],
+               [
+                 {:=,
+                  [
+                    end_of_expression: [newlines: 2, line: 52, column: 9],
+                    line: 45,
+                    column: 18
+                  ],
+                  [
+                    {{:args, [version: 1, line: 45, column: 6], nil}, {:opts, [version: 2, line: 45, column: 12], nil}},
+                    {{:., [line: 46, column: 14], [Keyword, :split]},
+                     [closing: [line: 52, column: 8], line: 46, column: 15],
+                     [
+                       {:args, [version: 0, line: 46, column: 21], nil},
+                       [:cache, :task_supervisor, :dynamic_supervisor, :extensions, :extension_registry]
+                     ]}
+                  ]},
+                 {{:., [line: 54, column: 11], [GenLSP, :start_link]},
+                  [closing: [line: 54, column: 45], line: 54, column: 12],
+                  [
+                    NextLS,
+                    {:args, [version: 1, line: 54, column: 35], nil},
+                    {:opts, [version: 2, line: 54, column: 41], nil}
+                  ]}
+               ]}}
+           ]}
+      ]
+    }
+  end
+end

test/next_ls_test.exs

@@ -15,14 +15,16 @@ defmodule NextLSTest do
     start_supervised!({Registry, [keys: :unique, name: Registry.NextLSTest]})
     extensions = [NextLS.ElixirExtension]
     cache = start_supervised!(NextLS.DiagnosticCache)
+    symbol_table = start_supervised!({NextLS.SymbolTable, [path: tmp_dir]})
 
     server =
       server(NextLS,
         task_supervisor: tvisor,
         dynamic_supervisor: rvisor,
         extension_registry: Registry.NextLSTest,
         extensions: extensions,
-        cache: cache
+        cache: cache,
+        symbol_table: symbol_table
       )
 
     Process.link(server.lsp)
@@ -154,6 +156,8 @@ defmodule NextLSTest do
           path: Path.join([cwd, "lib", file])
         })
 
+      char = if Version.match?(System.version(), ">= 1.15.0"), do: 11, else: 0
+
       assert_notification "textDocument/publishDiagnostics", %{
         "uri" => ^uri,
         "diagnostics" => [
@@ -163,7 +167,7 @@ defmodule NextLSTest do
             "message" =>
               "variable \"arg1\" is unused (if the variable is not meant to be used, prefix it with an underscore)",
             "range" => %{
-              "start" => %{"line" => 1, "character" => 0},
+              "start" => %{"line" => 1, "character" => ^char},
               "end" => %{"line" => 1, "character" => 999}
             }
           }

test/test_helper.exs

@@ -1,6 +1,6 @@
 {:ok, _pid} = Node.start(:"nextls#{System.system_time()}", :shortnames)
 
-Logger.configure(level: :warn)
+Logger.configure(level: :warning)
 
 timeout =
   if System.get_env("CI", "false") == "true" do