Build improvements (#859)

* return all warnings from compilation

make diagnostics do not disappear in umbrella
disable protoco consolidation

* unload MixProject modules from deps and apps

reduce number of warnings emitted on recompilation

* debug unexpected responses

* add TODO

fix typo

* we no longer need to purge protocols as we are not consolidating

* Do not emit protocol consolidation diagnostics

* since consolidation is disabled we don't need to dialize consolidated beams

revert to dialysing non consolidated protocol beams from the build dir

* add comment

* ignore_already_consolidated is available on elixir 1.14+

* fix dialyzer

* return 0 exit code on restart

* do not emit undefined warnings from mix.exs compile

* make debugger behave more similar to mix 1.14

* add todo

* remove dbg

* add comment

* correctly append params

* fix crash

Author: lukaszsamson

apps/elixir_ls_debugger/lib/debugger/server.ex

@@ -27,7 +27,6 @@ defmodule ElixirLS.Debugger.Server do
   }
 
   alias ElixirLS.Debugger.Stacktrace.Frame
-  alias ElixirLS.Utils.MixfileHelpers
   use GenServer
   use Protocol
 
@@ -926,27 +925,23 @@ defmodule ElixirLS.Debugger.Server do
   end
 
   defp initialize(%{"projectDir" => project_dir} = config) do
-    prev_env = Mix.env()
     task = config["task"]
-    task_args = config["taskArgs"]
+    task_args = config["taskArgs"] || []
     auto_interpret_files? = Map.get(config, "debugAutoInterpretAllModules", true)
 
     set_stack_trace_mode(config["stackTraceMode"])
     set_env_vars(config["env"])
 
     File.cd!(project_dir)
 
-    # Mixfile may already be loaded depending on cwd when launching debugger task
-    mixfile = Path.absname(MixfileHelpers.mix_exs())
+    # the startup sequence here is taken from
+    # https://github.com/elixir-lang/elixir/blob/v1.14.4/lib/mix/lib/mix/cli.ex#L158
+    # we assume that mix is already started and has archives and tasks loaded
+    ElixirLS.Utils.Launch.load_mix_exs()
+    {task, task_args} = ElixirLS.Utils.Launch.get_task(List.wrap(task) ++ task_args)
+    ElixirLS.Utils.Launch.maybe_change_env_and_target(task)
 
-    # FIXME: Private API
-    unless match?(%{file: ^mixfile}, Mix.ProjectStack.peek()) do
-      Code.compile_file(MixfileHelpers.mix_exs())
-    end
-
-    task = task || Mix.Project.config()[:default_task]
-    env = task_env(task)
-    if env != prev_env, do: change_env(env)
+    Output.debugger_console("Running with MIX_ENV: #{Mix.env()} MIX_TARGET: #{Mix.target()}\n")
 
     Mix.Task.run("loadconfig")
 
@@ -1068,32 +1063,8 @@ defmodule ElixirLS.Debugger.Server do
     end
   end
 
-  defp change_env(env) do
-    Mix.env(env)
-
-    # FIXME: Private API
-    if project = Mix.Project.pop() do
-      %{name: name, file: file} = project
-      :code.purge(name)
-      :code.delete(name)
-      # It's important to use `compile_file` here instead of `require_file`
-      # because we are recompiling this file to reload the mix project back onto
-      # the project stack.
-      Code.compile_file(file)
-    end
-  end
-
-  defp task_env(task) do
-    if System.get_env("MIX_ENV") do
-      String.to_atom(System.get_env("MIX_ENV"))
-    else
-      task = String.to_atom(task)
-      Mix.Project.config()[:preferred_cli_env][task] || Mix.Task.preferred_cli_env(task) || :dev
-    end
-  end
-
   defp launch_task(task, args) do
-    # This fixes a race condition in  the tests and likely improves reliability when using the
+    # This fixes a race condition in the tests and likely improves reliability when using the
     # debugger as well.
     Process.sleep(100)
 

apps/elixir_ls_debugger/mix.exs

@@ -13,6 +13,8 @@ defmodule ElixirLS.Debugger.Mixfile do
       build_embedded: false,
       start_permanent: true,
       build_per_environment: false,
+      # if we consolidate here debugged code will not work correctly
+      # and debugged protocol implementation will not be available
       consolidate_protocols: false,
       deps: deps(),
       xref: [exclude: [:int, :dbg_iserver]]

apps/elixir_ls_utils/lib/launch.ex

@@ -3,18 +3,26 @@ defmodule ElixirLS.Utils.Launch do
   @compiled_otp_version System.otp_release()
 
   def start_mix do
+    # reset env and target if it is set
+    Mix.env(:dev)
+    Mix.target(:host)
+    System.put_env([{"MIX_ENV", nil}, {"MIX_TARGET", nil}])
+
     Mix.start()
+    Mix.Local.append_archives()
+    Mix.Local.append_paths()
+    true = Mix.Hex.ensure_installed?(false)
     # when running via mix install script mix starts and stops hex
     # we need to make sure it's started
     if function_exported?(Hex, :start, 0) do
       Hex.start()
     end
 
-    # FIXME: Private API
-    Mix.Local.append_archives()
-    # FIXME: Private API
-    Mix.Local.append_paths()
     load_dot_config()
+
+    # as of 1.14 mix supports two environment variables MIX_QUIET and MIX_DEBUG
+    # that are not important for our use cases
+
     :ok
   end
 
@@ -53,11 +61,103 @@ defmodule ElixirLS.Utils.Launch do
   end
 
   defp load_dot_config do
-    # FIXME: Private API
     path = Path.join(Mix.Utils.mix_home(), "config.exs")
 
     if File.regular?(path) do
       Mix.Task.run("loadconfig", [path])
     end
   end
+
+  def load_mix_exs() do
+    file = ElixirLS.Utils.MixfileHelpers.mix_exs()
+
+    if File.regular?(file) do
+      # TODO elixir 1.15 calls
+      # Mix.ProjectStack.post_config(state_loader: {:cli, List.first(args)})
+      # added in https://github.com/elixir-lang/elixir/commit/9e07da862784ac7d18a1884141c49ab049e61691
+      # def cli
+      # do we need that?
+      old_undefined = Code.get_compiler_option(:no_warn_undefined)
+      Code.put_compiler_option(:no_warn_undefined, :all)
+      Code.compile_file(file)
+      Code.put_compiler_option(:no_warn_undefined, old_undefined)
+    end
+  end
+
+  # TODO add support for def cli
+  def get_task(["-" <> _ | _]) do
+    task = "mix #{Mix.Project.config()[:default_task]}"
+
+    Mix.shell().error(
+      "** (Mix) Mix only recognizes the options --help and --version.\n" <>
+        "You may have wanted to invoke a task instead, such as #{inspect(task)}"
+    )
+
+    display_usage()
+    exit({:shutdown, 1})
+  end
+
+  def get_task([h | t]) do
+    {h, t}
+  end
+
+  def get_task([]) do
+    case Mix.Project.get() do
+      nil ->
+        Mix.shell().error(
+          "** (Mix) \"mix\" with no arguments must be executed in a directory with a mix.exs file"
+        )
+
+        display_usage()
+        exit({:shutdown, 1})
+
+      _ ->
+        {Mix.Project.config()[:default_task], []}
+    end
+  end
+
+  def maybe_change_env_and_target(task) do
+    task = String.to_atom(task)
+    config = Mix.Project.config()
+
+    env = preferred_cli_env(task, config)
+    target = preferred_cli_target(task, config)
+    env && Mix.env(env)
+    target && Mix.target(target)
+
+    if env || target do
+      reload_project()
+    end
+  end
+
+  defp reload_project() do
+    if project = Mix.Project.pop() do
+      %{name: name, file: file} = project
+      Mix.Project.push(name, file)
+    end
+  end
+
+  defp preferred_cli_env(task, config) do
+    if System.get_env("MIX_ENV") do
+      nil
+    else
+      config[:preferred_cli_env][task] || Mix.Task.preferred_cli_env(task)
+    end
+  end
+
+  defp preferred_cli_target(task, config) do
+    config[:preferred_cli_target][task]
+  end
+
+  defp display_usage do
+    Mix.shell().info("""
+    Usage: mix [task]
+    Examples:
+        mix             - Invokes the default task (mix run) in a project
+        mix new PATH    - Creates a new Elixir project at the given path
+        mix help        - Lists all available tasks
+        mix help TASK   - Prints documentation for a given task
+    The --help and --version options can be given instead of a task for usage and versioning information.
+    """)
+  end
 end

apps/language_server/lib/language_server/build.ex

@@ -31,8 +31,6 @@ defmodule ElixirLS.LanguageServer.Build do
                       fetch_deps(current_deps)
                     end
 
-                    # if we won't do it elixir >= 1.11 warns that protocols have already been consolidated
-                    purge_consolidated_protocols()
                     {status, diagnostics} = run_mix_compile()
 
                     diagnostics = Diagnostics.normalize(diagnostics, root_path)
@@ -78,6 +76,19 @@ defmodule ElixirLS.LanguageServer.Build do
 
     if File.exists?(mixfile) do
       if module = Mix.Project.get() do
+        build_path = Mix.Project.config()[:build_path]
+
+        for {app, path} <- Mix.Project.deps_paths() do
+          child_module =
+            Mix.Project.in_project(app, path, [build_path: build_path], fn mix_project ->
+              mix_project
+            end)
+
+          if child_module do
+            purge_module(child_module)
+          end
+        end
+
         # FIXME: Private API
         Mix.Project.pop()
         purge_module(module)
@@ -102,6 +113,18 @@ defmodule ElixirLS.LanguageServer.Build do
       # FIXME: Private API
       Mix.ProjectStack.post_config(build_path: ".elixir_ls/build")
 
+      # TODO elixir 1.15 calls
+      # Mix.ProjectStack.post_config(state_loader: {:cli, List.first(args)})
+      # added in https://github.com/elixir-lang/elixir/commit/9e07da862784ac7d18a1884141c49ab049e61691
+      # def cli
+      # do we need that?
+
+      # since elixir 1.10 mix disables undefined warnings for mix.exs
+      # see discussion in https://github.com/elixir-lang/elixir/issues/9676
+      # https://github.com/elixir-lang/elixir/blob/6f96693b355a9b670f2630fd8e6217b69e325c5a/lib/mix/lib/mix/cli.ex#L41
+      old_undefined = Code.get_compiler_option(:no_warn_undefined)
+      Code.put_compiler_option(:no_warn_undefined, :all)
+
       # We can get diagnostics if Mixfile fails to load
       {status, diagnostics} =
         case Kernel.ParallelCompiler.compile([mixfile]) do
@@ -116,6 +139,9 @@ defmodule ElixirLS.LanguageServer.Build do
             }
         end
 
+      # restore warnings
+      Code.put_compiler_option(:no_warn_undefined, old_undefined)
+
       if status == :ok do
         # The project may override our logger config, so we reset it after loading their config
         logger_config = Application.get_all_env(:logger)
@@ -136,15 +162,21 @@ defmodule ElixirLS.LanguageServer.Build do
   end
 
   defp run_mix_compile do
-    # TODO consider adding --no-compile
-    case Mix.Task.run("compile", ["--return-errors", "--ignore-module-conflict"]) do
+    # TODO --all-warnings not needed on 1.15
+    case Mix.Task.run("compile", [
+           "--return-errors",
+           "--ignore-module-conflict",
+           "--all-warnings",
+           "--no-protocol-consolidation"
+         ]) do
       {status, diagnostics} when status in [:ok, :error, :noop] and is_list(diagnostics) ->
         {status, diagnostics}
 
       status when status in [:ok, :noop] ->
         {status, []}
 
-      _ ->
+      other ->
+        Logger.debug("mix compile returned #{inspect(other)}")
         {:ok, []}
     end
   end
@@ -169,26 +201,6 @@ defmodule ElixirLS.LanguageServer.Build do
     end
   end
 
-  defp purge_consolidated_protocols do
-    config = Mix.Project.config()
-    path = Mix.Project.consolidation_path(config)
-
-    with {:ok, beams} <- File.ls(path) do
-      Enum.map(beams, &(&1 |> Path.rootname(".beam") |> String.to_atom() |> purge_module()))
-    else
-      {:error, :enoent} ->
-        # consolidation_path does not exist
-        :ok
-
-      {:error, reason} ->
-        Logger.warn("Unable to purge consolidated protocols from #{path}: #{inspect(reason)}")
-    end
-
-    # NOTE this implementation is based on https://github.com/phoenixframework/phoenix/commit/b5580e9
-    # calling `Code.delete_path(path)` may be unnecessary in our case
-    Code.delete_path(path)
-  end
-
   defp purge_module(module) do
     :code.purge(module)
     :code.delete(module)
@@ -334,19 +346,29 @@ defmodule ElixirLS.LanguageServer.Build do
 
   def set_compiler_options(options \\ [], parser_options \\ []) do
     parser_options =
-      parser_options
-      |> Keyword.merge(
+      Keyword.merge(parser_options,
         columns: true,
         token_metadata: true
       )
 
     options =
-      options
-      |> Keyword.merge(
+      Keyword.merge(options,
         tracers: [Tracer],
         parser_options: parser_options
       )
 
+    options =
+      if Version.match?(System.version(), ">= 1.14.0") do
+        Keyword.merge(options,
+          # we are running the server with consolidated protocols
+          # this disables warnings `X has already been consolidated`
+          # when running `compile` task
+          ignore_already_consolidated: true
+        )
+      else
+        options
+      end
+
     Code.compiler_options(options)
   end
 

apps/language_server/lib/language_server/cli.ex

@@ -48,9 +48,6 @@ defmodule ElixirLS.LanguageServer.CLI do
     Launch.limit_num_schedulers()
 
     Mix.shell(ElixirLS.LanguageServer.MixShell)
-    # FIXME: Private API
-    true = Mix.Hex.ensure_installed?(false)
-    true = Mix.Hex.ensure_updated?()
 
     WireProtocol.stream_packets(&JsonRpc.receive_packet/1)
   end

apps/language_server/lib/language_server/dialyzer.ex

@@ -214,19 +214,9 @@ defmodule ElixirLS.LanguageServer.Dialyzer do
   defp update_stale(md5, removed_files, file_changes, timestamp) do
     prev_paths = Map.keys(md5) |> MapSet.new()
 
-    # FIXME: Private API
-    consolidation_path = Mix.Project.consolidation_path()
-
-    consolidated_protocol_beams =
-      for path <- Path.join(consolidation_path, "*.beam") |> Path.wildcard(),
-          into: MapSet.new(),
-          do: Path.basename(path)
-
     # FIXME: Private API
     all_paths =
       for path <- Mix.Utils.extract_files([Mix.Project.build_path()], [:beam]),
-          Path.basename(path) not in consolidated_protocol_beams or
-            Path.dirname(path) == consolidation_path,
           into: MapSet.new(),
           do: Path.relative_to_cwd(path)