Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve performance of algorithm to determine project deps #518

Merged
merged 1 commit into from
Sep 16, 2023
Merged

Improve performance of algorithm to determine project deps #518

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
215 changes: 141 additions & 74 deletions lib/dialyxir/project.ex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ defmodule Dialyxir.Project do
alias Dialyxir.Formatter.Short
alias Dialyxir.Formatter.Utils

# Maximum depth in the dependency tree to traverse before giving up.
@max_dep_traversal_depth 100

def plts_list(deps, include_project \\ true, exclude_core \\ false) do
elixir_apps = [:elixir]
erlang_apps = [:erts, :kernel, :stdlib, :crypto]
Expand Down Expand Up @@ -308,124 +311,136 @@ defmodule Dialyxir.Project do
defp include_deps do
method = dialyzer_config()[:plt_add_deps]

reduce_umbrella_children([], fn deps ->
deps ++
initial_acc = {
_loaded_apps = [],
_unloaded_apps = [],
_initial_load_statuses = %{}
}

{loaded_apps, _unloaded_apps, _final_load_statuses} =
reduce_umbrella_children(initial_acc, fn acc ->
case method do
false ->
[]
acc

# compatibility
true ->
warning(
"Dialyxir has deprecated plt_add_deps: true in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:project ->
warning(
"Dialyxir has deprecated plt_add_deps: :project in favor of apps_direct, which includes only runtime dependencies."
)

deps_project() ++ deps_app(false)
acc
|> load_project_deps()
|> load_external_deps(recursive: false)

:apps_direct ->
deps_app(false)
load_external_deps(acc, recursive: false)

:transitive ->
warning(
"Dialyxir has deprecated plt_add_deps: :transitive in favor of app_tree, which includes only runtime dependencies."
)

deps_transitive() ++ deps_app(true)
acc
|> load_transitive_deps()
|> load_external_deps(recursive: true)

_app_tree ->
deps_app(true)
load_external_deps(acc, recursive: true)
end
end)
end)

loaded_apps
end

defp deps_project do
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))
defp load_project_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps =
Mix.Project.config()[:deps]
|> Enum.filter(&env_dep(&1))
|> Enum.map(&elem(&1, 0))

app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

defp deps_transitive do
Mix.Project.deps_paths()
|> Map.keys()
defp load_transitive_deps({loaded_apps, unloaded_apps, load_statuses}) do
apps = Mix.Project.deps_paths() |> Map.values()
app_load_statuses = Map.new(apps, &{elem(&1, 0), :loaded})

update_load_statuses({loaded_apps, unloaded_apps -- apps, load_statuses}, app_load_statuses)
end

@spec deps_app(boolean()) :: [atom]
defp deps_app(recursive) do
defp load_external_deps({loaded_apps, _unloaded_apps, load_statuses}, opts) do
# Non-recursive traversal of 2 tries to load the app immediate deps.
traversal_depth =
case Keyword.fetch!(opts, :recursive) do
true -> @max_dep_traversal_depth
false -> 2
end

app = Mix.Project.config()[:app]
deps_app(app, recursive)
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
case do_load_app(app) do
:ok ->
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
# Even if already loaded, we'll need to traverse it again to get its deps.
load_statuses_w_app = Map.put(load_statuses, app, {:unloaded, :required})
traverse_deps_for_apps({loaded_apps -- [app], [app], load_statuses_w_app}, traversal_depth)
end

# Identify the optional applications which can't be loaded and thus not available
missing_apps =
Application.spec(app, :optional_applications)
|> List.wrap()
|> Enum.reject(&(do_load_app(&1) == :ok))
defp traverse_deps_for_apps({loaded_apps, [] = unloaded_deps, load_statuses}, _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

# Remove the optional applications which are not available from all the applications
required_apps =
Application.spec(app, :applications)
|> List.wrap()
|> Enum.reject(&(&1 in missing_apps))
defp traverse_deps_for_apps({loaded_apps, unloaded_deps, load_statuses}, 0 = _rem_depth),
do: {loaded_apps, unloaded_deps, load_statuses}

required_apps |> Stream.flat_map(&with_each.(&1)) |> Enum.concat(required_apps)
defp traverse_deps_for_apps({loaded_apps, apps_to_load, load_statuses}, rem_depth) do
initial_acc = {loaded_apps, [], load_statuses}

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses} =
Enum.reduce(apps_to_load, initial_acc, fn app, acc ->
required? = Map.fetch!(load_statuses, app) == {:unloaded, :required}
{app_load_status, app_dep_statuses} = load_app(app, required?)

[]
end
end
else
@spec deps_app(atom(), boolean()) :: [atom()]
defp deps_app(app, recursive) do
with_each =
if recursive do
&deps_app(&1, true)
else
fn _ -> [] end
end
acc
|> update_load_statuses(%{app => app_load_status})
|> update_load_statuses(app_dep_statuses)
end)

case do_load_app(app) do
:ok ->
nil
traverse_deps_for_apps(
{updated_loaded_apps, updated_unloaded_apps, updated_load_statuses},
rem_depth - 1
)
end

{:error, err} ->
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
defp load_app(app, required?) do
case do_load_app(app) do
:ok ->
{dependencies, optional_deps} = app_dep_specs(app)

nil
end
dep_statuses =
Map.new(dependencies, fn dep ->
case dep in optional_deps do
true -> {dep, {:unloaded, :optional}}
false -> {dep, {:unloaded, :required}}
end
end)

case Application.spec(app, :applications) do
[] ->
[]
{:loaded, dep_statuses}

nil ->
[]
{:error, err} ->
if required? do
error("Error loading #{app}, dependency list may be incomplete.\n #{inspect(err)}")
end

this_apps ->
Enum.map(this_apps, with_each)
|> List.flatten()
|> Enum.concat(this_apps)
end
{{:error, err}, %{}}
end
end

Expand All @@ -443,6 +458,58 @@ defmodule Dialyxir.Project do
end
end

if System.version() |> Version.parse!() |> then(&(&1.major >= 1 and &1.minor >= 15)) do
defp app_dep_specs(app) do
# Values returned by :optional_applications are also in :applications.
dependencies = Application.spec(app, :applications) || []
optional_deps = Application.spec(app, :optional_applications) || []

{dependencies, optional_deps}
end
else
defp app_dep_specs(app) do
{Application.spec(app, :applications) || [], []}
end
end

defp update_load_statuses({loaded_apps, unloaded_apps, load_statuses}, new_statuses) do
initial_acc = {loaded_apps, unloaded_apps, load_statuses}

Enum.reduce(new_statuses, initial_acc, fn {app, new_status}, acc ->
{current_loaded_apps, current_unloaded_apps, statuses} = acc
existing_status = Map.get(statuses, app, :unset)

{new_loaded_apps, new_unloaded_apps, updated_load_statuses} =
case {existing_status, new_status} do
{:unset, {:unloaded, _} = new_status} ->
# Haven't seen this app before.
{[], [app], Map.put(statuses, app, new_status)}

{{:unloaded, :optional}, {:unloaded, :required}} ->
# A previous app had this as optional, but another one requires it.
{[], [], Map.put(statuses, app, {:unloaded, :required})}

{{:unloaded, _}, :loaded} ->
# Final state. Dependency successfully loaded.
{[app], [], Map.put(statuses, app, :loaded)}

{{:unloaded, _}, {:error, err}} ->
# Final state. Dependency failed to load.
{[], [], Map.put(statuses, app, {:error, err})}

{_prev_unloaded_or_final, _nwe_unloaded_or_final} ->
# No status change, or one that doesn't matter like final to final.
{[], [], statuses}
end

{
new_loaded_apps ++ current_loaded_apps,
new_unloaded_apps ++ current_unloaded_apps,
updated_load_statuses
}
end)
end

defp env_dep(dep) do
only_envs = dep_only(dep)
only_envs == nil || Mix.env() in List.wrap(only_envs)
Expand All @@ -452,7 +519,7 @@ defmodule Dialyxir.Project do
defp dep_only({_, _, opts}) when is_list(opts), do: opts[:only]
defp dep_only(_), do: nil

@spec reduce_umbrella_children(list(), (list() -> list())) :: list()
@spec reduce_umbrella_children(acc, (acc -> acc)) :: acc when acc: term
defp reduce_umbrella_children(acc, f) do
if Mix.Project.umbrella?() do
children = Mix.Dep.Umbrella.loaded()
Expand Down