diff --git a/CHANGELOG.md b/CHANGELOG.md index 04bd4c9..1e2b955 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,17 @@ See [Conventional Commits](Https://conventionalcommits.org) for commit guideline +## Unreleased + +### Features: + +* add strategy-aware aggregate dispatch with lateral and grouped aggregate implementations + +### Improvements: + +* add grouped query aggregate support for root SQLite-compatible aggregate kinds +* declare `:jason` as a direct dependency for grouped list aggregate defaults + ## [v0.6.3](https://github.com/ash-project/ash_sql/compare/v0.6.2...v0.6.3) (2026-05-03) diff --git a/README.md b/README.md index 38ff703..42e4f4d 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,12 @@ def deps do ] end ``` + +## Aggregate Strategies + +`AshSql.Implementation` defaults aggregate planning to `:lateral`. SQL data +layers can override `aggregate_strategy/1` with `:grouped` when they need the +SQLite-style grouped aggregate implementation. + +The grouped strategy uses native aggregate `FILTER` support and JSON-backed list +aggregate defaults. diff --git a/lib/aggregate.ex b/lib/aggregate.ex index 50243ee..63d0a8f 100644 --- a/lib/aggregate.ex +++ b/lib/aggregate.ex @@ -5,13 +5,7 @@ defmodule AshSql.Aggregate do @moduledoc false - require Ecto.Query - require Ash.Query - import Ecto.Query, only: [from: 2, subquery: 1] - - @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> - Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") - end) + alias AshSql.Aggregate.Context def add_aggregates( query, @@ -22,2685 +16,78 @@ defmodule AshSql.Aggregate do root_data \\ nil ) - def add_aggregates(query, [], _, _, _, _), do: {:ok, query} + def add_aggregates(query, [], _resource, _select?, _source_binding, _root_data), + do: {:ok, query} def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do - case resource_aggregates_to_aggregates(resource, query, aggregates) do - {:ok, aggregates} -> - root_data_path = - case root_data do - {_, path} -> - path - - _ -> - [] - end - - tenant = - case Enum.at(aggregates, 0) do - %{context: %{tenant: tenant}} -> - Ash.ToTenant.to_tenant(tenant, resource) - - _ -> - nil - end - - {query, aggregates} = - Enum.reduce( - aggregates, - {query, []}, - fn aggregate, {query, aggregates} -> - if is_atom(aggregate.name) do - existing_agg = query.__ash_bindings__.aggregate_defs[aggregate.name] - - if existing_agg && different_queries?(existing_agg.query, aggregate.query) do - {query, name} = use_aggregate_name(query, aggregate.name) - {query, [%{aggregate | name: name} | aggregates]} - else - {query, [aggregate | aggregates]} - end - else - {query, name} = use_aggregate_name(query, aggregate.name) - - {query, [%{aggregate | name: name} | aggregates]} - end - end - ) - - {already_computed_aggregates, remaining_aggregates} = - aggregates - |> Enum.uniq_by(& &1.name) - |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, [])) - - query = - if Enum.any?(already_computed_aggregates) && select? do - query.__ash_bindings__.bindings - |> Enum.filter(fn - {_binding, %{type: :aggregate}} -> true - _ -> false - end) - |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> - q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) - - Enum.reduce(aggs, q, fn agg, q -> - if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do - q = - update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> - [agg.name | select_aggs] - end) - - if agg.default_value do - from(row in q, - select_merge: %{ - ^agg.name => - coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) - } - ) - else - from(row in q, - select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} - ) - end - else - q - end - end) - end) - else - query - end - - query = - if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && - !query.__ash_bindings__[:lateral_join?] && - Enum.any?( - remaining_aggregates, - &(not optimizable_first_aggregate?(resource, &1, query)) - ) do - wrap_in_subquery_for_aggregates(query) - else - query - end - - query = - if root_data_path == [] do - query - |> Map.update!(:__ash_bindings__, fn bindings -> - bindings - |> Map.update!(:aggregate_defs, fn aggregate_defs -> - Map.merge(aggregate_defs, Map.new(aggregates, &{&1.name, &1})) - end) - end) - else - query - end - - result = - remaining_aggregates - |> Enum.group_by(fn aggregate -> - expanded_path = - aggregate.resource - |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) - |> Enum.map(& &1.name) - - {expanded_path, aggregate.resource, aggregate.join_filters || %{}, - aggregate.query.action.name} - end) - |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> - {can_group, cant_group} = - Enum.split_with(aggregates, &can_group?(resource, &1, query)) - - [{{path, resource, join_filters, read_action}, can_group}] ++ - Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) - end) - |> Enum.reject(fn - {_, []} -> - true - - _ -> - false - end) - |> Enum.reduce_while( - {:ok, query, []}, - fn {{path, resource, join_filters, read_action}, aggregates}, - {:ok, query, dynamics} -> - related = Ash.Resource.Info.related(resource, path) - read_action = Ash.Resource.Info.action(related, read_action) - - if read_action.modify_query do - raise """ - Data layer does not currently support aggregates over read actions that use `modify_query`. - - Resource: #{inspect(resource)} - Relationship Path: #{inspect(path)} - Action: #{read_action.name} - """ - end - - {first_relationship, relationship_path} = - case path do - [] -> - {nil, []} - - [first_relationship | rest] -> - case Ash.Resource.Info.relationship(resource, first_relationship) do - nil -> - raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" - - first_relationship -> - {first_relationship, rest} - end - end - - hydrated_agg_refs = - aggregates - |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) - |> Ash.Filter.hydrate_refs(%{ - resource: Enum.at(aggregates, 0).query.resource, - parent_stack: - if(first_relationship, do: [first_relationship.source], else: [resource]) - }) - |> elem(1) - - parent_expr = - if first_relationship do - first_relationship.filter - |> Ash.Filter.hydrate_refs(%{ - resource: first_relationship.destination, - parent_stack: [first_relationship.source] - }) - |> elem(1) - |> then(&[&1 | hydrated_agg_refs]) - |> AshSql.Join.parent_expr() - end - - used_aggregates = - Ash.Filter.used_aggregates(parent_expr, []) - - {:ok, query} = - AshSql.Aggregate.add_aggregates( - query, - used_aggregates, - resource, - false, - query.__ash_bindings__.root_binding - ) - - {:ok, query} = - AshSql.Join.join_all_relationships( - query, - parent_expr, - [], - nil, - [], - nil, - true, - nil, - nil, - true - ) - - is_single? = match?([_], aggregates) - - cond do - is_single? && - optimizable_first_aggregate?( - resource, - Enum.at(aggregates, 0), - query - ) -> - case add_first_join_aggregate( - query, - resource, - hd(aggregates), - root_data, - first_relationship, - source_binding - ) do - {:ok, query, dynamic} -> - query = - if select? do - select_or_merge(query, hd(aggregates).name, dynamic) - else - query - end - - {:cont, {:ok, query, dynamics}} - - {:error, error} -> - {:halt, {:error, error}} - end - - is_single? && Enum.at(aggregates, 0).kind == :exists -> - [aggregate] = aggregates - - expr = - if is_nil(Map.get(aggregate.query, :filter)) do - true - else - Map.get(aggregate.query, :filter) - end - - {exists, acc} = - AshSql.Expr.dynamic_expr( - query, - %Ash.Query.Exists{ - path: root_data_path ++ aggregate.relationship_path, - related?: aggregate.related?, - resource: aggregate.query.resource, - expr: expr - }, - query.__ash_bindings__ - ) - - {:cont, - {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), - [{aggregate.load, aggregate.name, exists} | dynamics]}} - - true -> - tmp_query = - if first_relationship && first_relationship.type == :many_to_many do - put_in(query.__ash_bindings__[:lateral_join_bindings], [ - query.__ash_bindings__.current - ]) - |> AshSql.Bindings.explicitly_set_binding( - %{ - type: :left, - path: [first_relationship.join_relationship] - }, - query.__ash_bindings__.current - ) - else - query - end - - start_bindings_at = - if first_relationship && first_relationship.type == :many_to_many do - query.__ash_bindings__.current + 1 - else - query.__ash_bindings__.current - end - - case get_subquery( - resource, - aggregates, - is_single?, - first_relationship, - relationship_path, - tmp_query, - start_bindings_at, - query, - source_binding, - root_data_path, - tenant, - join_filters - ) do - {:error, error} -> - {:error, error} - - {:ok, subquery} -> - query = - join_subquery( - query, - subquery, - first_relationship, - relationship_path, - aggregates, - source_binding, - root_data_path - ) - - if select? do - new_dynamics = - Enum.map( - aggregates, - &{&1.load, &1.name, - select_dynamic( - resource, - query, - &1, - query.__ash_bindings__.current - 1 - )} - ) - - {:cont, {:ok, query, new_dynamics ++ dynamics}} - else - {:cont, {:ok, query, dynamics}} - end - end - end - end - ) - - case result do - {:ok, query, dynamics} -> - if select? do - {:ok, add_aggregate_selects(query, dynamics)} - else - {:ok, query} - end - - {:error, error} -> - {:error, error} - end - - {:error, error} -> - {:error, error} - end - end - - defp already_added?(aggregate, bindings, root_data_path) do - Enum.any?(bindings.bindings, fn - {_, %{type: :aggregate, aggregates: aggregates, path: ^root_data_path}} -> - aggregate.name in Enum.map(aggregates, & &1.name) - - _other -> - false - end) - end - - defp get_subquery( - _resource, - aggregates, - is_single?, - nil, - _relationship_path, - _tmp_query, - start_bindings_at, - query, - _source_binding, - root_data_path, - tenant, - _join_filters - ) do - first_aggregate = Enum.at(aggregates, 0) - aggregate_resource = first_aggregate.query.resource - - first_aggregate.query - |> Ash.Query.set_context(%{ - data_layer: %{ - table: nil, - parent_bindings: - Map.put( - query.__ash_bindings__, - :refs_at_path, - root_data_path - ), - start_bindings_at: start_bindings_at || 0 - } - }) - |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) - |> AshSql.Join.handle_attribute_multitenancy(tenant) - |> AshSql.Join.hydrate_refs(query.__ash_bindings__.context[:private][:actor]) - |> case do - %{valid?: true} = related_query -> - case Ash.Query.data_layer_query(related_query) do - {:ok, ecto_query} -> - {:ok, Ecto.Query.exclude(ecto_query, :select)} - - {:error, error} -> - {:error, error} - end - - %{errors: errors} -> - {:error, errors} - end - |> case do - {:ok, query} -> - maybe_filter_subquery( - query, - nil, - [], - aggregates, - is_single?, - query.__ash_bindings__.root_binding - ) - - {:error, error} -> - {:error, error} - end - |> case do - {:error, error} -> - {:error, error} - - {:ok, query} -> - if is_single? and has_filter?(Enum.at(aggregates, 0).query) do - AshSql.Filter.filter( - query, - Enum.at(aggregates, 0).query.filter, - aggregate_resource - ) - else - {:ok, query} - end - |> case do - {:error, error} -> - {:error, error} - - {:ok, filtered} -> - filtered = - AshSql.Join.set_join_prefix( - filtered, - %{query | prefix: tenant}, - aggregate_resource - ) - - {:ok, - select_all_aggregates( - aggregates, - filtered, - [], - query, - is_single?, - aggregate_resource, - nil - )} - end - end - end - - defp get_subquery( - resource, - aggregates, - is_single?, - first_relationship, - relationship_path, - tmp_query, - start_bindings_at, - query, - source_binding, - root_data_path, - tenant, - join_filters - ) do - AshSql.Join.related_subquery( - first_relationship, - tmp_query, - start_bindings_at: start_bindings_at, - refs_at_path: root_data_path, - skip_distinct_for_first_rel?: true, - on_subquery: fn subquery -> - base_binding = subquery.__ash_bindings__.root_binding - current_binding = subquery.__ash_bindings__.current - - subquery = - subquery - |> Ecto.Query.exclude(:select) - |> Ecto.Query.select(%{}) - - subquery = - apply_relationship_subquery( - subquery, - first_relationship, - query, - tenant, - source_binding, - current_binding, - base_binding - ) - - subquery = - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - first_relationship.destination - ) - - {:ok, subquery, _} = - apply_first_relationship_join_filters( - subquery, - query, - %AshSql.Expr.ExprInfo{}, - first_relationship, - join_filters - ) - - subquery = - set_in_group( - subquery, - query, - resource - ) - - {:ok, joined} = - join_all_relationships( - subquery, - aggregates, - relationship_path, - first_relationship, - is_single?, - join_filters - ) - - {:ok, filtered} = - maybe_filter_subquery( - joined, - first_relationship, - relationship_path, - aggregates, - is_single?, - subquery.__ash_bindings__.root_binding - ) - - select_all_aggregates( - aggregates, - filtered, - relationship_path, - query, - is_single?, - Ash.Resource.Info.related( - first_relationship.destination, - relationship_path - ), - first_relationship - ) - end - ) - end - - defp apply_relationship_subquery( - subquery, - %{manual: {module, opts}} = rel, - query, - tenant, - source_binding, - current_binding, - _base_binding - ) do - field = rel.destination_attribute - - from(row in subquery, - group_by: field(row, ^field), - select_merge: %{^field => field(row, ^field)} - ) - - subquery = - from(row in subquery, distinct: true) - - {:ok, subquery} = - apply( - module, - query.__ash_bindings__.sql_behaviour.manual_relationship_subquery_function(), - [ - opts, - source_binding, - current_binding - 1, - subquery - ] + context = + Context.new!( + query: query, + resource: resource, + select?: select?, + source_binding: source_binding, + root_data: root_data ) - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - rel.destination - ) - end - - defp apply_relationship_subquery( - subquery, - %{no_attributes?: true}, - _query, - _tenant, - _source_binding, - _current_binding, - _base_binding - ) do - subquery - end - - defp apply_relationship_subquery( - subquery, - %{type: :many_to_many} = rel, - query, - tenant, - source_binding, - _current_binding, - _base_binding - ) do - join_relationship_struct = - Ash.Resource.Info.relationship( - rel.source, - rel.join_relationship - ) - - {:ok, through} = - AshSql.Join.related_subquery( - join_relationship_struct, - query - ) - - field = rel.source_attribute_on_join_resource - - subquery = - from(sub in subquery, - join: through in ^through, - as: ^query.__ash_bindings__.current, - on: - field( - through, - ^rel.destination_attribute_on_join_resource - ) == - field(sub, ^rel.destination_attribute), - select_merge: map(through, ^[field]), - group_by: - field( - through, - ^rel.source_attribute_on_join_resource - ), - distinct: - field( - through, - ^rel.source_attribute_on_join_resource - ), - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - through, - ^rel.source_attribute_on_join_resource - ) - ) - - AshSql.Join.set_join_prefix( - subquery, - %{query | prefix: tenant}, - rel.destination - ) - end - - defp apply_relationship_subquery( - subquery, - rel, - _query, - _tenant, - source_binding, - _current_binding, - base_binding - ) do - field = rel.destination_attribute - - from(row in subquery, - group_by: field(row, ^field), - select_merge: %{^field => field(row, ^field)}, - where: - field( - parent_as(^source_binding), - ^rel.source_attribute - ) == - field( - as(^base_binding), - ^rel.destination_attribute - ) - ) - end - - defp set_in_group(%{__ash_bindings__: _} = query, _, _resource) do - Map.update!( - query, - :__ash_bindings__, - &Map.put(&1, :in_group?, true) - ) - end - - defp set_in_group(%Ecto.SubQuery{} = subquery, query, resource) do - subquery = from(row in subquery, []) - - subquery - |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) - |> Map.update!( - :__ash_bindings__, - &Map.put(&1, :in_group?, true) - ) - end - - defp set_in_group(other, query, resource) do - from(row in other, as: ^0) - |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) - |> Map.update!( - :__ash_bindings__, - &Map.put(&1, :in_group?, true) - ) + strategy(context).add_aggregates(context, aggregates) end - defp different_queries?(nil, nil), do: false - defp different_queries?(nil, _), do: true - defp different_queries?(_, nil), do: true - - defp different_queries?(query1, query2) do - query1.filter != query2.filter && query1.sort != query2.sort - end - - @doc false def extract_shared_filters(aggregates) do - aggregates - |> Enum.reduce_while({nil, []}, fn - %{query: %{filter: filter}} = agg, {global_filters, aggs} when not is_nil(filter) -> - and_statements = - AshSql.Expr.split_statements(filter, :and) - - global_filters = - if global_filters do - Enum.filter(global_filters, &(&1 in and_statements)) - else - and_statements - end - - {:cont, {global_filters, [{agg, and_statements} | aggs]}} - - _, _ -> - {:halt, {:error, aggregates}} - end) - |> case do - {:error, aggregates} -> - {:error, aggregates} - - {[], _} -> - {:error, aggregates} - - {nil, _} -> - {:error, aggregates} - - {global_filters, aggregates} -> - global_filter = and_filters(Enum.uniq(global_filters)) - - aggregates = - Enum.map(aggregates, fn {agg, and_statements} -> - applicable_and_statements = - and_statements - |> Enum.reject(&(&1 in global_filters)) - |> and_filters() - - %{agg | query: %{agg.query | filter: applicable_and_statements}} - end) - - {{:ok, global_filter}, aggregates} - end - end - - defp and_filters(filters) do - Enum.reduce(filters, nil, fn expr, acc -> - if is_nil(acc) do - expr - else - Ash.Query.BooleanExpression.new(:and, expr, acc) - end - end) - end - - defp apply_first_relationship_join_filters( - agg_root_query, - query, - acc, - first_relationship, - join_filters - ) do - case join_filters[[first_relationship.name]] do - nil -> - {:ok, agg_root_query, acc} - - filter -> - with {:ok, agg_root_query} <- - AshSql.Join.join_all_relationships(agg_root_query, filter) do - agg_root_query = - AshSql.Expr.set_parent_path( - agg_root_query, - query - ) - - {query, acc} = - AshSql.Join.maybe_apply_filter( - agg_root_query, - agg_root_query, - agg_root_query.__ash_bindings__, - filter - ) - - {:ok, query, acc} - end - end - end - - defp use_aggregate_name(query, aggregate_name) do - {%{ - query - | __ash_bindings__: %{ - query.__ash_bindings__ - | current_aggregate_name: - next_aggregate_name(query.__ash_bindings__.current_aggregate_name), - aggregate_names: - Map.put( - query.__ash_bindings__.aggregate_names, - aggregate_name, - query.__ash_bindings__.current_aggregate_name - ) - } - }, query.__ash_bindings__.current_aggregate_name} - end - - defp resource_aggregates_to_aggregates(resource, query, aggregates) do - private_context = query.__ash_bindings__.context[:private] - - Enum.reduce_while(aggregates, {:ok, []}, fn - %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> - aggregate = - Ash.Actions.Read.add_calc_context( - aggregate, - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - aggregate, {:ok, aggregates} -> - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - read_action = - aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name - - with %{valid?: true} = aggregate_query <- Ash.Query.for_read(related, read_action), - %{valid?: true} = aggregate_query <- - Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do - Ash.Query.Aggregate.new( - resource, - aggregate.name, - aggregate.kind, - path: aggregate.relationship_path, - query: aggregate_query, - field: aggregate.field, - default: aggregate.default, - filterable?: aggregate.filterable?, - type: aggregate.type, - sortable?: aggregate.filterable?, - include_nil?: aggregate.include_nil?, - constraints: aggregate.constraints, - implementation: aggregate.implementation, - uniq?: aggregate.uniq?, - read_action: - aggregate.read_action || - Ash.Resource.Info.primary_action!( - Ash.Resource.Info.related(resource, aggregate.relationship_path), - :read - ).name, - authorize?: aggregate.authorize? - ) - else - %{errors: errors} -> - {:error, errors} - end - |> case do - {:ok, aggregate} -> - aggregate = - aggregate - |> Map.put(:load, aggregate.name) - |> Ash.Actions.Read.add_calc_context( - private_context[:actor], - private_context[:authorize?], - private_context[:tenant], - private_context[:tracer], - query.__ash_bindings__[:domain], - query.__ash_bindings__[:resource], - parent_stack: query.__ash_bindings__[:parent_resources] || [] - ) - - {:cont, {:ok, [aggregate | aggregates]}} - - {:error, error} -> - {:halt, {:error, error}} - end - end) - end - - defp add_first_join_aggregate( - query, - _resource, - %{related?: false} = aggregate, - root_data, - _, - source_binding - ) do - path = - case root_data do - {_resource, path} -> - path - - _ -> - [] - end - - subquery_result = - aggregate.query - |> Ash.Query.set_context(%{ - data_layer: %{ - table: nil, - parent_bindings: - Map.put( - query.__ash_bindings__, - :refs_at_path, - path - ), - start_bindings_at: (query.__ash_bindings__.current || 0) + 1 - } - }) - |> Ash.Query.limit(1) - |> Ash.Query.data_layer_query() - - case subquery_result do - {:ok, ecto_query} -> - ref = - %Ash.Query.Ref{ - attribute: aggregate.field, - resource: aggregate.query.resource - } - - {:ok, ecto_query} = AshSql.Join.join_all_relationships(ecto_query, ref) - - ecto_query = - case aggregate.field do - %Ash.Query.Aggregate{} = aggregate -> - {:ok, ecto_query} = - add_aggregates( - ecto_query, - [aggregate], - aggregate.query.resource, - true, - source_binding, - root_data - ) - - ecto_query - - %Ash.Resource.Aggregate{} = aggregate -> - {:ok, ecto_query} = - add_aggregates( - ecto_query, - [aggregate], - Ash.Resource.Info.related(aggregate.resource, aggregate.relationship_path), - true, - source_binding, - root_data - ) - - ecto_query - - %Ash.Resource.Calculation{ - name: name, - calculation: {module, opts}, - type: type, - constraints: constraints - } -> - {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) - expression = module.expression(opts, new_calc.context) - - expression = - Ash.Expr.fill_template( - expression, - actor: aggregate.context.actor, - tenant: aggregate.query.to_tenant, - args: %{}, - context: aggregate.context - ) - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: ecto_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, ecto_query} = - AshSql.Calculation.add_calculations( - ecto_query, - [{new_calc, expression}], - ecto_query.__ash_bindings__.resource, - source_binding, - true - ) - - ecto_query - - %Ash.Query.Calculation{ - module: module, - opts: opts, - context: context - } = calc -> - expression = module.expression(opts, context) - - expression = - Ash.Expr.fill_template( - expression, - actor: context.actor, - tenant: aggregate.query.to_tenant, - args: context.arguments, - context: context.source_context - ) - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: ecto_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, ecto_query} = - AshSql.Calculation.add_calculations( - ecto_query, - [{calc, expression}], - ecto_query.__ash_bindings__.resource, - source_binding, - true - ) - - ecto_query - - _ -> - ecto_query - end - - ref = - %Ash.Query.Ref{ - attribute: aggregate_field(aggregate, aggregate.query.resource, query), - relationship_path: [], - resource: aggregate.query.resource - } - - value = - Ecto.Query.dynamic(field(as(^query.__ash_bindings__.current), ^ref.attribute.name)) - - AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - query = - if has_parent_expr?(aggregate.query.filter) do - from(row in query, - left_lateral_join: related in subquery(ecto_query), - on: true, - as: ^query.__ash_bindings__.current - ) - else - from(row in query, - left_join: related in subquery(ecto_query), - on: true, - as: ^query.__ash_bindings__.current - ) - end - - query = - AshSql.Bindings.add_binding( - query, - %{ - path: path, - type: :aggregate, - aggregates: [aggregate] - } - ) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - with_default = - if aggregate.default_value do - if type do - type_expr = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^type_expr)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - {:ok, query, casted} - - {:error, error} -> - {:error, error} - end - end - - defp add_first_join_aggregate( - query, - resource, - aggregate, - root_data, - first_relationship, - _source_binding - ) do - {resource, path} = - case root_data do - {resource, path} -> - {resource, path} - - _ -> - {resource, []} - end - - join_filters = - if has_filter?(aggregate) do - %{(path ++ aggregate.relationship_path) => aggregate.query.filter} - else - %{} - end - - case AshSql.Join.join_all_relationships( - query, - nil, - [], - [ - {:left, - AshSql.Join.relationship_path_to_relationships( - resource, - path ++ aggregate.relationship_path - )} - ], - [], - nil, - false, - join_filters - ) do - {:ok, query} -> - ref = - aggregate_field_ref( - aggregate, - Ash.Resource.Info.related(resource, path ++ aggregate.relationship_path), - path ++ aggregate.relationship_path, - query, - first_relationship - ) - - {:ok, query} = AshSql.Join.join_all_relationships(query, ref) - - {value, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - with_default = - if aggregate.default_value do - if type do - type_expr = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^type_expr)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), casted} - - {:error, error} -> - {:error, error} - end - end - - defp maybe_filter_subquery( - agg_query, - first_relationship, - relationship_path, - aggregates, - is_single?, - source_binding - ) do - Enum.reduce_while(aggregates, {:ok, agg_query}, fn aggregate, {:ok, agg_query} -> - filter = - if !Enum.empty?(relationship_path) && aggregate.query.filter do - Ash.Filter.move_to_relationship_path( - aggregate.query.filter, - relationship_path - ) - |> Map.put(:resource, first_relationship.destination) - else - aggregate.query.filter - end - - # For unrelated aggregates (first_relationship is nil), use the aggregate's resource - # For related aggregates, use the relationship destination - related = - if first_relationship do - first_relationship.destination - else - aggregate.query.resource - end - - field = - case aggregate.field do - field when is_atom(field) -> - Ash.Resource.Info.field(related, field) - - field -> - field - end - - root_data = - case first_relationship do - nil -> - nil - - %{destination: destination, name: name} -> - {destination, [name]} - end - - agg_query = - case field do - %Ash.Query.Aggregate{} = aggregate -> - {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) - - agg_query - - %Ash.Resource.Aggregate{} = aggregate -> - {:ok, agg_query} = - add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) - - agg_query - - %Ash.Resource.Calculation{ - name: name, - calculation: {module, opts}, - type: type, - constraints: constraints - } -> - {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) - expression = module.expression(opts, new_calc.context) - - expression = - Ash.Expr.fill_template( - expression, - actor: aggregate.context.actor, - tenant: aggregate.query.to_tenant, - args: %{}, - context: aggregate.context - ) - - expression = - if Enum.empty?(relationship_path) do - expression - else - Ash.Filter.move_to_relationship_path( - expression, - relationship_path - ) - end - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: agg_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, agg_query} = - AshSql.Calculation.add_calculations( - agg_query, - [{new_calc, expression}], - agg_query.__ash_bindings__.resource, - source_binding, - false - ) - - agg_query - - %Ash.Query.Calculation{ - module: module, - opts: opts, - context: context - } = calc -> - expression = module.expression(opts, context) - - expression = - Ash.Expr.fill_template( - expression, - actor: context.actor, - tenant: aggregate.query.to_tenant, - args: context.arguments, - context: context.source_context - ) - - expression = - if Enum.empty?(relationship_path) do - expression - else - Ash.Filter.move_to_relationship_path( - expression, - relationship_path - ) - end - - {:ok, expression} = - Ash.Filter.hydrate_refs(expression, %{ - resource: agg_query.__ash_bindings__.resource, - public?: false - }) - - {:ok, agg_query} = - AshSql.Calculation.add_calculations( - agg_query, - [{calc, expression}], - agg_query.__ash_bindings__.resource, - source_binding, - false - ) - - agg_query - - _ -> - agg_query - end - - if has_filter?(aggregate.query) && is_single? do - {:cont, AshSql.Filter.filter(agg_query, filter, agg_query.__ash_bindings__.resource)} - else - {:cont, {:ok, agg_query}} - end - end) - end - - defp join_subquery( - query, - subquery, - nil, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: sub in subquery(subquery), - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( - query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } - ) - end - - defp join_subquery( - query, - subquery, - %{manual: {_, _}}, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: sub in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( - query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } - ) - end - - defp join_subquery( - query, - subquery, - %{type: :many_to_many}, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: agg in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - query - |> AshSql.Bindings.add_binding(%{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - }) - |> AshSql.Bindings.merge_expr_accumulator(%AshSql.Expr.ExprInfo{}) + AshSql.Aggregate.Lateral.extract_shared_filters(aggregates) end - defp join_subquery( - query, - subquery, - _first_relationship, - _relationship_path, - aggregates, - _source_binding, - root_data_path - ) do - query = - from(row in query, - left_lateral_join: agg in ^subquery, - as: ^query.__ash_bindings__.current, - on: true - ) - - AshSql.Bindings.add_binding( - query, - %{ - path: root_data_path, - type: :aggregate, - aggregates: aggregates - } - ) + def next_aggregate_name(index) do + AshSql.Aggregate.Lateral.next_aggregate_name(index) end - def next_aggregate_name(i) do - @next_aggregate_names[i] || - raise Ash.Error.Framework.AssumptionFailed, - message: """ - All 1000 static names for aggregates have been used in a single query. - Congratulations, this means that you have gone so wildly beyond our imagination - of how much can fit into a single quer. Please file an issue and we will raise the limit. - """ - end - - defp select_all_aggregates( - aggregates, - joined, - relationship_path, - _query, - is_single?, - resource, - first_relationship - ) do - Enum.reduce(aggregates, joined, fn aggregate, joined -> - add_subquery_aggregate_select( - joined, - relationship_path, - aggregate, - resource, - is_single?, - first_relationship - ) - end) - end - - defp join_all_relationships( - agg_root_query, - _aggregates, - relationship_path, - first_relationship, - _is_single?, - join_filters - ) do - if Enum.empty?(relationship_path) do - {:ok, agg_root_query} - else - join_filters = - Enum.reduce(join_filters, %{}, fn {key, value}, acc -> - if List.starts_with?(key, [first_relationship.name]) do - Map.put(acc, Enum.drop(key, 1), value) - else - acc - end - end) - - AshSql.Join.join_all_relationships( - agg_root_query, - Map.values(join_filters), - [], - [ - {:inner, - AshSql.Join.relationship_path_to_relationships( - first_relationship.destination, - relationship_path - )} - ], - [], - nil, - false, - join_filters, - agg_root_query - ) - end - end - - @doc false - def can_group?(_, %{kind: :exists}, _), do: false - def can_group?(_, %{kind: :list}, _), do: false - def can_group?(resource, aggregate, query) do - can_group_kind?(aggregate, resource, query) && !has_exists?(aggregate) && - !references_to_many_relationships?(aggregate) && - !optimizable_first_aggregate?(resource, aggregate, query) && - !has_parent_expr?(aggregate.query.filter) - end - - defp has_parent_expr?(filter, depth \\ 0) do - not is_nil( - Ash.Filter.find( - filter, - fn - %Ash.Query.Call{name: :parent, args: [expr]} -> - if depth == 0 do - true - else - has_parent_expr?(expr, depth - 1) - end - - %Ash.Query.Exists{expr: expr} -> - has_parent_expr?(expr, depth + 1) - - %Ash.Query.Parent{expr: expr} -> - if depth == 0 do - true - else - has_parent_expr?(expr, depth - 1) - end - - %Ash.Query.Ref{ - attribute: %Ash.Query.Aggregate{ - field: %Ash.Query.Calculation{module: module, opts: opts, context: context} - } - } -> - if module.has_expression?() do - module.expression(opts, context) - |> has_parent_expr?(depth + 1) - else - false - end - - _other -> - false - end, - true, - true, - true - ) - ) - end - - # We can potentially optimize this. We don't have to prevent aggregates that reference - # relationships from joining, we can - # 1. group up the ones that do join relationships by the relationships they join - # 2. potentially group them all up that join to relationships and just join to all the relationships - # but this method is predictable and easy so we're starting by just not grouping them - defp references_to_many_relationships?(aggregate) do - if aggregate.query do - aggregate.query.filter - |> Ash.Filter.relationship_paths() - |> Enum.any?(&to_many_path?(aggregate.query.resource, &1)) - else - false - end - end - - defp to_many_path?(_resource, []), do: false - - defp to_many_path?(resource, [rel | rest]) do - case Ash.Resource.Info.relationship(resource, rel) do - %{cardinality: :many} -> - true - - nil -> - raise """ - No such relationship #{inspect(resource)}.#{rel} - """ - - rel -> - to_many_path?(rel.destination, rest) - end - end - - defp can_group_kind?(aggregate, resource, query) do - if aggregate.kind == :first do - if array_type?(resource, aggregate) || - optimizable_first_aggregate?(resource, aggregate, query) do - false - else - true - end - else - true - end - end - - @doc false - def optimizable_first_aggregate?( - resource, - %{ - kind: :first, - relationship_path: relationship_path, - join_filters: join_filters, - field: %Ash.Query.Calculation{} = field - }, - _ - ) do - ref = - %Ash.Query.Ref{ - attribute: field, - relationship_path: relationship_path, - resource: resource - } - - with true <- join_filters == %{}, - [] <- Ash.Filter.used_aggregates(ref, :all), - [] <- Ash.Filter.relationship_paths(ref) do - true - else - _ -> - false - end - end - - def optimizable_first_aggregate?( - _resource, - %{ - kind: :first, - field: %Ash.Query.Aggregate{} - }, - _ - ) do - false - end - - def optimizable_first_aggregate?( - resource, - %{ - name: name, - kind: :first, - relationship_path: relationship_path, - join_filters: join_filters, - query: %{resource: related}, - field: field - } = aggregate, - query - ) do - related - |> Ash.Resource.Info.field(field) - |> case do - %Ash.Resource.Aggregate{} -> - false - - %Ash.Resource.Calculation{} -> - field = aggregate_field(aggregate, resource, query) - - ref = - %Ash.Query.Ref{ - attribute: field, - relationship_path: relationship_path, - resource: resource - } - - with [] <- Ash.Filter.used_aggregates(ref, :all), - [] <- Ash.Filter.relationship_paths(ref) do - true - else - _ -> - false - end - - nil -> - false - - _ -> - name in query.__ash_bindings__.sql_behaviour.simple_join_first_aggregates(resource) || - (join_filters in [nil, %{}, []] && - single_path?(resource, relationship_path)) - end - end - - def optimizable_first_aggregate?(_, _, _), do: false - - defp array_type?(resource, aggregate) do - related = Ash.Resource.Info.related(resource, aggregate.relationship_path) - - case aggregate.field do - nil -> - false - - %{type: {:array, _}} -> - true - - type when is_atom(type) -> - case Ash.Resource.Info.field(related, aggregate.field).type do - {:array, _} -> - true - - _ -> - false - end - - _ -> - false - end + AshSql.Aggregate.Lateral.can_group?(resource, aggregate, query) end - defp has_exists?(aggregate) do - !!Ash.Filter.find(aggregate.query && aggregate.query.filter, fn - %Ash.Query.Exists{} -> true - _ -> false - end) + def optimizable_first_aggregate?(resource, aggregate, query) do + AshSql.Aggregate.Lateral.optimizable_first_aggregate?(resource, aggregate, query) end - defp add_aggregate_selects(query, dynamics) do - {in_aggregates, in_body} = - Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) - - aggs = - in_body - |> Map.new(fn {load, _, dynamic} -> - {load, dynamic} - end) - - aggs = - if Enum.empty?(in_aggregates) do - aggs - else - Map.put( - aggs, - :aggregates, - Map.new(in_aggregates, fn {_, name, dynamic} -> - {name, dynamic} - end) - ) - end - - Ecto.Query.select_merge(query, ^aggs) - end - - defp select_dynamic(_resource, query, aggregate, binding) do - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - field = - if type do - field_ref = Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) - query.__ash_bindings__.sql_behaviour.type_expr(field_ref, type) - else - Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) - end - - coalesced = - if is_nil(aggregate.default_value) do - field - else - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic( - coalesce( - ^field, - ^typed_default - ) - ) - else - Ecto.Query.dynamic( - coalesce( - ^field, - ^aggregate.default_value - ) - ) - end - end - - if type do - query.__ash_bindings__.sql_behaviour.type_expr(coalesced, type) - else - coalesced - end - end - - defp has_filter?(nil), do: false - defp has_filter?(%{filter: nil}), do: false - defp has_filter?(%{filter: %Ash.Filter{expression: nil}}), do: false - defp has_filter?(_), do: true - - defp has_sort?(nil), do: false - defp has_sort?(%{sort: nil}), do: false - defp has_sort?(%{sort: []}), do: false - defp has_sort?(%{sort: _}), do: true - defp has_sort?(_), do: false - def add_subquery_aggregate_select( query, relationship_path, - %{kind: :first} = aggregate, - resource, - is_single?, - first_relationship - ) do - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - {field, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - has_sort? = has_sort?(aggregate.query) - - array_agg = - if aggregate.include_nil? do - # any_value() ignores NULLs by design in PostgreSQL, so we must use - # array_agg with [1] indexing when include_nil? is true - "array_agg" - else - query.__ash_bindings__.sql_behaviour.list_aggregate(aggregate.query.resource) - end - - {sorted, include_nil_filter_field, query} = - if has_sort? || first_relationship.sort not in [nil, []] do - {sort, binding} = - if has_sort? do - {aggregate.query.sort, binding} - else - {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} - end - - {:ok, sort_expr, query} = - AshSql.Sort.sort( - query, - sort, - Ash.Resource.Info.related( - query.__ash_bindings__.resource, - relationship_path - ), - relationship_path, - binding, - :return - ) - - if aggregate.include_nil? do - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - ["#{array_agg}(? ORDER BY #{question_marks})", field] ++ sort_expr - ) - - {sort_expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {sort_expr, nil, query} - else - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - {expr, include_nil_filter_field} = - if has_filter?(aggregate.query) and !is_single? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "#{array_agg}(? ORDER BY #{question_marks})", - field - ] ++ - sort_expr - ) - - {expr, field} - else - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "#{array_agg}(? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, nil} - end - - {sort_expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {sort_expr, include_nil_filter_field, query} - end - else - case array_agg do - "array_agg" -> - {Ecto.Query.dynamic( - [row], - fragment("array_agg(?)", ^field) - ), nil, query} - - "any_value" -> - {Ecto.Query.dynamic( - [row], - fragment("any_value(?)", ^field) - ), nil, query} - end - end - - {query, filtered} = - filter_field( - sorted, - include_nil_filter_field, - query, aggregate, - relationship_path, - is_single? - ) - - value = - if array_agg == "array_agg" do - Ecto.Query.dynamic(fragment("(?)[1]", ^filtered)) - else - filtered - end - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^value, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) - end - else - value - end - - casted = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - query = AshSql.Bindings.merge_expr_accumulator(query, acc) - - select_or_merge( - query, - aggregate.name, - casted - ) - end - - def add_subquery_aggregate_select( - query, - relationship_path, - %{kind: :list} = aggregate, resource, is_single?, first_relationship ) do - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - {field, acc} = - AshSql.Expr.dynamic_expr( - query, - ref, - Map.put(query.__ash_bindings__, :location, :aggregate), - false - ) - - related = - Ash.Resource.Info.related( - query.__ash_bindings__.resource, - relationship_path - ) - - has_sort? = has_sort?(aggregate.query) - - {sorted, include_nil_filter_field, query} = - if has_sort? || (first_relationship && first_relationship.sort not in [nil, []]) do - {sort, binding} = - if has_sort? do - {aggregate.query.sort, binding} - else - {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} - end - - {:ok, sort_expr, query} = - AshSql.Sort.sort( - query, - sort, - related, - relationship_path, - binding, - :return - ) - - question_marks = Enum.map(sort_expr, fn _ -> " ? " end) - - distinct = - if Map.get(aggregate, :uniq?) do - "DISTINCT " - else - "" - end - - {expr, include_nil_filter_field} = - if aggregate.include_nil? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - ["array_agg(#{distinct}? ORDER BY #{question_marks})", field] ++ sort_expr - ) - - {expr, nil} - else - if has_filter?(aggregate.query) and !is_single? do - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "array_agg(#{distinct}? ORDER BY #{question_marks})", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, field} - else - {:ok, expr} = - Ash.Query.Function.Fragment.casted_new( - [ - "array_agg(#{distinct}? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", - field - ] ++ - sort_expr ++ [field] - ) - - {expr, nil} - end - end - - {expr, acc} = - AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) - - query = - AshSql.Bindings.merge_expr_accumulator(query, acc) - - {expr, include_nil_filter_field, query} - else - if Map.get(aggregate, :uniq?) do - {Ecto.Query.dynamic( - [row], - fragment("array_agg(DISTINCT ?)", ^field) - ), nil, query} - else - {Ecto.Query.dynamic( - [row], - fragment("array_agg(?)", ^field) - ), nil, query} - end - end - - {query, filtered} = - filter_field( - sorted, - include_nil_filter_field, - query, - aggregate, - relationship_path, - is_single? - ) - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) - end - else - filtered - end - - cast = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - query = AshSql.Bindings.merge_expr_accumulator(query, acc) - - select_or_merge( + AshSql.Aggregate.Lateral.add_subquery_aggregate_select( query, - aggregate.name, - cast + relationship_path, + aggregate, + resource, + is_single?, + first_relationship ) end - def add_subquery_aggregate_select( - query, - relationship_path, - %{kind: kind} = aggregate, - resource, - is_single?, - first_relationship - ) - when kind in [:count, :sum, :avg, :max, :min, :custom] do - ref = - aggregate_field_ref( - aggregate, - resource, - relationship_path, - query, - first_relationship - ) - - {field, query} = - case kind do - :custom -> - # we won't use this if its custom so don't try to make one - {nil, query} - - :count -> - if aggregate.field do - {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} - else - {nil, query} - end - - _ -> - {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) - - {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} - end - - type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - aggregate.type, - aggregate.constraints, - :aggregate - ) - - binding = - AshSql.Bindings.get_binding( - query.__ash_bindings__.resource, - relationship_path, - query, - [:left, :inner, :root] - ) - - field = - case kind do - :count -> - cond do - !aggregate.field -> - Ecto.Query.dynamic([row], count()) - - Map.get(aggregate, :uniq?) -> - Ecto.Query.dynamic([row], count(^field, :distinct)) - - match?(%{attribute: %{allow_nil?: false}}, ref) -> - Ecto.Query.dynamic([row], count()) - - true -> - Ecto.Query.dynamic([row], count(^field)) - end - - :sum -> - Ecto.Query.dynamic([row], sum(^field)) - - :avg -> - Ecto.Query.dynamic([row], avg(^field)) - - :max -> - Ecto.Query.dynamic([row], max(^field)) - - :min -> - Ecto.Query.dynamic([row], min(^field)) - - :custom -> - {module, opts} = aggregate.implementation - - module.dynamic(opts, binding) - end - - {query, filtered} = filter_field(field, nil, query, aggregate, relationship_path, is_single?) - - with_default = - if aggregate.default_value do - if type do - typed_default = - query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) - - Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) - else - Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) - end - else - filtered - end - - cast = - if type do - query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) - else - with_default - end - - select_or_merge(query, aggregate.name, cast) - end - - defp filter_field(field, include_nil_filter_field, query, _aggregate, _relationship_path, true) do - if include_nil_filter_field do - {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} - else - {query, field} - end - end - - defp filter_field( - field, - include_nil_filter_field, - query, - aggregate, - relationship_path, - _is_single? - ) do - if has_filter?(aggregate.query) do - filter = - Ash.Filter.move_to_relationship_path( - aggregate.query.filter, - relationship_path - ) - - used_aggregates = Ash.Filter.used_aggregates(filter, []) - - # here we bypass an inner join. - # Really, we should check if all aggs in a group - # could do the same inner join, then do an inner join - {:ok, query} = - AshSql.Join.join_all_relationships( - query, - filter, - [], - nil, - [], - nil, - true, - nil, - nil, - true - ) - - {:ok, query} = - add_aggregates( - query, - used_aggregates, - query.__ash_bindings__.resource, - false, - query.__ash_bindings__.root_binding - ) - - {expr, acc} = - AshSql.Expr.dynamic_expr( - query, - filter, - query.__ash_bindings__, - false, - {aggregate.type, aggregate.constraints} - ) - - if include_nil_filter_field do - {AshSql.Bindings.merge_expr_accumulator(query, acc), - Ecto.Query.dynamic(filter(^field, ^expr and not is_nil(^include_nil_filter_field)))} - else - {AshSql.Bindings.merge_expr_accumulator(query, acc), - Ecto.Query.dynamic(filter(^field, ^expr))} - end - else - if include_nil_filter_field do - {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} - else - {query, field} - end - end - end - - defp select_or_merge(query, aggregate_name, casted) do - query = - if query.select do - query - else - Ecto.Query.select(query, %{}) - end - - Ecto.Query.select_merge(query, ^%{aggregate_name => casted}) - end - def aggregate_field_ref(aggregate, resource, relationship_path, query, first_relationship) do - if aggregate.kind == :count && !aggregate.field do - nil - else - %Ash.Query.Ref{ - attribute: aggregate_field(aggregate, resource, query), - relationship_path: relationship_path, - resource: query.__ash_bindings__.resource - } - |> case do - %{attribute: %Ash.Resource.Aggregate{}} = ref when not is_nil(first_relationship) -> - if first_relationship do - %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} - else - ref - end - - %{attribute: %Ash.Query.Aggregate{}} = ref when not is_nil(first_relationship) -> - if first_relationship do - %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} - else - ref - end - - other -> - other - end - end - end - - defp single_path?(_, []), do: true - - defp single_path?(resource, [relationship | rest]) do - relationship = Ash.Resource.Info.relationship(resource, relationship) - - !Map.get(relationship, :from_many?) && - (relationship.type == :belongs_to || - has_one_with_identity?(relationship)) && - single_path?(relationship.destination, rest) - end - - defp has_one_with_identity?(%{type: :has_one, from_many?: false} = relationship) do - Ash.Resource.Info.primary_key(relationship.destination) == [ - relationship.destination_attribute - ] || - relationship.destination - |> Ash.Resource.Info.identities() - |> Enum.any?(fn %{keys: keys} -> - keys == [relationship.destination_attribute] - end) + AshSql.Aggregate.Lateral.aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) end - defp has_one_with_identity?(_), do: false - - @doc false def aggregate_field(aggregate, resource, query) do - if is_atom(aggregate.field) do - case Ash.Resource.Info.field( - resource, - aggregate.field || List.first(Ash.Resource.Info.primary_key(resource)) - ) do - %Ash.Resource.Calculation{calculation: {module, opts}} = calculation -> - calc_type = - AshSql.Expr.parameterized_type( - query.__ash_bindings__.sql_behaviour, - calculation.type, - Map.get(calculation, :constraints, []), - :calculation - ) - - AshSql.Expr.validate_type!(query, calc_type, "#{inspect(calculation.name)}") - - {:ok, query_calc} = - Ash.Query.Calculation.new( - calculation.name, - module, - opts, - calculation.type, - calculation.constraints - ) - - Ash.Actions.Read.add_calc_context( - query_calc, - aggregate.context.actor, - aggregate.context.authorize?, - aggregate.context.tenant, - aggregate.context.tracer, - query.__ash_bindings__[:domain], - aggregate.query.resource, - parent_stack: [ - query.__ash_bindings__.resource | query.__ash_bindings__[:parent_resources] || [] - ] - ) - - nil -> - raise "no such aggregate field: #{inspect(resource)}.#{aggregate.field}" - - other -> - other - end - else - aggregate.field - end + AshSql.Aggregate.Lateral.aggregate_field(aggregate, resource, query) end def wrap_in_subquery_for_aggregates(query) do - resource = query.__ash_bindings__.resource - selected_by_default = Ash.Resource.Info.selected_by_default_attribute_names(resource) - - selected_fields = - query.__ash_bindings__[:select] || - extract_selected_fields(query, resource, selected_by_default) - - all_attr_names = - resource - |> Ash.Resource.Info.attribute_names() - |> MapSet.to_list() - - to_select = - Enum.reject(all_attr_names, &(&1 in selected_fields)) - - query_with_all_attrs = - case query.select do - %Ecto.Query.SelectExpr{expr: {:merge, _, [l, {:%{}, [], kw}]}} -> - put_in( - query.select.expr, - {:merge, [], - [ - l, - {:%{}, [], - kw ++ - Enum.map( - to_select, - &{&1, - {{:., [], [{:&, [], [query.__ash_bindings__.root_binding]}, &1]}, [], []}} - )} - ]} - ) - - _ -> - from(row in query, - select_merge: struct(row, ^to_select) - ) - end - - # Flatten nested calculations/aggregates maps before creating subquery - # Ecto doesn't allow nested maps in subquery select expressions - {calculations_require_rewrite, aggregates_require_rewrite, query_with_all_attrs} = - AshSql.Query.rewrite_nested_selects(query_with_all_attrs) - - # After flattening, we need to: - # 1. Use the updated select_calculations from the rewritten query (which excludes :calculations) - # 2. Add the flattened calculation/aggregate field names to the select - flattened_calc_fields = Map.keys(calculations_require_rewrite) - flattened_agg_fields = Map.keys(aggregates_require_rewrite) - - # Get select_calculations from the rewritten query (it has :calculations removed) - select_calculations = - (query_with_all_attrs.__ash_bindings__[:select_calculations] || []) -- [:calculations] - - select_aggregates = - (query_with_all_attrs.__ash_bindings__[:select_aggregates] || []) -- [:aggregates] - - subquery_query = - from(row in subquery(query_with_all_attrs), - as: ^query.__ash_bindings__.root_binding, - select: - struct( - row, - ^Enum.concat([ - selected_fields, - select_calculations, - select_aggregates, - flattened_calc_fields, - flattened_agg_fields - ]) - ) - ) - - root_binding = query.__ash_bindings__.root_binding - - only_root_binding = %{ - root_binding => query.__ash_bindings__.bindings[root_binding] - } - - new_bindings = - query.__ash_bindings__ - |> Map.put(:bindings, only_root_binding) - |> Map.delete(:__order__?) - |> Map.update( - :calculations_require_rewrite, - calculations_require_rewrite, - &Map.merge(&1, calculations_require_rewrite) - ) - |> Map.update( - :aggregates_require_rewrite, - aggregates_require_rewrite, - &Map.merge(&1, aggregates_require_rewrite) - ) - - Map.put(subquery_query, :__ash_bindings__, new_bindings) - end - - defp extract_selected_fields( - %{select: %Ecto.Query.SelectExpr{expr: expr, take: take}}, - resource, - all_attribute_names - ) do - Enum.uniq(extract_fields_from_expr(expr, resource, take, all_attribute_names)) + AshSql.Aggregate.Lateral.wrap_in_subquery_for_aggregates(query) end - defp extract_fields_from_expr(expr, resource, take, all_attribute_names) do - case expr do - {:&, [], [ix]} -> - case take do - %{^ix => {:struct, fields}} when is_list(fields) -> - fields - - %{^ix => {:map, fields}} when is_list(fields) -> - fields - - take when take == %{} -> - all_attribute_names - - _ -> - [] - end - - {:%{}, [], fields} -> - Enum.map(fields, fn {field_name, _} -> field_name end) - - {:%, [], [_struct, {:%{}, [], fields}]} -> - Enum.map(fields, fn {field_name, _} -> field_name end) - - {:merge, _, [sel1, sel2]} -> - Enum.concat( - extract_fields_from_expr(sel1, resource, take, all_attribute_names), - extract_fields_from_expr(sel2, resource, take, all_attribute_names) - ) - - _other -> - all_attribute_names + defp strategy(%Context{} = context) do + case context.sql_behaviour.aggregate_strategy(context.resource) do + :lateral -> AshSql.Aggregate.Lateral + :grouped -> AshSql.Aggregate.Grouped end end end diff --git a/lib/aggregate/context.ex b/lib/aggregate/context.ex new file mode 100644 index 0000000..e45d4af --- /dev/null +++ b/lib/aggregate/context.ex @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Context do + @moduledoc false + + defstruct [ + :query, + :resource, + :select?, + :source_binding, + :root_binding, + :root_data, + :root_data_path, + :aggregate_path, + :sql_behaviour, + :tenant + ] + + def new!(opts) do + query = Keyword.fetch!(opts, :query) + resource = Keyword.fetch!(opts, :resource) + root_data = Keyword.get(opts, :root_data) + + %__MODULE__{ + query: query, + resource: resource, + select?: Keyword.fetch!(opts, :select?), + source_binding: Keyword.fetch!(opts, :source_binding), + root_binding: query.__ash_bindings__.root_binding, + root_data: root_data, + root_data_path: root_data_path(root_data), + aggregate_path: Keyword.get(opts, :aggregate_path, []), + sql_behaviour: query.__ash_bindings__.sql_behaviour, + tenant: Keyword.get(opts, :tenant) + } + end + + defp root_data_path({_, path}), do: path + defp root_data_path(_), do: [] +end diff --git a/lib/aggregate/grouped.ex b/lib/aggregate/grouped.ex new file mode 100644 index 0000000..dfbc1a0 --- /dev/null +++ b/lib/aggregate/grouped.ex @@ -0,0 +1,1641 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Grouped do + @moduledoc false + + import Ecto.Query, only: [from: 2] + + @scalar_aggregate_kinds [:count, :sum, :avg, :max, :min, :exists] + @window_aggregate_kinds [:first, :list] + @supported_aggregate_kinds @scalar_aggregate_kinds ++ @window_aggregate_kinds ++ [:custom] + @window_value_field :__ash_sql_grouped_aggregate_value__ + @window_row_number_field :__ash_sql_grouped_aggregate_row_number__ + @window_count_field :__ash_sql_grouped_aggregate_count__ + @unrelated_join_field :__ash_sql_grouped_unrelated_join__ + + def add_aggregates(%AshSql.Aggregate.Context{} = context, aggregates) do + add_aggregates(context.query, aggregates, context.resource, select?: context.select?) + end + + def add_aggregates(query, aggregates, resource, opts \\ []) do + select? = Keyword.get(opts, :select?, true) + + do_add_aggregates(query, aggregates, resource, select?) + end + + def add_sort_aggregates(query, sort, _resource) when sort in [nil, []], do: {:ok, query} + + def add_sort_aggregates(query, sort, resource) do + with {:ok, aggregates} <- aggregates_from_sort(query, sort, resource) do + add_aggregates(query, aggregates, resource, select?: false) + end + end + + def relationship_filter_uses_parent?(%{filter: nil}), do: false + + def relationship_filter_uses_parent?(%{filter: filter}) do + filter_uses_parent?(filter) + end + + defp do_add_aggregates(query, [], _resource, _select?), do: {:ok, query} + + defp do_add_aggregates(query, aggregates, resource, select?) do + primary_key = Ash.Resource.Info.primary_key(resource) + + cond do + primary_key == [] -> + {:error, "AshSql cannot load aggregates on resources with no primary key"} + + Enum.any?(aggregates, &(not supported?(&1))) -> + {:error, + "AshSql only supports loading related count, sum, avg, min, max, exists, first, list and custom aggregates"} + + true -> + {already_added, remaining} = + aggregates + |> Enum.uniq_by(& &1.name) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__)) + + already_added_dynamics = + if select? do + Enum.map(already_added, &existing_aggregate_dynamic(&1, query.__ash_bindings__)) + else + [] + end + + remaining + |> Enum.group_by(&aggregate_group_key/1) + |> Enum.reduce_while({:ok, query, already_added_dynamics}, fn {relationship_path, + aggregates}, + {:ok, query, dynamics} -> + case add_aggregate_group( + query, + resource, + aggregate_relationship_path(relationship_path), + aggregates + ) do + {:ok, query, new_dynamics} -> + {:cont, {:ok, query, new_dynamics ++ dynamics}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + |> case do + {:ok, query, dynamics} -> + if select? do + {:ok, select_aggregates(query, dynamics)} + else + {:ok, query} + end + + {:error, error} -> + {:error, error} + end + end + end + + defp supported?(%{name: name}) when not is_atom(name), do: false + + defp supported?(%{kind: kind, related?: false}) when kind in @supported_aggregate_kinds do + true + end + + defp supported?(%{kind: kind, related?: related?, relationship_path: path}) + when kind in @supported_aggregate_kinds do + related? != false && match?([_ | _], path) + end + + defp supported?(_), do: false + + defp aggregate_group_key(aggregate) do + read_action = (aggregate.query.action && aggregate.query.action.name) || aggregate.read_action + + relationship_key = + case aggregate do + %{related?: false, query: %{resource: resource}} -> {:unrelated, resource} + %{relationship_path: relationship_path} -> {:related, relationship_path} + end + + {relationship_key, read_action, aggregate.join_filters || %{}, + aggregate_filter_group_key(aggregate), aggregate_kind_group_key(aggregate)} + end + + defp aggregate_relationship_path( + {{:related, relationship_path}, _read_action, _join_filters, _aggregate_filter_group, + _kind_group} + ) do + relationship_path + end + + defp aggregate_relationship_path( + {{:unrelated, _resource}, _read_action, _join_filters, _aggregate_filter_group, + _kind_group} + ) do + [] + end + + defp aggregate_kind_group_key(%{kind: kind, name: name}) when kind in @window_aggregate_kinds do + {kind, name} + end + + defp aggregate_kind_group_key(_aggregate), do: :shared + + defp aggregate_filter_group_key(aggregate) do + if aggregate_filter_uses_relationships?(aggregate) do + {:filter, aggregate.name} + else + :shared + end + end + + defp already_added?(aggregate, bindings) do + Enum.any?(bindings.bindings, fn + {_binding, %{type: :aggregate, aggregates: aggregates}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _binding -> + false + end) + end + + defp existing_aggregate_dynamic(aggregate, bindings) do + {binding, _aggregate_binding} = + Enum.find(bindings.bindings, fn + {_binding, %{type: :aggregate, aggregates: aggregates}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _binding -> + false + end) + + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, bindings.sql_behaviour)} + end + + defp aggregates_from_sort(query, sort, resource) do + sort + |> List.wrap() + |> Enum.reduce_while({:ok, []}, fn sort, {:ok, aggregates} -> + case sort_aggregates(query, sort, resource) do + {:ok, new_aggregates} -> + {:cont, {:ok, new_aggregates ++ aggregates}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + |> case do + {:ok, aggregates} -> {:ok, Enum.uniq(aggregates)} + {:error, error} -> {:error, error} + end + end + + defp sort_aggregates(query, {sort, _order}, resource) do + sort_key_aggregates(query, sort, resource) + end + + defp sort_aggregates(query, sort, resource) do + sort_key_aggregates(query, sort, resource) + end + + defp sort_key_aggregates(_query, %Ash.Query.Aggregate{} = aggregate, _resource) do + {:ok, [aggregate]} + end + + defp sort_key_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do + calculation_aggregates(query, calculation, resource) + end + + defp sort_key_aggregates(query, sort, resource) when is_atom(sort) do + case Ash.Resource.Info.field(resource, sort) do + %Ash.Resource.Aggregate{} = aggregate -> + query_aggregate(resource, aggregate) + + %Ash.Resource.Calculation{} = calculation -> + calculation_aggregates(query, calculation, resource) + + _ -> + {:ok, []} + end + end + + defp sort_key_aggregates(_query, _sort, _resource), do: {:ok, []} + + defp calculation_aggregates(query, %Ash.Resource.Calculation{} = calculation, resource) do + {module, opts} = calculation.calculation + + with {:ok, calculation} <- + Ash.Query.Calculation.new( + calculation.name, + module, + opts, + calculation.type, + calculation.constraints + ) do + calculation = + Ash.Actions.Read.add_calc_context( + calculation, + query.__ash_bindings__.context[:private][:actor], + query.__ash_bindings__.context[:private][:authorize?], + query.__ash_bindings__.context[:private][:tenant], + query.__ash_bindings__.context[:private][:tracer], + query.__ash_bindings__.context[:private][:domain], + query.__ash_bindings__.context[:private][:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + calculation_aggregates(query, calculation, resource) + end + end + + defp calculation_aggregates(query, %Ash.Query.Calculation{} = calculation, resource) do + calculation.opts + |> calculation.module.expression(calculation.context) + |> Ash.Filter.hydrate_refs(%{ + resource: resource, + aggregates: %{}, + parent_stack: query.__ash_bindings__[:parent_resources] || [], + calculations: %{}, + public?: false + }) + |> case do + {:ok, expression} -> + {:ok, Ash.Filter.used_aggregates(expression)} + + {:error, error} -> + {:error, error} + end + end + + defp query_aggregate(resource, aggregate) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + read_action = + aggregate.read_action || + Ash.Resource.Info.primary_action!(related, :read).name + + with %{valid?: true} = aggregate_query <- Ash.Query.for_read(related, read_action), + %{valid?: true} = aggregate_query <- + Ash.Query.build(aggregate_query, + filter: aggregate.filter, + sort: aggregate.sort + ), + {:ok, aggregate} <- + Ash.Query.Aggregate.new( + resource, + aggregate.name, + aggregate.kind, + path: aggregate.relationship_path, + query: aggregate_query, + field: aggregate.field, + default: aggregate.default, + filterable?: aggregate.filterable?, + type: aggregate.type, + sortable?: aggregate.sortable?, + include_nil?: aggregate.include_nil?, + constraints: aggregate.constraints, + implementation: aggregate.implementation, + uniq?: aggregate.uniq?, + read_action: read_action, + authorize?: aggregate.authorize?, + join_filters: aggregate.join_filters + ) do + {:ok, [aggregate]} + else + %{errors: errors} -> + {:error, errors} + + {:error, error} -> + {:error, error} + end + end + + defp add_aggregate_group(query, _resource, [], aggregates) do + if Enum.all?(aggregates, &(&1.related? == false)) do + do_add_unrelated_aggregate_group(query, aggregates) + else + {:error, "AshSql only supports loading unrelated aggregates with no relationship path"} + end + end + + defp add_aggregate_group(query, resource, relationship_path, aggregates) do + with {:ok, relationships} <- relationships(resource, relationship_path), + :ok <- validate_relationships(resource, relationship_path, relationships, aggregates) do + do_add_aggregate_group(query, relationships, aggregates) + end + end + + defp relationships(resource, relationship_path) do + relationship_path + |> Enum.reduce_while({:ok, resource, []}, fn relationship_name, {:ok, resource, acc} -> + case Ash.Resource.Info.relationship(resource, relationship_name) do + nil -> + {:halt, {:error, "No such relationship #{inspect(resource)}.#{relationship_name}"}} + + relationship -> + {:cont, {:ok, relationship.destination, [relationship | acc]}} + end + end) + |> case do + {:ok, _resource, relationships} -> {:ok, Enum.reverse(relationships)} + {:error, error} -> {:error, error} + end + end + + defp validate_relationships(resource, relationship_path, relationships, aggregates) do + cond do + Enum.any?(relationships, &match?(%{manual: {_, _}}, &1)) -> + {:error, "AshSql does not support loading aggregates over manual relationships"} + + Enum.any?(relationships, &Map.get(&1, :no_attributes?, false)) -> + {:error, "AshSql does not support loading aggregates over no_attributes? relationships"} + + Enum.any?(relationships, &relationship_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates over relationships with parent-dependent filters"} + + Enum.any?(relationships, &join_relationship_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates over many_to_many relationships with parent-dependent join filters"} + + unsupported_multi_hop_many_to_many?(relationships, aggregates) -> + {:error, + "AshSql does not support loading aggregates over multi-hop paths that include many_to_many relationships"} + + Enum.empty?(relationships) -> + {:error, + "AshSql only supports loading aggregates over a relationship path from #{inspect(resource)}, got: #{inspect(relationship_path)}"} + + true -> + :ok + end + end + + defp unsupported_multi_hop_many_to_many?(relationships, aggregates) do + length(relationships) > 1 && + Enum.any?(relationships, &(&1.type == :many_to_many)) && + !supported_multi_hop_many_to_many?(relationships, aggregates) + end + + defp supported_multi_hop_many_to_many?(relationships, aggregates) do + List.last(relationships).type == :many_to_many && + Enum.count(relationships, &(&1.type == :many_to_many)) == 1 && + Enum.all?(aggregates, &(&1.kind in @scalar_aggregate_kinds)) + end + + defp do_add_unrelated_aggregate_group(query, aggregates) do + binding = query.__ash_bindings__.current + + with :ok <- validate_aggregate_filters(aggregates), + {:ok, aggregate_query} <- unrelated_aggregate_query(query, aggregates, binding) do + aggregate_query = Ecto.Query.subquery(aggregate_query) + + query = + from(_row in query, + left_join: aggregate in ^aggregate_query, + as: ^binding, + on: true + ) + + query = + AshSql.Bindings.add_binding(query, %{ + type: :aggregate, + path: [], + aggregates: aggregates + }) + + dynamics = + Enum.map(aggregates, fn aggregate -> + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, query.__ash_bindings__.sql_behaviour)} + end) + + {:ok, query, dynamics} + end + end + + defp do_add_aggregate_group(query, [first_relationship | _] = relationships, aggregates) do + binding = query.__ash_bindings__.current + + with :ok <- validate_aggregate_filters(aggregates), + {:ok, aggregate_query} <- + aggregate_query(query, relationships, aggregates, binding) do + aggregate_query = Ecto.Query.subquery(aggregate_query) + root_binding = query.__ash_bindings__.root_binding + + query = + from(_row in query, + left_join: aggregate in ^aggregate_query, + as: ^binding, + on: + field(as(^root_binding), ^first_relationship.source_attribute) == + field(aggregate, ^aggregate_join_attribute(first_relationship)) + ) + + query = + AshSql.Bindings.add_binding(query, %{ + type: :aggregate, + path: [], + aggregates: aggregates + }) + + dynamics = + Enum.map(aggregates, fn aggregate -> + {aggregate.load, aggregate.name, + loaded_aggregate_dynamic(aggregate, binding, query.__ash_bindings__.sql_behaviour)} + end) + + {:ok, query, dynamics} + end + end + + defp aggregate_query(parent_query, [relationship], [%{kind: kind} = aggregate], binding) + when kind in @window_aggregate_kinds do + case relationship do + %{type: :many_to_many} -> + many_to_many_window_aggregate_query(parent_query, relationship, aggregate, binding) + + relationship -> + related_window_aggregate_query(parent_query, relationship, aggregate, binding) + end + end + + defp aggregate_query( + parent_query, + [_ | _] = relationships, + [%{kind: kind} = aggregate], + binding + ) + when kind in @window_aggregate_kinds do + multi_hop_window_aggregate_query(parent_query, relationships, aggregate, binding) + end + + defp aggregate_query(parent_query, [relationship], aggregates, binding) do + case relationship do + %{type: :many_to_many} -> + many_to_many_aggregate_query(parent_query, relationship, aggregates, binding) + + relationship -> + related_aggregate_query(parent_query, relationship, aggregates, binding) + end + end + + defp aggregate_query(parent_query, relationships, aggregates, binding) do + case List.last(relationships) do + %{type: :many_to_many} -> + multi_hop_many_to_many_aggregate_query(parent_query, relationships, aggregates, binding) + + _relationship -> + multi_hop_aggregate_query(parent_query, relationships, aggregates, binding) + end + end + + defp unrelated_aggregate_query(parent_query, [%{kind: kind} = aggregate], binding) + when kind in @window_aggregate_kinds do + unrelated_window_aggregate_query(parent_query, aggregate, binding) + end + + defp unrelated_aggregate_query(parent_query, aggregates, binding) do + with {:ok, query} <- unrelated_query(parent_query, hd(aggregates), binding, filter?: false) do + root_binding = query.__ash_bindings__.root_binding + relationship = %{destination: hd(aggregates).query.resource} + + query = from(row in query, select: %{}) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp unrelated_window_aggregate_query(parent_query, aggregate, binding) do + with {:ok, query} <- unrelated_query(parent_query, aggregate, binding, filter?: true) do + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + @unrelated_join_field, + nil, + root_binding, + %{sort: []} + ) + end + end + + defp related_window_aggregate_query(parent_query, relationship, aggregate, binding) do + with {:ok, query} <- + related_window_query(parent_query, relationship, aggregate, binding, [ + relationship.name + ]) do + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + relationship.destination_attribute, + root_binding, + root_binding, + relationship + ) + end + end + + defp many_to_many_window_aggregate_query(parent_query, relationship, aggregate, binding) do + with {:ok, query} <- + related_window_query(parent_query, relationship, aggregate, binding, [ + relationship.name + ]) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- through_query(parent_query, relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^relationship.destination_attribute) + ) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: relationship + }) + + window_aggregate_query( + query, + aggregate, + relationship.source_attribute_on_join_resource, + through_binding, + root_binding, + relationship + ) + end + end + end + + defp multi_hop_window_aggregate_query(parent_query, relationships, aggregate, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_window_query( + parent_query, + final_relationship, + aggregate, + binding, + relationship_path + ), + {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, aggregate) do + first_relationship = hd(relationships) + root_binding = query.__ash_bindings__.root_binding + + window_aggregate_query( + query, + aggregate, + first_relationship.destination_attribute, + first_related_binding, + root_binding, + final_relationship + ) + end + end + + defp related_aggregate_query(parent_query, relationship, aggregates, binding) do + with {:ok, query} <- + related_query(parent_query, relationship, hd(aggregates), binding, [relationship.name]) do + root_binding = query.__ash_bindings__.root_binding + + query = + from(row in query, + group_by: field(as(^root_binding), ^relationship.destination_attribute), + select: %{ + ^relationship.destination_attribute => + field(as(^root_binding), ^relationship.destination_attribute) + } + ) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp many_to_many_aggregate_query(parent_query, relationship, aggregates, binding) do + with {:ok, query} <- + related_query(parent_query, relationship, hd(aggregates), binding, [relationship.name]) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- through_query(parent_query, relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + ash_bindings = query.__ash_bindings__ + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^relationship.destination_attribute), + group_by: field(through, ^relationship.source_attribute_on_join_resource), + select: %{ + ^relationship.source_attribute_on_join_resource => + field(through, ^relationship.source_attribute_on_join_resource) + } + ) + |> Map.put(:__ash_bindings__, ash_bindings) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: relationship + }) + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + end + + defp multi_hop_many_to_many_aggregate_query(parent_query, relationships, aggregates, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_query( + parent_query, + final_relationship, + hd(aggregates), + binding, + relationship_path + ) do + through_binding = query.__ash_bindings__.current + + with {:ok, through_query} <- + through_query(parent_query, final_relationship, through_binding) do + root_binding = query.__ash_bindings__.root_binding + through_query = Ecto.Query.subquery(through_query) + + query = + from(row in query, + join: through in ^through_query, + as: ^through_binding, + on: + field(through, ^final_relationship.destination_attribute_on_join_resource) == + field(as(^root_binding), ^final_relationship.destination_attribute) + ) + |> AshSql.Bindings.add_binding(%{ + type: :through, + relationship: final_relationship + }) + + with {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, hd(aggregates), + current_binding: through_binding + ) do + first_relationship = hd(relationships) + + query = + from(row in query, + group_by: + field(as(^first_related_binding), ^first_relationship.destination_attribute), + select: %{ + ^first_relationship.destination_attribute => + field(as(^first_related_binding), ^first_relationship.destination_attribute) + } + ) + + root_binding = query.__ash_bindings__.root_binding + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, final_relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + end + end + + defp multi_hop_aggregate_query(parent_query, relationships, aggregates, binding) do + final_relationship = List.last(relationships) + relationship_path = Enum.map(relationships, & &1.name) + + with {:ok, query} <- + related_query( + parent_query, + final_relationship, + hd(aggregates), + binding, + relationship_path + ), + {:ok, query, first_related_binding} <- + join_intermediate_relationships(parent_query, query, relationships, hd(aggregates)) do + first_relationship = hd(relationships) + + query = + from(row in query, + group_by: field(as(^first_related_binding), ^first_relationship.destination_attribute), + select: %{ + ^first_relationship.destination_attribute => + field(as(^first_related_binding), ^first_relationship.destination_attribute) + } + ) + + root_binding = query.__ash_bindings__.root_binding + + Enum.reduce_while(aggregates, {:ok, query}, fn aggregate, {:ok, query} -> + case aggregate_dynamic(query, final_relationship, aggregate, root_binding) do + {:ok, query, dynamic} -> + {:cont, {:ok, Ecto.Query.select_merge(query, ^%{aggregate.name => dynamic})}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + end + + defp join_intermediate_relationships(parent_query, query, relationships, aggregate, opts \\ []) do + current_binding = Keyword.get(opts, :current_binding, query.__ash_bindings__.root_binding) + + relationships + |> Enum.zip(tl(relationships)) + |> Enum.with_index() + |> Enum.reverse() + |> Enum.reduce_while( + {:ok, query, current_binding, query.__ash_bindings__.current, nil}, + fn {{relationship, next_relationship}, index}, + {:ok, query, current_binding, next_binding, _first_related_binding} -> + path = + relationships + |> Enum.take(index + 1) + |> Enum.map(& &1.name) + + case intermediate_query(parent_query, relationship, next_binding, aggregate, path) do + {:ok, related_query} -> + related_query = Ecto.Query.subquery(related_query) + + on = intermediate_join_on(next_relationship, next_binding, current_binding) + + query = + from(row in query, + join: related in ^related_query, + as: ^next_binding, + on: ^on + ) + + {:cont, {:ok, query, next_binding, next_binding + 1, next_binding}} + + {:error, error} -> + {:halt, {:error, error}} + end + end + ) + |> case do + {:ok, query, _current_binding, _next_binding, first_related_binding} + when not is_nil(first_related_binding) -> + {:ok, query, first_related_binding} + + {:ok, _query, _current_binding, _next_binding, nil} -> + {:error, "AshSql could not build multi-hop aggregate joins"} + + {:error, error} -> + {:error, error} + end + end + + defp intermediate_join_on( + %{type: :many_to_many} = next_relationship, + related_binding, + current_binding + ) do + Ecto.Query.dynamic( + field(as(^related_binding), ^next_relationship.source_attribute) == + field(as(^current_binding), ^next_relationship.source_attribute_on_join_resource) + ) + end + + defp intermediate_join_on(next_relationship, related_binding, current_binding) do + Ecto.Query.dynamic( + field(as(^related_binding), ^next_relationship.source_attribute) == + field(as(^current_binding), ^next_relationship.destination_attribute) + ) + end + + defp related_query(parent_query, relationship, aggregate, binding, relationship_path) do + aggregate.query + |> Ash.Query.unset([:filter, :sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp related_window_query(parent_query, relationship, aggregate, binding, relationship_path) do + aggregate.query + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp unrelated_query(parent_query, aggregate, binding, opts) do + unset = + if Keyword.fetch!(opts, :filter?) do + [:sort, :distinct, :select, :limit, :offset] + else + [:filter, :sort, :distinct, :select, :limit, :offset] + end + + aggregate.query + |> Ash.Query.unset(unset) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp intermediate_query(parent_query, relationship, binding, aggregate, relationship_path) do + read_action = + relationship.read_action || + Ash.Resource.Info.primary_action!(relationship.destination, :read).name + + relationship.destination + |> Ash.Query.for_read(read_action) + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> Ash.Query.set_context(relationship.context) + |> Ash.Query.do_filter(relationship.filter, parent_stack: [relationship.source]) + |> Ash.Query.do_filter(join_filter(aggregate, relationship_path)) + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp through_query(parent_query, relationship, binding) do + join_relationship = + Ash.Resource.Info.relationship(relationship.source, relationship.join_relationship) + + relationship.through + |> Ash.Query.new() + |> Ash.Query.set_context(%{ + data_layer: %{ + start_bindings_at: binding, + parent_bindings: parent_query.__ash_bindings__ + } + }) + |> Ash.Query.set_context(join_relationship.context) + |> Ash.Query.do_filter(join_relationship.filter) + |> Ash.Query.data_layer_query(run_return_query?: false) + |> case do + {:ok, query} -> + {:ok, + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by)} + + {:error, error} -> + {:error, error} + end + end + + defp aggregate_join_attribute(%{type: :many_to_many} = relationship) do + relationship.source_attribute_on_join_resource + end + + defp aggregate_join_attribute(relationship), do: relationship.destination_attribute + + defp join_relationship_filter_uses_parent?(%{type: :many_to_many} = relationship) do + relationship.source + |> Ash.Resource.Info.relationship(relationship.join_relationship) + |> relationship_filter_uses_parent?() + end + + defp join_relationship_filter_uses_parent?(_relationship), do: false + + defp join_filter(%{join_filters: join_filters}, relationship_path) + when is_map(join_filters) do + Map.get(join_filters, relationship_path) + end + + defp join_filter(_aggregate, _relationship_path), do: nil + + defp validate_aggregate_filters(aggregates) do + cond do + Enum.any?(aggregates, &aggregate_filter_uses_parent?/1) -> + {:error, + "AshSql does not support loading aggregates with parent-dependent aggregate filters"} + + Enum.any?(aggregates, &aggregate_filter_uses_parent_dependent_relationship?/1) -> + {:error, + "AshSql does not support loading aggregates with filters that reference relationships with parent-dependent filters"} + + Enum.any?(aggregates, &aggregate_filter_uses_aggregates?/1) -> + {:error, + "AshSql does not support loading aggregates with aggregate filters that reference other aggregates"} + + Enum.any?(aggregates, &unsupported_to_many_aggregate_filter?/1) -> + {:error, + "AshSql does not support loading sum, avg, list, custom, or field-based count aggregates with filters that reference to-many relationships"} + + Enum.any?(aggregates, &join_filters_use_parent?/1) -> + {:error, "AshSql does not support loading aggregates with parent-dependent join filters"} + + true -> + :ok + end + end + + defp aggregate_filter_uses_parent?(%{query: %{filter: filter}}) do + filter_uses_parent?(filter) + end + + defp aggregate_filter_uses_parent_dependent_relationship?(%{ + query: %{filter: filter, resource: resource} + }) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?(&parent_dependent_relationship_path?(resource, &1)) + end + + defp aggregate_filter_uses_parent_dependent_relationship?(_aggregate), do: false + + defp aggregate_filter_uses_relationships?(%{query: %{filter: filter}}) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?() + end + + defp aggregate_filter_uses_relationships?(_aggregate), do: false + + defp aggregate_filter_uses_aggregates?(%{query: %{filter: filter}}) when not is_nil(filter) do + filter + |> Ash.Filter.used_aggregates([]) + |> Enum.any?() + end + + defp aggregate_filter_uses_aggregates?(_aggregate), do: false + + defp unsupported_to_many_aggregate_filter?(%{kind: :count, field: field} = aggregate) + when not is_nil(field) do + aggregate_filter_references_to_many_relationship?(aggregate) && !aggregate.uniq? + end + + defp unsupported_to_many_aggregate_filter?(%{kind: kind} = aggregate) + when kind in [:sum, :avg, :list, :custom] do + aggregate_filter_references_to_many_relationship?(aggregate) + end + + defp unsupported_to_many_aggregate_filter?(_aggregate), do: false + + defp aggregate_filter_references_to_many_relationship?(%{ + query: %{filter: filter, resource: resource} + }) do + filter + |> aggregate_filter_relationship_paths() + |> Enum.any?(&to_many_relationship_path?(resource, &1)) + end + + defp aggregate_filter_references_to_many_relationship?(_aggregate), do: false + + defp aggregate_filter_relationship_paths(nil), do: [] + + defp aggregate_filter_relationship_paths(%{expression: nil}), do: [] + + defp aggregate_filter_relationship_paths(filter) do + Ash.Filter.relationship_paths(filter) + end + + defp parent_dependent_relationship_path?(_resource, []), do: false + + defp parent_dependent_relationship_path?(resource, [relationship_name | rest]) do + case Ash.Resource.Info.relationship(resource, relationship_name) do + nil -> + false + + relationship -> + relationship_filter_uses_parent?(relationship) || + parent_dependent_relationship_path?(relationship.destination, rest) + end + end + + defp to_many_relationship_path?(_resource, []), do: false + + defp to_many_relationship_path?(resource, [relationship_name | rest]) do + case Ash.Resource.Info.relationship(resource, relationship_name) do + %{cardinality: :many} -> + true + + nil -> + false + + relationship -> + to_many_relationship_path?(relationship.destination, rest) + end + end + + defp join_filters_use_parent?(%{join_filters: join_filters}) when is_map(join_filters) do + Enum.any?(join_filters, fn {_path, filter} -> filter_uses_parent?(filter) end) + end + + defp join_filters_use_parent?(_aggregate), do: false + + defp filter_uses_parent?(nil), do: false + + defp filter_uses_parent?(%{expression: nil}), do: false + + defp filter_uses_parent?(filter) do + Ash.Filter.find( + filter, + fn + %Ash.Query.Parent{} -> true + %Ash.Query.Call{name: :parent} -> true + _ -> false + end, + true, + true, + true + ) + |> case do + nil -> false + _ -> true + end + end + + defp window_aggregate_query( + query, + aggregate, + join_attribute, + partition_binding, + value_binding, + relationship + ) do + with :ok <- validate_window_aggregate(aggregate), + {:ok, sort} <- window_aggregate_sort(aggregate, relationship), + :ok <- validate_window_aggregate_sort(aggregate, sort) do + sql_behaviour = query.__ash_bindings__.sql_behaviour + + query = + query + |> maybe_filter_window_nil_values(aggregate, value_binding) + |> window_source_query(aggregate, join_attribute, partition_binding, value_binding, sort) + |> Ecto.Query.subquery() + |> window_result_query(aggregate, join_attribute, sort, sql_behaviour) + + {:ok, query} + end + end + + defp validate_window_aggregate(%{field: field, kind: kind}) + when kind in @window_aggregate_kinds and is_atom(field) and not is_nil(field) do + :ok + end + + defp validate_window_aggregate(%{name: name, field: field}) do + {:error, + "AshSql cannot load first or list aggregate #{inspect(name)} with field #{inspect(field)}"} + end + + defp validate_window_aggregate_sort(%{kind: :list, uniq?: true, field: field}, sort) do + if Enum.all?(sort, fn {sort_field, _order} -> sort_field == field end) do + :ok + else + {:error, + "AshSql only supports uniq list aggregates when sorting by the list aggregate field"} + end + end + + defp validate_window_aggregate_sort(_aggregate, _sort), do: :ok + + defp maybe_filter_window_nil_values(query, %{include_nil?: true}, _binding), do: query + + defp maybe_filter_window_nil_values(query, aggregate, binding) do + from(row in query, where: not is_nil(field(as(^binding), ^aggregate.field))) + end + + defp window_source_query( + query, + aggregate, + join_attribute, + partition_binding, + value_binding, + sort + ) do + sort_selects = + sort + |> Enum.with_index() + |> Map.new(fn {{field, _order}, index} -> + {window_sort_field(index), Ecto.Query.dynamic(field(as(^value_binding), ^field))} + end) + + select = + Map.merge( + %{ + join_attribute => window_join_field(partition_binding, join_attribute), + @window_value_field => Ecto.Query.dynamic(field(as(^value_binding), ^aggregate.field)) + }, + sort_selects + ) + + query = + if aggregate.kind == :list && aggregate.uniq? do + # This relies on validate_window_aggregate_sort/2 requiring uniq lists to + # sort by the listed field, so distinct applies to {parent, value}. + from(row in query, distinct: true) + else + query + end + + from(row in query, select: ^select) + end + + defp window_result_query(source_query, aggregate, join_attribute, sort, sql_behaviour) do + order_by = + sort + |> Enum.with_index() + |> Enum.map(fn {{_field, order}, index} -> + {ecto_sort_order(order), Ecto.Query.dynamic([row], field(row, ^window_sort_field(index)))} + end) + + partition_by = Ecto.Query.dynamic([row], field(row, ^join_attribute)) + aggregate_value = window_aggregate_value(sql_behaviour, aggregate) + + query = + from(row in source_query, + windows: [ + ash_sql_grouped_aggregate_window: [ + partition_by: ^partition_by, + order_by: ^order_by + ], + ash_sql_grouped_aggregate_partition_window: [ + partition_by: ^partition_by + ] + ], + select: %{ + ^join_attribute => field(row, ^join_attribute), + @window_row_number_field => over(row_number(), :ash_sql_grouped_aggregate_window), + @window_count_field => over(count(), :ash_sql_grouped_aggregate_partition_window) + } + ) + |> Ecto.Query.select_merge(^%{aggregate.name => aggregate_value}) + + row_filter = window_row_filter(aggregate) + + from(row in Ecto.Query.subquery(query), + where: ^row_filter, + select: %{ + ^join_attribute => field(row, ^join_attribute), + ^aggregate.name => field(row, ^aggregate.name) + } + ) + end + + defp window_row_filter(%{kind: :list}) do + row_number_field = @window_row_number_field + count_field = @window_count_field + + Ecto.Query.dynamic( + [row], + field(row, ^row_number_field) == field(row, ^count_field) + ) + end + + defp window_row_filter(_aggregate) do + row_number_field = @window_row_number_field + + Ecto.Query.dynamic([row], field(row, ^row_number_field) == 1) + end + + defp window_aggregate_value(sql_behaviour, %{kind: :first, type: type}) do + value_field = @window_value_field + + value = + Ecto.Query.dynamic( + [row], + over(first_value(field(row, ^value_field)), :ash_sql_grouped_aggregate_window) + ) + + maybe_type_dynamic(sql_behaviour, value, type) + end + + defp window_aggregate_value(sql_behaviour, %{kind: :list, include_nil?: true, type: type}) do + value_field = @window_value_field + + value = + Ecto.Query.dynamic( + [row], + over( + fragment("json_group_array(?)", field(row, ^value_field)), + :ash_sql_grouped_aggregate_window + ) + ) + + maybe_type_dynamic(sql_behaviour, value, type) + end + + defp window_aggregate_value(sql_behaviour, %{kind: :list, type: type}) do + value_field = @window_value_field + + value = + Ecto.Query.dynamic( + [row], + over( + fragment( + "json_group_array(?) FILTER (WHERE ? IS NOT NULL)", + field(row, ^value_field), + field(row, ^value_field) + ), + :ash_sql_grouped_aggregate_window + ) + ) + + maybe_type_dynamic(sql_behaviour, value, type) + end + + defp maybe_type_dynamic(_sql_behaviour, dynamic, nil), do: dynamic + + defp maybe_type_dynamic(sql_behaviour, dynamic, type) do + case sqlite_aggregate_type(sql_behaviour, type) do + nil -> dynamic + type -> sql_behaviour.type_expr(dynamic, type) + end + end + + defp sqlite_aggregate_type(sql_behaviour, type) do + sql_behaviour.parameterized_type(type, []) + end + + defp window_aggregate_sort(%{query: %{sort: sort}} = aggregate, relationship) do + sort = + cond do + sort not in [nil, []] -> + List.wrap(sort) + + relationship.sort not in [nil, []] -> + List.wrap(relationship.sort) + + true -> + [{aggregate.field, :asc}] + end + + sort + |> Enum.reduce_while({:ok, []}, fn + {field, order}, {:ok, acc} when is_atom(field) and is_atom(order) -> + {:cont, {:ok, [{field, order} | acc]}} + + field, {:ok, acc} when is_atom(field) -> + {:cont, {:ok, [{field, :asc} | acc]}} + + sort, _acc -> + {:halt, + {:error, + "AshSql only supports first and list aggregate sorting by related fields, got: #{inspect(sort)}"}} + end) + |> case do + {:ok, sort} -> {:ok, Enum.reverse(sort)} + {:error, error} -> {:error, error} + end + end + + defp window_sort_field(index) do + :"__ash_sql_grouped_aggregate_sort_#{index}__" + end + + defp window_join_field(nil, _join_attribute) do + Ecto.Query.dynamic(fragment("1")) + end + + defp window_join_field(partition_binding, join_attribute) do + Ecto.Query.dynamic(field(as(^partition_binding), ^join_attribute)) + end + + defp ecto_sort_order(:asc), do: :asc + defp ecto_sort_order(:desc), do: :desc + defp ecto_sort_order(:asc_nils_first), do: :asc_nulls_first + defp ecto_sort_order(:asc_nils_last), do: :asc_nulls_last + defp ecto_sort_order(:desc_nils_first), do: :desc_nulls_first + defp ecto_sort_order(:desc_nils_last), do: :desc_nulls_last + defp ecto_sort_order(other), do: other + + defp aggregate_dynamic(query, relationship, %{kind: :exists} = aggregate, binding) do + count_dynamic = count_dynamic(relationship, aggregate, binding) + + with {:ok, query, count_dynamic} <- + maybe_filter_aggregate(query, aggregate, count_dynamic) do + {:ok, query, Ecto.Query.dynamic(^count_dynamic > 0)} + end + end + + defp aggregate_dynamic(query, relationship, %{kind: :count} = aggregate, binding) do + dynamic = count_dynamic(relationship, aggregate, binding) + + with {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(query, _relationship, aggregate, binding) + when aggregate.kind in [:sum, :avg, :max, :min] and is_atom(aggregate.field) do + field = Ecto.Query.dynamic(field(as(^binding), ^aggregate.field)) + + dynamic = + case aggregate.kind do + :sum -> Ecto.Query.dynamic(sum(^field)) + :avg -> Ecto.Query.dynamic(avg(^field)) + :max -> Ecto.Query.dynamic(max(^field)) + :min -> Ecto.Query.dynamic(min(^field)) + end + + with {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(query, _relationship, %{kind: :custom} = aggregate, binding) do + {module, opts} = aggregate.implementation + dynamic = module.dynamic(opts, binding) + + with {:ok, query, dynamic} <- maybe_filter_aggregate(query, aggregate, dynamic) do + {:ok, query, maybe_default_aggregate(query, dynamic, aggregate)} + end + end + + defp aggregate_dynamic(_query, _relationship, aggregate, _binding) do + {:error, + "AshSql cannot load aggregate #{inspect(aggregate.name)} with field #{inspect(aggregate.field)}"} + end + + defp count_dynamic(relationship, %{field: nil} = aggregate, binding) do + if count_distinct?(aggregate) do + count_field = count_field(relationship, aggregate) + + Ecto.Query.dynamic(count(field(as(^binding), ^count_field), :distinct)) + else + Ecto.Query.dynamic(count()) + end + end + + defp count_dynamic(relationship, aggregate, binding) do + count_field = count_field(relationship, aggregate) + + if count_distinct?(aggregate) do + Ecto.Query.dynamic(count(field(as(^binding), ^count_field), :distinct)) + else + Ecto.Query.dynamic(count(field(as(^binding), ^count_field))) + end + end + + defp count_field(_relationship, %{field: field}) when is_atom(field) and not is_nil(field) do + field + end + + defp count_field(relationship, _aggregate) do + relationship.destination + |> Ash.Resource.Info.primary_key() + |> List.first() + |> case do + nil -> relationship.destination_attribute + field -> field + end + end + + defp count_distinct?(%{uniq?: true}), do: true + + defp count_distinct?(%{field: nil} = aggregate) do + aggregate_filter_references_to_many_relationship?(aggregate) + end + + defp count_distinct?(_aggregate), do: false + + defp maybe_filter_aggregate(query, aggregate, dynamic) do + case aggregate.query.filter do + nil -> + {:ok, query, dynamic} + + %{expression: nil} -> + {:ok, query, dynamic} + + filter -> + with {:ok, query} <- + AshSql.Join.join_all_relationships( + query, + filter, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) do + {filter_dynamic, acc} = + AshSql.Expr.dynamic_expr( + query, + filter, + Map.put(query.__ash_bindings__, :location, :aggregate), + false + ) + + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^dynamic, ^filter_dynamic))} + end + end + end + + defp maybe_default_aggregate(query, dynamic, %{kind: :list, default_value: nil, type: type}) + when not is_nil(type) do + sql_behaviour = query.__ash_bindings__.sql_behaviour + + case sqlite_aggregate_type(sql_behaviour, type) do + nil -> + dynamic + + type -> + default = list_default_expr([], type, sql_behaviour) + + Ecto.Query.dynamic(coalesce(^dynamic, ^default)) + |> sql_behaviour.type_expr(type) + end + end + + defp maybe_default_aggregate(_query, dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_aggregate(_query, dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp loaded_aggregate_dynamic( + %{kind: :exists, default_value: nil} = aggregate, + binding, + _sql_behaviour + ) do + aggregate + |> loaded_aggregate_field(binding) + |> then(&Ecto.Query.dynamic(coalesce(^&1, false))) + end + + defp loaded_aggregate_dynamic(%{kind: :list} = aggregate, binding, sql_behaviour) do + type = sqlite_aggregate_type(sql_behaviour, aggregate.type) + default_value = aggregate.default_value || [] + + aggregate + |> loaded_aggregate_field(binding) + |> then(fn field -> + if type do + default = list_default_expr(default_value, type, sql_behaviour) + + Ecto.Query.dynamic(coalesce(^field, ^default)) + |> sql_behaviour.type_expr(type) + else + Ecto.Query.dynamic(coalesce(^field, ^default_value)) + end + end) + end + + defp loaded_aggregate_dynamic(aggregate, binding, _sql_behaviour) do + aggregate + |> loaded_aggregate_field(binding) + |> maybe_default_loaded_aggregate(aggregate) + end + + defp maybe_default_loaded_aggregate(dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_loaded_aggregate(dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp list_default_expr(default_value, type, sql_behaviour) when is_list(default_value) do + default_value = Jason.encode!(default_value) + + Ecto.Query.dynamic(^default_value) + |> sql_behaviour.type_expr(type) + end + + defp list_default_expr(default_value, type, sql_behaviour) do + Ecto.Query.dynamic(^default_value) + |> sql_behaviour.type_expr(type) + end + + defp loaded_aggregate_field(aggregate, binding) do + Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + end + + defp select_aggregates(query, dynamics) do + {in_aggregates, in_body} = + Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) + + aggregates = + in_body + |> Map.new(fn {load, _name, dynamic} -> {load, dynamic} end) + + aggregates = + if Enum.empty?(in_aggregates) do + aggregates + else + Map.put( + aggregates, + :aggregates, + Map.new(in_aggregates, fn {_load, name, dynamic} -> {name, dynamic} end) + ) + end + + query = + if query.select do + query + else + from(row in query, select: %{}) + end + + Ecto.Query.select_merge(query, ^aggregates) + end +end diff --git a/lib/aggregate/grouped/query.ex b/lib/aggregate/grouped/query.ex new file mode 100644 index 0000000..bd11197 --- /dev/null +++ b/lib/aggregate/grouped/query.ex @@ -0,0 +1,206 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Grouped.Query do + @moduledoc false + + import Ecto.Query, only: [from: 2, subquery: 1] + + @supported_kinds [:count, :first, :sum, :max, :min, :avg, :exists] + + def run_aggregate_query(original_query, aggregates, resource, implementation) do + aggregates + |> Enum.reduce_while({:ok, %{}}, fn aggregate, {:ok, acc} -> + case run_single_aggregate(original_query, aggregate, resource, implementation) do + {:ok, value} -> {:cont, {:ok, Map.put(acc, aggregate.name, value)}} + {:error, error} -> {:halt, {:error, error}} + end + end) + end + + defp run_single_aggregate(_original_query, %{kind: kind}, _resource, _implementation) + when kind not in @supported_kinds do + {:error, "AshSql grouped query aggregates do not support #{inspect(kind)} aggregates"} + end + + defp run_single_aggregate( + _original_query, + %{relationship_path: [_ | _]} = aggregate, + _resource, + _implementation + ) do + {:error, + "AshSql grouped query aggregates do not yet support relationship aggregate #{inspect(aggregate.name)}"} + end + + defp run_single_aggregate( + original_query, + %{kind: :exists} = aggregate, + resource, + implementation + ) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource) do + repo = AshSql.dynamic_repo(resource, implementation, query) + {:ok, repo.exists?(query, AshSql.repo_opts(repo, implementation, nil, nil, resource))} + end + end + + defp run_single_aggregate(original_query, %{kind: :first} = aggregate, resource, implementation) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource), + {:ok, field} <- aggregate_field(aggregate) do + query = + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> maybe_sort_first(aggregate) + |> Ecto.Query.limit(1) + |> Ecto.Query.select([row], field(row, ^field)) + + repo = AshSql.dynamic_repo(resource, implementation, query) + {:ok, repo.one(query, AshSql.repo_opts(repo, implementation, nil, nil, resource))} + end + end + + defp run_single_aggregate(original_query, aggregate, resource, implementation) do + with {:ok, query} <- filtered_query(original_query, aggregate, resource), + {:ok, dynamic} <- aggregate_dynamic(query, aggregate, resource) do + query = + query + |> aggregate_base_query() + |> Ecto.Query.select(^%{aggregate.name => dynamic}) + + repo = AshSql.dynamic_repo(resource, implementation, query) + + result = + query + |> repo.one(AshSql.repo_opts(repo, implementation, nil, nil, resource)) + |> Map.get(aggregate.name) + + {:ok, result} + end + end + + defp filtered_query(original_query, aggregate, resource) do + case aggregate.query.filter do + nil -> {:ok, original_query} + %{expression: nil} -> {:ok, original_query} + filter -> AshSql.Filter.filter(original_query, filter, resource) + end + end + + defp aggregate_base_query(query) do + if query.distinct || query.limit do + query = + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(query), as: ^query.__ash_bindings__.root_binding) + |> Map.put(:__ash_bindings__, query.__ash_bindings__) + else + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + end + end + + defp aggregate_dynamic(query, %{kind: :count, field: nil} = aggregate, _resource) do + dynamic = + if aggregate.uniq? do + Ecto.Query.dynamic(count(field(as(^query.__ash_bindings__.root_binding), :id), :distinct)) + else + Ecto.Query.dynamic(count()) + end + + {:ok, dynamic} + end + + defp aggregate_dynamic(query, %{kind: :count} = aggregate, resource) do + with {:ok, field} <- aggregate_field(aggregate, resource) do + dynamic = + if aggregate.uniq? do + Ecto.Query.dynamic( + count(field(as(^query.__ash_bindings__.root_binding), ^field), :distinct) + ) + else + Ecto.Query.dynamic(count(field(as(^query.__ash_bindings__.root_binding), ^field))) + end + + {:ok, dynamic} + end + end + + defp aggregate_dynamic(query, aggregate, resource) + when aggregate.kind in [:sum, :max, :min, :avg] do + with {:ok, field} <- aggregate_field(aggregate, resource) do + field_dynamic = Ecto.Query.dynamic(field(as(^query.__ash_bindings__.root_binding), ^field)) + + dynamic = + case aggregate.kind do + :sum -> Ecto.Query.dynamic(sum(^field_dynamic)) + :max -> Ecto.Query.dynamic(max(^field_dynamic)) + :min -> Ecto.Query.dynamic(min(^field_dynamic)) + :avg -> Ecto.Query.dynamic(avg(^field_dynamic)) + end + + {:ok, maybe_type_dynamic(query, maybe_default_dynamic(dynamic, aggregate), aggregate)} + end + end + + defp aggregate_dynamic(_query, aggregate, _resource) do + {:error, "AshSql grouped query aggregate #{inspect(aggregate.name)} is unsupported"} + end + + defp aggregate_field(aggregate, resource \\ nil) + + defp aggregate_field(%{field: field}, _resource) + when is_atom(field) and not is_nil(field) do + {:ok, field} + end + + defp aggregate_field(%{kind: :count, field: nil}, _resource), do: {:ok, nil} + + defp aggregate_field(%{name: name, field: field}, _resource) do + {:error, "AshSql grouped query aggregate #{inspect(name)} cannot use field #{inspect(field)}"} + end + + defp maybe_type_dynamic(query, dynamic, aggregate) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + if type do + query.__ash_bindings__.sql_behaviour.type_expr(dynamic, type) + else + dynamic + end + end + + defp maybe_default_dynamic(dynamic, %{default_value: nil}), do: dynamic + + defp maybe_default_dynamic(dynamic, aggregate) do + Ecto.Query.dynamic(coalesce(^dynamic, ^aggregate.default_value)) + end + + defp maybe_sort_first(query, %{query: %{sort: sort}}) when sort not in [nil, []] do + order_by = + sort + |> List.wrap() + |> Enum.map(fn + {field, order} -> {order, field} + field when is_atom(field) -> {:asc, field} + end) + + Ecto.Query.order_by(query, ^order_by) + end + + defp maybe_sort_first(query, _aggregate), do: query +end diff --git a/lib/aggregate/lateral.ex b/lib/aggregate/lateral.ex new file mode 100644 index 0000000..910321b --- /dev/null +++ b/lib/aggregate/lateral.ex @@ -0,0 +1,2717 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Lateral do + @moduledoc false + + require Ecto.Query + require Ash.Query + import Ecto.Query, only: [from: 2, subquery: 1] + + @next_aggregate_names Enum.reduce(0..999, %{}, fn i, acc -> + Map.put(acc, :"aggregate_#{i}", :"aggregate_#{i + 1}") + end) + + def add_aggregates(%AshSql.Aggregate.Context{} = context, aggregates) do + add_aggregates( + context.query, + aggregates, + context.resource, + context.select?, + context.source_binding, + context.root_data + ) + end + + def add_aggregates( + query, + aggregates, + resource, + select?, + source_binding, + root_data \\ nil + ) + + def add_aggregates(query, [], _, _, _, _), do: {:ok, query} + + def add_aggregates(query, aggregates, resource, select?, source_binding, root_data) do + case resource_aggregates_to_aggregates(resource, query, aggregates) do + {:ok, aggregates} -> + root_data_path = + case root_data do + {_, path} -> + path + + _ -> + [] + end + + tenant = + case Enum.at(aggregates, 0) do + %{context: %{tenant: tenant}} -> + Ash.ToTenant.to_tenant(tenant, resource) + + _ -> + nil + end + + {query, aggregates} = + Enum.reduce( + aggregates, + {query, []}, + fn aggregate, {query, aggregates} -> + if is_atom(aggregate.name) do + existing_agg = query.__ash_bindings__.aggregate_defs[aggregate.name] + + if existing_agg && different_queries?(existing_agg.query, aggregate.query) do + {query, name} = use_aggregate_name(query, aggregate.name) + {query, [%{aggregate | name: name} | aggregates]} + else + {query, [aggregate | aggregates]} + end + else + {query, name} = use_aggregate_name(query, aggregate.name) + + {query, [%{aggregate | name: name} | aggregates]} + end + end + ) + + {already_computed_aggregates, remaining_aggregates} = + aggregates + |> Enum.uniq_by(& &1.name) + |> Enum.split_with(&already_added?(&1, query.__ash_bindings__, [])) + + query = + if Enum.any?(already_computed_aggregates) && select? do + query.__ash_bindings__.bindings + |> Enum.filter(fn + {_binding, %{type: :aggregate}} -> true + _ -> false + end) + |> Enum.reduce(query, fn {agg_binding, %{aggregates: aggs}}, q -> + q = update_in(q.__ash_bindings__, &Map.put_new(&1, :select_aggregates, [])) + + Enum.reduce(aggs, q, fn agg, q -> + if Enum.any?(already_computed_aggregates, &(&1.name == agg.name)) do + q = + update_in(q.__ash_bindings__.select_aggregates, fn select_aggs -> + [agg.name | select_aggs] + end) + + if agg.default_value do + from(row in q, + select_merge: %{ + ^agg.name => + coalesce(field(as(^agg_binding), ^agg.name), ^agg.default_value) + } + ) + else + from(row in q, + select_merge: %{^agg.name => field(as(^agg_binding), ^agg.name)} + ) + end + else + q + end + end) + end) + else + query + end + + query = + if (query.limit || query.offset || query.distinct) && root_data_path == [] && select? && + !query.__ash_bindings__[:lateral_join?] && + Enum.any?( + remaining_aggregates, + &(not optimizable_first_aggregate?(resource, &1, query)) + ) do + wrap_in_subquery_for_aggregates(query) + else + query + end + + query = + if root_data_path == [] do + query + |> Map.update!(:__ash_bindings__, fn bindings -> + bindings + |> Map.update!(:aggregate_defs, fn aggregate_defs -> + Map.merge(aggregate_defs, Map.new(aggregates, &{&1.name, &1})) + end) + end) + else + query + end + + result = + remaining_aggregates + |> Enum.group_by(fn aggregate -> + expanded_path = + aggregate.resource + |> AshSql.Join.relationship_path_to_relationships(aggregate.relationship_path) + |> Enum.map(& &1.name) + + {expanded_path, aggregate.resource, aggregate.join_filters || %{}, + aggregate.query.action.name} + end) + |> Enum.flat_map(fn {{path, resource, join_filters, read_action}, aggregates} -> + {can_group, cant_group} = + Enum.split_with(aggregates, &can_group?(resource, &1, query)) + + [{{path, resource, join_filters, read_action}, can_group}] ++ + Enum.map(cant_group, &{{path, resource, join_filters, read_action}, [&1]}) + end) + |> Enum.reject(fn + {_, []} -> + true + + _ -> + false + end) + |> Enum.reduce_while( + {:ok, query, []}, + fn {{path, resource, join_filters, read_action}, aggregates}, + {:ok, query, dynamics} -> + related = Ash.Resource.Info.related(resource, path) + read_action = Ash.Resource.Info.action(related, read_action) + + if read_action.modify_query do + raise """ + Data layer does not currently support aggregates over read actions that use `modify_query`. + + Resource: #{inspect(resource)} + Relationship Path: #{inspect(path)} + Action: #{read_action.name} + """ + end + + {first_relationship, relationship_path} = + case path do + [] -> + {nil, []} + + [first_relationship | rest] -> + case Ash.Resource.Info.relationship(resource, first_relationship) do + nil -> + raise "No such relationship #{inspect(resource)}.#{first_relationship}. aggregates: #{inspect(aggregates)}" + + first_relationship -> + {first_relationship, rest} + end + end + + hydrated_agg_refs = + aggregates + |> Enum.map(&(&1.query.filter && &1.query.filter.expression)) + |> Ash.Filter.hydrate_refs(%{ + resource: Enum.at(aggregates, 0).query.resource, + parent_stack: + if(first_relationship, do: [first_relationship.source], else: [resource]) + }) + |> elem(1) + + parent_expr = + if first_relationship do + first_relationship.filter + |> Ash.Filter.hydrate_refs(%{ + resource: first_relationship.destination, + parent_stack: [first_relationship.source] + }) + |> elem(1) + |> then(&[&1 | hydrated_agg_refs]) + |> AshSql.Join.parent_expr() + end + + used_aggregates = + Ash.Filter.used_aggregates(parent_expr, []) + + {:ok, query} = + AshSql.Aggregate.add_aggregates( + query, + used_aggregates, + resource, + false, + query.__ash_bindings__.root_binding + ) + + {:ok, query} = + AshSql.Join.join_all_relationships( + query, + parent_expr, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) + + is_single? = match?([_], aggregates) + + cond do + is_single? && + optimizable_first_aggregate?( + resource, + Enum.at(aggregates, 0), + query + ) -> + case add_first_join_aggregate( + query, + resource, + hd(aggregates), + root_data, + first_relationship, + source_binding + ) do + {:ok, query, dynamic} -> + query = + if select? do + select_or_merge(query, hd(aggregates).name, dynamic) + else + query + end + + {:cont, {:ok, query, dynamics}} + + {:error, error} -> + {:halt, {:error, error}} + end + + is_single? && Enum.at(aggregates, 0).kind == :exists -> + [aggregate] = aggregates + + expr = + if is_nil(Map.get(aggregate.query, :filter)) do + true + else + Map.get(aggregate.query, :filter) + end + + {exists, acc} = + AshSql.Expr.dynamic_expr( + query, + %Ash.Query.Exists{ + path: root_data_path ++ aggregate.relationship_path, + related?: aggregate.related?, + resource: aggregate.query.resource, + expr: expr + }, + query.__ash_bindings__ + ) + + {:cont, + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), + [{aggregate.load, aggregate.name, exists} | dynamics]}} + + true -> + tmp_query = + if first_relationship && first_relationship.type == :many_to_many do + put_in(query.__ash_bindings__[:lateral_join_bindings], [ + query.__ash_bindings__.current + ]) + |> AshSql.Bindings.explicitly_set_binding( + %{ + type: :left, + path: [first_relationship.join_relationship] + }, + query.__ash_bindings__.current + ) + else + query + end + + start_bindings_at = + if first_relationship && first_relationship.type == :many_to_many do + query.__ash_bindings__.current + 1 + else + query.__ash_bindings__.current + end + + case get_subquery( + resource, + aggregates, + is_single?, + first_relationship, + relationship_path, + tmp_query, + start_bindings_at, + query, + source_binding, + root_data_path, + tenant, + join_filters + ) do + {:error, error} -> + {:error, error} + + {:ok, subquery} -> + query = + join_subquery( + query, + subquery, + first_relationship, + relationship_path, + aggregates, + source_binding, + root_data_path + ) + + if select? do + new_dynamics = + Enum.map( + aggregates, + &{&1.load, &1.name, + select_dynamic( + resource, + query, + &1, + query.__ash_bindings__.current - 1 + )} + ) + + {:cont, {:ok, query, new_dynamics ++ dynamics}} + else + {:cont, {:ok, query, dynamics}} + end + end + end + end + ) + + case result do + {:ok, query, dynamics} -> + if select? do + {:ok, add_aggregate_selects(query, dynamics)} + else + {:ok, query} + end + + {:error, error} -> + {:error, error} + end + + {:error, error} -> + {:error, error} + end + end + + defp already_added?(aggregate, bindings, root_data_path) do + Enum.any?(bindings.bindings, fn + {_, %{type: :aggregate, aggregates: aggregates, path: ^root_data_path}} -> + aggregate.name in Enum.map(aggregates, & &1.name) + + _other -> + false + end) + end + + defp get_subquery( + _resource, + aggregates, + is_single?, + nil, + _relationship_path, + _tmp_query, + start_bindings_at, + query, + _source_binding, + root_data_path, + tenant, + _join_filters + ) do + first_aggregate = Enum.at(aggregates, 0) + aggregate_resource = first_aggregate.query.resource + + first_aggregate.query + |> Ash.Query.set_context(%{ + data_layer: %{ + table: nil, + parent_bindings: + Map.put( + query.__ash_bindings__, + :refs_at_path, + root_data_path + ), + start_bindings_at: start_bindings_at || 0 + } + }) + |> Ash.Query.unset([:sort, :distinct, :select, :limit, :offset]) + |> AshSql.Join.handle_attribute_multitenancy(tenant) + |> AshSql.Join.hydrate_refs(query.__ash_bindings__.context[:private][:actor]) + |> case do + %{valid?: true} = related_query -> + case Ash.Query.data_layer_query(related_query) do + {:ok, ecto_query} -> + {:ok, Ecto.Query.exclude(ecto_query, :select)} + + {:error, error} -> + {:error, error} + end + + %{errors: errors} -> + {:error, errors} + end + |> case do + {:ok, query} -> + maybe_filter_subquery( + query, + nil, + [], + aggregates, + is_single?, + query.__ash_bindings__.root_binding + ) + + {:error, error} -> + {:error, error} + end + |> case do + {:error, error} -> + {:error, error} + + {:ok, query} -> + if is_single? and has_filter?(Enum.at(aggregates, 0).query) do + AshSql.Filter.filter( + query, + Enum.at(aggregates, 0).query.filter, + aggregate_resource + ) + else + {:ok, query} + end + |> case do + {:error, error} -> + {:error, error} + + {:ok, filtered} -> + filtered = + AshSql.Join.set_join_prefix( + filtered, + %{query | prefix: tenant}, + aggregate_resource + ) + + {:ok, + select_all_aggregates( + aggregates, + filtered, + [], + query, + is_single?, + aggregate_resource, + nil + )} + end + end + end + + defp get_subquery( + resource, + aggregates, + is_single?, + first_relationship, + relationship_path, + tmp_query, + start_bindings_at, + query, + source_binding, + root_data_path, + tenant, + join_filters + ) do + AshSql.Join.related_subquery( + first_relationship, + tmp_query, + start_bindings_at: start_bindings_at, + refs_at_path: root_data_path, + skip_distinct_for_first_rel?: true, + on_subquery: fn subquery -> + base_binding = subquery.__ash_bindings__.root_binding + current_binding = subquery.__ash_bindings__.current + + subquery = + subquery + |> Ecto.Query.exclude(:select) + |> Ecto.Query.select(%{}) + + subquery = + apply_relationship_subquery( + subquery, + first_relationship, + query, + tenant, + source_binding, + current_binding, + base_binding + ) + + subquery = + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + first_relationship.destination + ) + + {:ok, subquery, _} = + apply_first_relationship_join_filters( + subquery, + query, + %AshSql.Expr.ExprInfo{}, + first_relationship, + join_filters + ) + + subquery = + set_in_group( + subquery, + query, + resource + ) + + {:ok, joined} = + join_all_relationships( + subquery, + aggregates, + relationship_path, + first_relationship, + is_single?, + join_filters + ) + + {:ok, filtered} = + maybe_filter_subquery( + joined, + first_relationship, + relationship_path, + aggregates, + is_single?, + subquery.__ash_bindings__.root_binding + ) + + select_all_aggregates( + aggregates, + filtered, + relationship_path, + query, + is_single?, + Ash.Resource.Info.related( + first_relationship.destination, + relationship_path + ), + first_relationship + ) + end + ) + end + + defp apply_relationship_subquery( + subquery, + %{manual: {module, opts}} = rel, + query, + tenant, + source_binding, + current_binding, + _base_binding + ) do + field = rel.destination_attribute + + from(row in subquery, + group_by: field(row, ^field), + select_merge: %{^field => field(row, ^field)} + ) + + subquery = + from(row in subquery, distinct: true) + + {:ok, subquery} = + apply( + module, + query.__ash_bindings__.sql_behaviour.manual_relationship_subquery_function(), + [ + opts, + source_binding, + current_binding - 1, + subquery + ] + ) + + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + rel.destination + ) + end + + defp apply_relationship_subquery( + subquery, + %{no_attributes?: true}, + _query, + _tenant, + _source_binding, + _current_binding, + _base_binding + ) do + subquery + end + + defp apply_relationship_subquery( + subquery, + %{type: :many_to_many} = rel, + query, + tenant, + source_binding, + _current_binding, + _base_binding + ) do + join_relationship_struct = + Ash.Resource.Info.relationship( + rel.source, + rel.join_relationship + ) + + {:ok, through} = + AshSql.Join.related_subquery( + join_relationship_struct, + query + ) + + field = rel.source_attribute_on_join_resource + + subquery = + from(sub in subquery, + join: through in ^through, + as: ^query.__ash_bindings__.current, + on: + field( + through, + ^rel.destination_attribute_on_join_resource + ) == + field(sub, ^rel.destination_attribute), + select_merge: map(through, ^[field]), + group_by: + field( + through, + ^rel.source_attribute_on_join_resource + ), + distinct: + field( + through, + ^rel.source_attribute_on_join_resource + ), + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + through, + ^rel.source_attribute_on_join_resource + ) + ) + + AshSql.Join.set_join_prefix( + subquery, + %{query | prefix: tenant}, + rel.destination + ) + end + + defp apply_relationship_subquery( + subquery, + rel, + _query, + _tenant, + source_binding, + _current_binding, + base_binding + ) do + field = rel.destination_attribute + + from(row in subquery, + group_by: field(row, ^field), + select_merge: %{^field => field(row, ^field)}, + where: + field( + parent_as(^source_binding), + ^rel.source_attribute + ) == + field( + as(^base_binding), + ^rel.destination_attribute + ) + ) + end + + defp set_in_group(%{__ash_bindings__: _} = query, _, _resource) do + Map.update!( + query, + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp set_in_group(%Ecto.SubQuery{} = subquery, query, resource) do + subquery = from(row in subquery, []) + + subquery + |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) + |> Map.update!( + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp set_in_group(other, query, resource) do + from(row in other, as: ^0) + |> AshSql.Bindings.default_bindings(resource, query.__ash_bindings__.sql_behaviour) + |> Map.update!( + :__ash_bindings__, + &Map.put(&1, :in_group?, true) + ) + end + + defp different_queries?(nil, nil), do: false + defp different_queries?(nil, _), do: true + defp different_queries?(_, nil), do: true + + defp different_queries?(query1, query2) do + query1.filter != query2.filter && query1.sort != query2.sort + end + + @doc false + def extract_shared_filters(aggregates) do + aggregates + |> Enum.reduce_while({nil, []}, fn + %{query: %{filter: filter}} = agg, {global_filters, aggs} when not is_nil(filter) -> + and_statements = + AshSql.Expr.split_statements(filter, :and) + + global_filters = + if global_filters do + Enum.filter(global_filters, &(&1 in and_statements)) + else + and_statements + end + + {:cont, {global_filters, [{agg, and_statements} | aggs]}} + + _, _ -> + {:halt, {:error, aggregates}} + end) + |> case do + {:error, aggregates} -> + {:error, aggregates} + + {[], _} -> + {:error, aggregates} + + {nil, _} -> + {:error, aggregates} + + {global_filters, aggregates} -> + global_filter = and_filters(Enum.uniq(global_filters)) + + aggregates = + Enum.map(aggregates, fn {agg, and_statements} -> + applicable_and_statements = + and_statements + |> Enum.reject(&(&1 in global_filters)) + |> and_filters() + + %{agg | query: %{agg.query | filter: applicable_and_statements}} + end) + + {{:ok, global_filter}, aggregates} + end + end + + defp and_filters(filters) do + Enum.reduce(filters, nil, fn expr, acc -> + if is_nil(acc) do + expr + else + Ash.Query.BooleanExpression.new(:and, expr, acc) + end + end) + end + + defp apply_first_relationship_join_filters( + agg_root_query, + query, + acc, + first_relationship, + join_filters + ) do + case join_filters[[first_relationship.name]] do + nil -> + {:ok, agg_root_query, acc} + + filter -> + with {:ok, agg_root_query} <- + AshSql.Join.join_all_relationships(agg_root_query, filter) do + agg_root_query = + AshSql.Expr.set_parent_path( + agg_root_query, + query + ) + + {query, acc} = + AshSql.Join.maybe_apply_filter( + agg_root_query, + agg_root_query, + agg_root_query.__ash_bindings__, + filter + ) + + {:ok, query, acc} + end + end + end + + defp use_aggregate_name(query, aggregate_name) do + {%{ + query + | __ash_bindings__: %{ + query.__ash_bindings__ + | current_aggregate_name: + next_aggregate_name(query.__ash_bindings__.current_aggregate_name), + aggregate_names: + Map.put( + query.__ash_bindings__.aggregate_names, + aggregate_name, + query.__ash_bindings__.current_aggregate_name + ) + } + }, query.__ash_bindings__.current_aggregate_name} + end + + defp resource_aggregates_to_aggregates(resource, query, aggregates) do + private_context = query.__ash_bindings__.context[:private] + + Enum.reduce_while(aggregates, {:ok, []}, fn + %Ash.Query.Aggregate{} = aggregate, {:ok, aggregates} -> + aggregate = + Ash.Actions.Read.add_calc_context( + aggregate, + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + aggregate, {:ok, aggregates} -> + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + read_action = + aggregate.read_action || Ash.Resource.Info.primary_action!(related, :read).name + + with %{valid?: true} = aggregate_query <- Ash.Query.for_read(related, read_action), + %{valid?: true} = aggregate_query <- + Ash.Query.build(aggregate_query, filter: aggregate.filter, sort: aggregate.sort) do + Ash.Query.Aggregate.new( + resource, + aggregate.name, + aggregate.kind, + path: aggregate.relationship_path, + query: aggregate_query, + field: aggregate.field, + default: aggregate.default, + filterable?: aggregate.filterable?, + type: aggregate.type, + sortable?: aggregate.filterable?, + include_nil?: aggregate.include_nil?, + constraints: aggregate.constraints, + implementation: aggregate.implementation, + uniq?: aggregate.uniq?, + read_action: + aggregate.read_action || + Ash.Resource.Info.primary_action!( + Ash.Resource.Info.related(resource, aggregate.relationship_path), + :read + ).name, + authorize?: aggregate.authorize? + ) + else + %{errors: errors} -> + {:error, errors} + end + |> case do + {:ok, aggregate} -> + aggregate = + aggregate + |> Map.put(:load, aggregate.name) + |> Ash.Actions.Read.add_calc_context( + private_context[:actor], + private_context[:authorize?], + private_context[:tenant], + private_context[:tracer], + query.__ash_bindings__[:domain], + query.__ash_bindings__[:resource], + parent_stack: query.__ash_bindings__[:parent_resources] || [] + ) + + {:cont, {:ok, [aggregate | aggregates]}} + + {:error, error} -> + {:halt, {:error, error}} + end + end) + end + + defp add_first_join_aggregate( + query, + _resource, + %{related?: false} = aggregate, + root_data, + _, + source_binding + ) do + path = + case root_data do + {_resource, path} -> + path + + _ -> + [] + end + + subquery_result = + aggregate.query + |> Ash.Query.set_context(%{ + data_layer: %{ + table: nil, + parent_bindings: + Map.put( + query.__ash_bindings__, + :refs_at_path, + path + ), + start_bindings_at: (query.__ash_bindings__.current || 0) + 1 + } + }) + |> Ash.Query.limit(1) + |> Ash.Query.data_layer_query() + + case subquery_result do + {:ok, ecto_query} -> + ref = + %Ash.Query.Ref{ + attribute: aggregate.field, + resource: aggregate.query.resource + } + + {:ok, ecto_query} = AshSql.Join.join_all_relationships(ecto_query, ref) + + ecto_query = + case aggregate.field do + %Ash.Query.Aggregate{} = aggregate -> + {:ok, ecto_query} = + add_aggregates( + ecto_query, + [aggregate], + aggregate.query.resource, + true, + source_binding, + root_data + ) + + ecto_query + + %Ash.Resource.Aggregate{} = aggregate -> + {:ok, ecto_query} = + add_aggregates( + ecto_query, + [aggregate], + Ash.Resource.Info.related(aggregate.resource, aggregate.relationship_path), + true, + source_binding, + root_data + ) + + ecto_query + + %Ash.Resource.Calculation{ + name: name, + calculation: {module, opts}, + type: type, + constraints: constraints + } -> + {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) + expression = module.expression(opts, new_calc.context) + + expression = + Ash.Expr.fill_template( + expression, + actor: aggregate.context.actor, + tenant: aggregate.query.to_tenant, + args: %{}, + context: aggregate.context + ) + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: ecto_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, ecto_query} = + AshSql.Calculation.add_calculations( + ecto_query, + [{new_calc, expression}], + ecto_query.__ash_bindings__.resource, + source_binding, + true + ) + + ecto_query + + %Ash.Query.Calculation{ + module: module, + opts: opts, + context: context + } = calc -> + expression = module.expression(opts, context) + + expression = + Ash.Expr.fill_template( + expression, + actor: context.actor, + tenant: aggregate.query.to_tenant, + args: context.arguments, + context: context.source_context + ) + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: ecto_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, ecto_query} = + AshSql.Calculation.add_calculations( + ecto_query, + [{calc, expression}], + ecto_query.__ash_bindings__.resource, + source_binding, + true + ) + + ecto_query + + _ -> + ecto_query + end + + ref = + %Ash.Query.Ref{ + attribute: aggregate_field(aggregate, aggregate.query.resource, query), + relationship_path: [], + resource: aggregate.query.resource + } + + value = + Ecto.Query.dynamic(field(as(^query.__ash_bindings__.current), ^ref.attribute.name)) + + AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + query = + if has_parent_expr?(aggregate.query.filter) do + from(row in query, + left_lateral_join: related in subquery(ecto_query), + on: true, + as: ^query.__ash_bindings__.current + ) + else + from(row in query, + left_join: related in subquery(ecto_query), + on: true, + as: ^query.__ash_bindings__.current + ) + end + + query = + AshSql.Bindings.add_binding( + query, + %{ + path: path, + type: :aggregate, + aggregates: [aggregate] + } + ) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + with_default = + if aggregate.default_value do + if type do + type_expr = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^type_expr)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + {:ok, query, casted} + + {:error, error} -> + {:error, error} + end + end + + defp add_first_join_aggregate( + query, + resource, + aggregate, + root_data, + first_relationship, + _source_binding + ) do + {resource, path} = + case root_data do + {resource, path} -> + {resource, path} + + _ -> + {resource, []} + end + + join_filters = + if has_filter?(aggregate) do + %{(path ++ aggregate.relationship_path) => aggregate.query.filter} + else + %{} + end + + case AshSql.Join.join_all_relationships( + query, + nil, + [], + [ + {:left, + AshSql.Join.relationship_path_to_relationships( + resource, + path ++ aggregate.relationship_path + )} + ], + [], + nil, + false, + join_filters + ) do + {:ok, query} -> + ref = + aggregate_field_ref( + aggregate, + Ash.Resource.Info.related(resource, path ++ aggregate.relationship_path), + path ++ aggregate.relationship_path, + query, + first_relationship + ) + + {:ok, query} = AshSql.Join.join_all_relationships(query, ref) + + {value, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + with_default = + if aggregate.default_value do + if type do + type_expr = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^type_expr)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + {:ok, AshSql.Bindings.merge_expr_accumulator(query, acc), casted} + + {:error, error} -> + {:error, error} + end + end + + defp maybe_filter_subquery( + agg_query, + first_relationship, + relationship_path, + aggregates, + is_single?, + source_binding + ) do + Enum.reduce_while(aggregates, {:ok, agg_query}, fn aggregate, {:ok, agg_query} -> + filter = + if !Enum.empty?(relationship_path) && aggregate.query.filter do + Ash.Filter.move_to_relationship_path( + aggregate.query.filter, + relationship_path + ) + |> Map.put(:resource, first_relationship.destination) + else + aggregate.query.filter + end + + # For unrelated aggregates (first_relationship is nil), use the aggregate's resource + # For related aggregates, use the relationship destination + related = + if first_relationship do + first_relationship.destination + else + aggregate.query.resource + end + + field = + case aggregate.field do + field when is_atom(field) -> + Ash.Resource.Info.field(related, field) + + field -> + field + end + + root_data = + case first_relationship do + nil -> + nil + + %{destination: destination, name: name} -> + {destination, [name]} + end + + agg_query = + case field do + %Ash.Query.Aggregate{} = aggregate -> + {:ok, agg_query} = + add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + + agg_query + + %Ash.Resource.Aggregate{} = aggregate -> + {:ok, agg_query} = + add_aggregates(agg_query, [aggregate], related, false, source_binding, root_data) + + agg_query + + %Ash.Resource.Calculation{ + name: name, + calculation: {module, opts}, + type: type, + constraints: constraints + } -> + {:ok, new_calc} = Ash.Query.Calculation.new(name, module, opts, type, constraints) + expression = module.expression(opts, new_calc.context) + + expression = + Ash.Expr.fill_template( + expression, + actor: aggregate.context.actor, + tenant: aggregate.query.to_tenant, + args: %{}, + context: aggregate.context + ) + + expression = + if Enum.empty?(relationship_path) do + expression + else + Ash.Filter.move_to_relationship_path( + expression, + relationship_path + ) + end + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: agg_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, agg_query} = + AshSql.Calculation.add_calculations( + agg_query, + [{new_calc, expression}], + agg_query.__ash_bindings__.resource, + source_binding, + false + ) + + agg_query + + %Ash.Query.Calculation{ + module: module, + opts: opts, + context: context + } = calc -> + expression = module.expression(opts, context) + + expression = + Ash.Expr.fill_template( + expression, + actor: context.actor, + tenant: aggregate.query.to_tenant, + args: context.arguments, + context: context.source_context + ) + + expression = + if Enum.empty?(relationship_path) do + expression + else + Ash.Filter.move_to_relationship_path( + expression, + relationship_path + ) + end + + {:ok, expression} = + Ash.Filter.hydrate_refs(expression, %{ + resource: agg_query.__ash_bindings__.resource, + public?: false + }) + + {:ok, agg_query} = + AshSql.Calculation.add_calculations( + agg_query, + [{calc, expression}], + agg_query.__ash_bindings__.resource, + source_binding, + false + ) + + agg_query + + _ -> + agg_query + end + + if has_filter?(aggregate.query) && is_single? do + {:cont, AshSql.Filter.filter(agg_query, filter, agg_query.__ash_bindings__.resource)} + else + {:cont, {:ok, agg_query}} + end + end) + end + + defp join_subquery( + query, + subquery, + nil, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: sub in subquery(subquery), + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + defp join_subquery( + query, + subquery, + %{manual: {_, _}}, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: sub in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + defp join_subquery( + query, + subquery, + %{type: :many_to_many}, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: agg in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + query + |> AshSql.Bindings.add_binding(%{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + }) + |> AshSql.Bindings.merge_expr_accumulator(%AshSql.Expr.ExprInfo{}) + end + + defp join_subquery( + query, + subquery, + _first_relationship, + _relationship_path, + aggregates, + _source_binding, + root_data_path + ) do + query = + from(row in query, + left_lateral_join: agg in ^subquery, + as: ^query.__ash_bindings__.current, + on: true + ) + + AshSql.Bindings.add_binding( + query, + %{ + path: root_data_path, + type: :aggregate, + aggregates: aggregates + } + ) + end + + def next_aggregate_name(i) do + @next_aggregate_names[i] || + raise Ash.Error.Framework.AssumptionFailed, + message: """ + All 1000 static names for aggregates have been used in a single query. + Congratulations, this means that you have gone so wildly beyond our imagination + of how much can fit into a single quer. Please file an issue and we will raise the limit. + """ + end + + defp select_all_aggregates( + aggregates, + joined, + relationship_path, + _query, + is_single?, + resource, + first_relationship + ) do + Enum.reduce(aggregates, joined, fn aggregate, joined -> + add_subquery_aggregate_select( + joined, + relationship_path, + aggregate, + resource, + is_single?, + first_relationship + ) + end) + end + + defp join_all_relationships( + agg_root_query, + _aggregates, + relationship_path, + first_relationship, + _is_single?, + join_filters + ) do + if Enum.empty?(relationship_path) do + {:ok, agg_root_query} + else + join_filters = + Enum.reduce(join_filters, %{}, fn {key, value}, acc -> + if List.starts_with?(key, [first_relationship.name]) do + Map.put(acc, Enum.drop(key, 1), value) + else + acc + end + end) + + AshSql.Join.join_all_relationships( + agg_root_query, + Map.values(join_filters), + [], + [ + {:inner, + AshSql.Join.relationship_path_to_relationships( + first_relationship.destination, + relationship_path + )} + ], + [], + nil, + false, + join_filters, + agg_root_query + ) + end + end + + @doc false + def can_group?(_, %{kind: :exists}, _), do: false + def can_group?(_, %{kind: :list}, _), do: false + + def can_group?(resource, aggregate, query) do + can_group_kind?(aggregate, resource, query) && !has_exists?(aggregate) && + !references_to_many_relationships?(aggregate) && + !optimizable_first_aggregate?(resource, aggregate, query) && + !has_parent_expr?(aggregate.query.filter) + end + + defp has_parent_expr?(filter, depth \\ 0) do + not is_nil( + Ash.Filter.find( + filter, + fn + %Ash.Query.Call{name: :parent, args: [expr]} -> + if depth == 0 do + true + else + has_parent_expr?(expr, depth - 1) + end + + %Ash.Query.Exists{expr: expr} -> + has_parent_expr?(expr, depth + 1) + + %Ash.Query.Parent{expr: expr} -> + if depth == 0 do + true + else + has_parent_expr?(expr, depth - 1) + end + + %Ash.Query.Ref{ + attribute: %Ash.Query.Aggregate{ + field: %Ash.Query.Calculation{module: module, opts: opts, context: context} + } + } -> + if module.has_expression?() do + module.expression(opts, context) + |> has_parent_expr?(depth + 1) + else + false + end + + _other -> + false + end, + true, + true, + true + ) + ) + end + + # We can potentially optimize this. We don't have to prevent aggregates that reference + # relationships from joining, we can + # 1. group up the ones that do join relationships by the relationships they join + # 2. potentially group them all up that join to relationships and just join to all the relationships + # but this method is predictable and easy so we're starting by just not grouping them + defp references_to_many_relationships?(aggregate) do + if aggregate.query do + aggregate.query.filter + |> Ash.Filter.relationship_paths() + |> Enum.any?(&to_many_path?(aggregate.query.resource, &1)) + else + false + end + end + + defp to_many_path?(_resource, []), do: false + + defp to_many_path?(resource, [rel | rest]) do + case Ash.Resource.Info.relationship(resource, rel) do + %{cardinality: :many} -> + true + + nil -> + raise """ + No such relationship #{inspect(resource)}.#{rel} + """ + + rel -> + to_many_path?(rel.destination, rest) + end + end + + defp can_group_kind?(aggregate, resource, query) do + if aggregate.kind == :first do + if array_type?(resource, aggregate) || + optimizable_first_aggregate?(resource, aggregate, query) do + false + else + true + end + else + true + end + end + + @doc false + def optimizable_first_aggregate?( + resource, + %{ + kind: :first, + relationship_path: relationship_path, + join_filters: join_filters, + field: %Ash.Query.Calculation{} = field + }, + _ + ) do + ref = + %Ash.Query.Ref{ + attribute: field, + relationship_path: relationship_path, + resource: resource + } + + with true <- join_filters == %{}, + [] <- Ash.Filter.used_aggregates(ref, :all), + [] <- Ash.Filter.relationship_paths(ref) do + true + else + _ -> + false + end + end + + def optimizable_first_aggregate?( + _resource, + %{ + kind: :first, + field: %Ash.Query.Aggregate{} + }, + _ + ) do + false + end + + def optimizable_first_aggregate?( + resource, + %{ + name: name, + kind: :first, + relationship_path: relationship_path, + join_filters: join_filters, + query: %{resource: related}, + field: field + } = aggregate, + query + ) do + related + |> Ash.Resource.Info.field(field) + |> case do + %Ash.Resource.Aggregate{} -> + false + + %Ash.Resource.Calculation{} -> + field = aggregate_field(aggregate, resource, query) + + ref = + %Ash.Query.Ref{ + attribute: field, + relationship_path: relationship_path, + resource: resource + } + + with [] <- Ash.Filter.used_aggregates(ref, :all), + [] <- Ash.Filter.relationship_paths(ref) do + true + else + _ -> + false + end + + nil -> + false + + _ -> + name in query.__ash_bindings__.sql_behaviour.simple_join_first_aggregates(resource) || + (join_filters in [nil, %{}, []] && + single_path?(resource, relationship_path)) + end + end + + def optimizable_first_aggregate?(_, _, _), do: false + + defp array_type?(resource, aggregate) do + related = Ash.Resource.Info.related(resource, aggregate.relationship_path) + + case aggregate.field do + nil -> + false + + %{type: {:array, _}} -> + true + + type when is_atom(type) -> + case Ash.Resource.Info.field(related, aggregate.field).type do + {:array, _} -> + true + + _ -> + false + end + + _ -> + false + end + end + + defp has_exists?(aggregate) do + !!Ash.Filter.find(aggregate.query && aggregate.query.filter, fn + %Ash.Query.Exists{} -> true + _ -> false + end) + end + + defp add_aggregate_selects(query, dynamics) do + {in_aggregates, in_body} = + Enum.split_with(dynamics, fn {load, _name, _dynamic} -> is_nil(load) end) + + aggs = + in_body + |> Map.new(fn {load, _, dynamic} -> + {load, dynamic} + end) + + aggs = + if Enum.empty?(in_aggregates) do + aggs + else + Map.put( + aggs, + :aggregates, + Map.new(in_aggregates, fn {_, name, dynamic} -> + {name, dynamic} + end) + ) + end + + Ecto.Query.select_merge(query, ^aggs) + end + + defp select_dynamic(_resource, query, aggregate, binding) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + field = + if type do + field_ref = Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + query.__ash_bindings__.sql_behaviour.type_expr(field_ref, type) + else + Ecto.Query.dynamic(field(as(^binding), ^aggregate.name)) + end + + coalesced = + if is_nil(aggregate.default_value) do + field + else + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic( + coalesce( + ^field, + ^typed_default + ) + ) + else + Ecto.Query.dynamic( + coalesce( + ^field, + ^aggregate.default_value + ) + ) + end + end + + if type do + query.__ash_bindings__.sql_behaviour.type_expr(coalesced, type) + else + coalesced + end + end + + defp has_filter?(nil), do: false + defp has_filter?(%{filter: nil}), do: false + defp has_filter?(%{filter: %Ash.Filter{expression: nil}}), do: false + defp has_filter?(_), do: true + + defp has_sort?(nil), do: false + defp has_sort?(%{sort: nil}), do: false + defp has_sort?(%{sort: []}), do: false + defp has_sort?(%{sort: _}), do: true + defp has_sort?(_), do: false + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: :first} = aggregate, + resource, + is_single?, + first_relationship + ) do + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + {field, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + has_sort? = has_sort?(aggregate.query) + + array_agg = + if aggregate.include_nil? do + # any_value() ignores NULLs by design in PostgreSQL, so we must use + # array_agg with [1] indexing when include_nil? is true + "array_agg" + else + query.__ash_bindings__.sql_behaviour.list_aggregate(aggregate.query.resource) + end + + {sorted, include_nil_filter_field, query} = + if has_sort? || first_relationship.sort not in [nil, []] do + {sort, binding} = + if has_sort? do + {aggregate.query.sort, binding} + else + {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} + end + + {:ok, sort_expr, query} = + AshSql.Sort.sort( + query, + sort, + Ash.Resource.Info.related( + query.__ash_bindings__.resource, + relationship_path + ), + relationship_path, + binding, + :return + ) + + if aggregate.include_nil? do + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + ["#{array_agg}(? ORDER BY #{question_marks})", field] ++ sort_expr + ) + + {sort_expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {sort_expr, nil, query} + else + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + {expr, include_nil_filter_field} = + if has_filter?(aggregate.query) and !is_single? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "#{array_agg}(? ORDER BY #{question_marks})", + field + ] ++ + sort_expr + ) + + {expr, field} + else + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "#{array_agg}(? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, nil} + end + + {sort_expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {sort_expr, include_nil_filter_field, query} + end + else + case array_agg do + "array_agg" -> + {Ecto.Query.dynamic( + [row], + fragment("array_agg(?)", ^field) + ), nil, query} + + "any_value" -> + {Ecto.Query.dynamic( + [row], + fragment("any_value(?)", ^field) + ), nil, query} + end + end + + {query, filtered} = + filter_field( + sorted, + include_nil_filter_field, + query, + aggregate, + relationship_path, + is_single? + ) + + value = + if array_agg == "array_agg" do + Ecto.Query.dynamic(fragment("(?)[1]", ^filtered)) + else + filtered + end + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^value, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^value, ^aggregate.default_value)) + end + else + value + end + + casted = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + query = AshSql.Bindings.merge_expr_accumulator(query, acc) + + select_or_merge( + query, + aggregate.name, + casted + ) + end + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: :list} = aggregate, + resource, + is_single?, + first_relationship + ) do + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + {field, acc} = + AshSql.Expr.dynamic_expr( + query, + ref, + Map.put(query.__ash_bindings__, :location, :aggregate), + false + ) + + related = + Ash.Resource.Info.related( + query.__ash_bindings__.resource, + relationship_path + ) + + has_sort? = has_sort?(aggregate.query) + + {sorted, include_nil_filter_field, query} = + if has_sort? || (first_relationship && first_relationship.sort not in [nil, []]) do + {sort, binding} = + if has_sort? do + {aggregate.query.sort, binding} + else + {List.wrap(first_relationship.sort), query.__ash_bindings__.root_binding} + end + + {:ok, sort_expr, query} = + AshSql.Sort.sort( + query, + sort, + related, + relationship_path, + binding, + :return + ) + + question_marks = Enum.map(sort_expr, fn _ -> " ? " end) + + distinct = + if Map.get(aggregate, :uniq?) do + "DISTINCT " + else + "" + end + + {expr, include_nil_filter_field} = + if aggregate.include_nil? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + ["array_agg(#{distinct}? ORDER BY #{question_marks})", field] ++ sort_expr + ) + + {expr, nil} + else + if has_filter?(aggregate.query) and !is_single? do + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "array_agg(#{distinct}? ORDER BY #{question_marks})", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, field} + else + {:ok, expr} = + Ash.Query.Function.Fragment.casted_new( + [ + "array_agg(#{distinct}? ORDER BY #{question_marks}) FILTER (WHERE ? IS NOT NULL)", + field + ] ++ + sort_expr ++ [field] + ) + + {expr, nil} + end + end + + {expr, acc} = + AshSql.Expr.dynamic_expr(query, expr, query.__ash_bindings__, false) + + query = + AshSql.Bindings.merge_expr_accumulator(query, acc) + + {expr, include_nil_filter_field, query} + else + if Map.get(aggregate, :uniq?) do + {Ecto.Query.dynamic( + [row], + fragment("array_agg(DISTINCT ?)", ^field) + ), nil, query} + else + {Ecto.Query.dynamic( + [row], + fragment("array_agg(?)", ^field) + ), nil, query} + end + end + + {query, filtered} = + filter_field( + sorted, + include_nil_filter_field, + query, + aggregate, + relationship_path, + is_single? + ) + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) + end + else + filtered + end + + cast = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + query = AshSql.Bindings.merge_expr_accumulator(query, acc) + + select_or_merge( + query, + aggregate.name, + cast + ) + end + + def add_subquery_aggregate_select( + query, + relationship_path, + %{kind: kind} = aggregate, + resource, + is_single?, + first_relationship + ) + when kind in [:count, :sum, :avg, :max, :min, :custom] do + ref = + aggregate_field_ref( + aggregate, + resource, + relationship_path, + query, + first_relationship + ) + + {field, query} = + case kind do + :custom -> + # we won't use this if its custom so don't try to make one + {nil, query} + + :count -> + if aggregate.field do + {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} + else + {nil, query} + end + + _ -> + {expr, acc} = AshSql.Expr.dynamic_expr(query, ref, query.__ash_bindings__, false) + + {expr, AshSql.Bindings.merge_expr_accumulator(query, acc)} + end + + type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + aggregate.type, + aggregate.constraints, + :aggregate + ) + + binding = + AshSql.Bindings.get_binding( + query.__ash_bindings__.resource, + relationship_path, + query, + [:left, :inner, :root] + ) + + field = + case kind do + :count -> + cond do + !aggregate.field -> + Ecto.Query.dynamic([row], count()) + + Map.get(aggregate, :uniq?) -> + Ecto.Query.dynamic([row], count(^field, :distinct)) + + match?(%{attribute: %{allow_nil?: false}}, ref) -> + Ecto.Query.dynamic([row], count()) + + true -> + Ecto.Query.dynamic([row], count(^field)) + end + + :sum -> + Ecto.Query.dynamic([row], sum(^field)) + + :avg -> + Ecto.Query.dynamic([row], avg(^field)) + + :max -> + Ecto.Query.dynamic([row], max(^field)) + + :min -> + Ecto.Query.dynamic([row], min(^field)) + + :custom -> + {module, opts} = aggregate.implementation + + module.dynamic(opts, binding) + end + + {query, filtered} = filter_field(field, nil, query, aggregate, relationship_path, is_single?) + + with_default = + if aggregate.default_value do + if type do + typed_default = + query.__ash_bindings__.sql_behaviour.type_expr(aggregate.default_value, type) + + Ecto.Query.dynamic(coalesce(^filtered, ^typed_default)) + else + Ecto.Query.dynamic(coalesce(^filtered, ^aggregate.default_value)) + end + else + filtered + end + + cast = + if type do + query.__ash_bindings__.sql_behaviour.type_expr(with_default, type) + else + with_default + end + + select_or_merge(query, aggregate.name, cast) + end + + defp filter_field(field, include_nil_filter_field, query, _aggregate, _relationship_path, true) do + if include_nil_filter_field do + {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} + else + {query, field} + end + end + + defp filter_field( + field, + include_nil_filter_field, + query, + aggregate, + relationship_path, + _is_single? + ) do + if has_filter?(aggregate.query) do + filter = + Ash.Filter.move_to_relationship_path( + aggregate.query.filter, + relationship_path + ) + + used_aggregates = Ash.Filter.used_aggregates(filter, []) + + # here we bypass an inner join. + # Really, we should check if all aggs in a group + # could do the same inner join, then do an inner join + {:ok, query} = + AshSql.Join.join_all_relationships( + query, + filter, + [], + nil, + [], + nil, + true, + nil, + nil, + true + ) + + {:ok, query} = + add_aggregates( + query, + used_aggregates, + query.__ash_bindings__.resource, + false, + query.__ash_bindings__.root_binding + ) + + {expr, acc} = + AshSql.Expr.dynamic_expr( + query, + filter, + query.__ash_bindings__, + false, + {aggregate.type, aggregate.constraints} + ) + + if include_nil_filter_field do + {AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^field, ^expr and not is_nil(^include_nil_filter_field)))} + else + {AshSql.Bindings.merge_expr_accumulator(query, acc), + Ecto.Query.dynamic(filter(^field, ^expr))} + end + else + if include_nil_filter_field do + {query, Ecto.Query.dynamic(filter(^field, not is_nil(^include_nil_filter_field)))} + else + {query, field} + end + end + end + + defp select_or_merge(query, aggregate_name, casted) do + query = + if query.select do + query + else + Ecto.Query.select(query, %{}) + end + + Ecto.Query.select_merge(query, ^%{aggregate_name => casted}) + end + + def aggregate_field_ref(aggregate, resource, relationship_path, query, first_relationship) do + if aggregate.kind == :count && !aggregate.field do + nil + else + %Ash.Query.Ref{ + attribute: aggregate_field(aggregate, resource, query), + relationship_path: relationship_path, + resource: query.__ash_bindings__.resource + } + |> case do + %{attribute: %Ash.Resource.Aggregate{}} = ref when not is_nil(first_relationship) -> + if first_relationship do + %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} + else + ref + end + + %{attribute: %Ash.Query.Aggregate{}} = ref when not is_nil(first_relationship) -> + if first_relationship do + %{ref | relationship_path: [first_relationship.name | ref.relationship_path]} + else + ref + end + + other -> + other + end + end + end + + defp single_path?(_, []), do: true + + defp single_path?(resource, [relationship | rest]) do + relationship = Ash.Resource.Info.relationship(resource, relationship) + + !Map.get(relationship, :from_many?) && + (relationship.type == :belongs_to || + has_one_with_identity?(relationship)) && + single_path?(relationship.destination, rest) + end + + defp has_one_with_identity?(%{type: :has_one, from_many?: false} = relationship) do + Ash.Resource.Info.primary_key(relationship.destination) == [ + relationship.destination_attribute + ] || + relationship.destination + |> Ash.Resource.Info.identities() + |> Enum.any?(fn %{keys: keys} -> + keys == [relationship.destination_attribute] + end) + end + + defp has_one_with_identity?(_), do: false + + @doc false + def aggregate_field(aggregate, resource, query) do + if is_atom(aggregate.field) do + case Ash.Resource.Info.field( + resource, + aggregate.field || List.first(Ash.Resource.Info.primary_key(resource)) + ) do + %Ash.Resource.Calculation{calculation: {module, opts}} = calculation -> + calc_type = + AshSql.Expr.parameterized_type( + query.__ash_bindings__.sql_behaviour, + calculation.type, + Map.get(calculation, :constraints, []), + :calculation + ) + + AshSql.Expr.validate_type!(query, calc_type, "#{inspect(calculation.name)}") + + {:ok, query_calc} = + Ash.Query.Calculation.new( + calculation.name, + module, + opts, + calculation.type, + calculation.constraints + ) + + Ash.Actions.Read.add_calc_context( + query_calc, + aggregate.context.actor, + aggregate.context.authorize?, + aggregate.context.tenant, + aggregate.context.tracer, + query.__ash_bindings__[:domain], + aggregate.query.resource, + parent_stack: [ + query.__ash_bindings__.resource | query.__ash_bindings__[:parent_resources] || [] + ] + ) + + nil -> + raise "no such aggregate field: #{inspect(resource)}.#{aggregate.field}" + + other -> + other + end + else + aggregate.field + end + end + + def wrap_in_subquery_for_aggregates(query) do + resource = query.__ash_bindings__.resource + selected_by_default = Ash.Resource.Info.selected_by_default_attribute_names(resource) + + selected_fields = + query.__ash_bindings__[:select] || + extract_selected_fields(query, resource, selected_by_default) + + all_attr_names = + resource + |> Ash.Resource.Info.attribute_names() + |> MapSet.to_list() + + to_select = + Enum.reject(all_attr_names, &(&1 in selected_fields)) + + query_with_all_attrs = + case query.select do + %Ecto.Query.SelectExpr{expr: {:merge, _, [l, {:%{}, [], kw}]}} -> + put_in( + query.select.expr, + {:merge, [], + [ + l, + {:%{}, [], + kw ++ + Enum.map( + to_select, + &{&1, + {{:., [], [{:&, [], [query.__ash_bindings__.root_binding]}, &1]}, [], []}} + )} + ]} + ) + + _ -> + from(row in query, + select_merge: struct(row, ^to_select) + ) + end + + # Flatten nested calculations/aggregates maps before creating subquery + # Ecto doesn't allow nested maps in subquery select expressions + {calculations_require_rewrite, aggregates_require_rewrite, query_with_all_attrs} = + AshSql.Query.rewrite_nested_selects(query_with_all_attrs) + + # After flattening, we need to: + # 1. Use the updated select_calculations from the rewritten query (which excludes :calculations) + # 2. Add the flattened calculation/aggregate field names to the select + flattened_calc_fields = Map.keys(calculations_require_rewrite) + flattened_agg_fields = Map.keys(aggregates_require_rewrite) + + # Get select_calculations from the rewritten query (it has :calculations removed) + select_calculations = + (query_with_all_attrs.__ash_bindings__[:select_calculations] || []) -- [:calculations] + + select_aggregates = + (query_with_all_attrs.__ash_bindings__[:select_aggregates] || []) -- [:aggregates] + + subquery_query = + from(row in subquery(query_with_all_attrs), + as: ^query.__ash_bindings__.root_binding, + select: + struct( + row, + ^Enum.concat([ + selected_fields, + select_calculations, + select_aggregates, + flattened_calc_fields, + flattened_agg_fields + ]) + ) + ) + + root_binding = query.__ash_bindings__.root_binding + + only_root_binding = %{ + root_binding => query.__ash_bindings__.bindings[root_binding] + } + + new_bindings = + query.__ash_bindings__ + |> Map.put(:bindings, only_root_binding) + |> Map.delete(:__order__?) + |> Map.update( + :calculations_require_rewrite, + calculations_require_rewrite, + &Map.merge(&1, calculations_require_rewrite) + ) + |> Map.update( + :aggregates_require_rewrite, + aggregates_require_rewrite, + &Map.merge(&1, aggregates_require_rewrite) + ) + + Map.put(subquery_query, :__ash_bindings__, new_bindings) + end + + defp extract_selected_fields( + %{select: %Ecto.Query.SelectExpr{expr: expr, take: take}}, + resource, + all_attribute_names + ) do + Enum.uniq(extract_fields_from_expr(expr, resource, take, all_attribute_names)) + end + + defp extract_fields_from_expr(expr, resource, take, all_attribute_names) do + case expr do + {:&, [], [ix]} -> + case take do + %{^ix => {:struct, fields}} when is_list(fields) -> + fields + + %{^ix => {:map, fields}} when is_list(fields) -> + fields + + take when take == %{} -> + all_attribute_names + + _ -> + [] + end + + {:%{}, [], fields} -> + Enum.map(fields, fn {field_name, _} -> field_name end) + + {:%, [], [_struct, {:%{}, [], fields}]} -> + Enum.map(fields, fn {field_name, _} -> field_name end) + + {:merge, _, [sel1, sel2]} -> + Enum.concat( + extract_fields_from_expr(sel1, resource, take, all_attribute_names), + extract_fields_from_expr(sel2, resource, take, all_attribute_names) + ) + + _other -> + all_attribute_names + end + end +end diff --git a/lib/aggregate/lateral/query.ex b/lib/aggregate/lateral/query.ex new file mode 100644 index 0000000..f465c6c --- /dev/null +++ b/lib/aggregate/lateral/query.ex @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: 2024 ash_sql contributors +# +# SPDX-License-Identifier: MIT + +defmodule AshSql.Aggregate.Lateral.Query do + @moduledoc false + import Ecto.Query, only: [from: 2, subquery: 1] + + def run_aggregate_query(original_query, aggregates, resource, implementation) do + original_query = + AshSql.Bindings.default_bindings(original_query, resource, implementation) + + {can_group, cant_group} = + aggregates + |> Enum.split_with(&AshSql.Aggregate.can_group?(resource, &1, original_query)) + |> case do + {[one], cant_group} -> {[], [one | cant_group]} + {can_group, cant_group} -> {can_group, cant_group} + end + + {global_filter, can_group} = + AshSql.Aggregate.extract_shared_filters(can_group) + + query = + case global_filter do + {:ok, global_filter} -> + AshSql.Filter.filter(original_query, global_filter, resource) + + :error -> + {:ok, original_query} + end + + case query do + {:error, error} -> + {:error, error} + + {:ok, query} -> + query = + if query.distinct || query.limit do + query = + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(query), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + query + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + |> Map.put(:__ash_bindings__, query.__ash_bindings__) + + group_query = + Enum.reduce( + can_group, + query, + fn agg, query -> + first_relationship = + Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) + + AshSql.Aggregate.add_subquery_aggregate_select( + query, + agg.relationship_path |> Enum.drop(1), + agg, + resource, + false, + first_relationship + ) + end + ) + + result = + case can_group do + [] -> + %{} + + _ -> + repo = AshSql.dynamic_repo(resource, implementation, query) + repo.one(group_query, AshSql.repo_opts(repo, implementation, nil, nil, resource)) + end + + {:ok, add_single_aggs(result, resource, query, cant_group, implementation)} + end + end + + def add_single_aggs(result, resource, query, cant_group, implementation) do + Enum.reduce(cant_group, result, fn + %{kind: :exists} = agg, result -> + {:ok, filtered} = + case agg do + %{query: %{filter: filter}} when not is_nil(filter) -> + AshSql.Filter.filter(query, filter, resource) + + _ -> + {:ok, query} + end + + filtered = + if filtered.distinct || filtered.limit do + filtered = + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + + repo = AshSql.dynamic_repo(resource, implementation, filtered) + + Map.put( + result || %{}, + agg.name, + repo.exists?(filtered, AshSql.repo_opts(repo, implementation, nil, nil, resource)) + ) + + agg, result -> + {:ok, filtered} = + case agg do + %{query: %{filter: filter}} when not is_nil(filter) -> + AshSql.Filter.filter(query, filter, resource) + + _ -> + {:ok, query} + end + + filtered = + if filtered.distinct do + in_query = filtered |> Ecto.Query.exclude(:distinct) |> Ecto.Query.exclude(:select) + + dynamic = + Enum.reduce(Ash.Resource.Info.primary_key(resource), nil, fn key, dynamic -> + if dynamic do + Ecto.Query.dynamic( + [row], + ^dynamic and + field(parent_as(^query.__ash_bindings__.root_binding), ^key) == + field(row, ^key) + ) + else + Ecto.Query.dynamic( + [row], + field(parent_as(^query.__ash_bindings__.root_binding), ^key) == + field(row, ^key) + ) + end + end) + + in_query = + from(row in in_query, where: ^dynamic) + + in_query = Ecto.Query.exclude(in_query, :distinct) + + from(row in query.from.source, + as: ^query.__ash_bindings__.root_binding, + where: exists(in_query) + ) + else + filtered + end + + filtered = + if filtered.limit do + filtered = + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + + from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) + else + filtered + |> Ecto.Query.exclude(:select) + |> Ecto.Query.exclude(:order_by) + |> Map.put(:windows, []) + |> Ecto.Query.select(%{}) + end + + first_relationship = + Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) + + filtered = AshSql.Bindings.default_bindings(filtered, resource, implementation) + + ref = + AshSql.Aggregate.aggregate_field_ref( + agg, + Ash.Resource.Info.related(resource, agg.relationship_path), + agg.relationship_path, + filtered, + first_relationship + ) + + {:ok, filtered} = + if ref do + {:ok, filtered} = + case ref.attribute do + %struct{} = agg when struct in [Ash.Query.Aggregate, Ash.Resource.Aggregate] -> + AshSql.Aggregate.add_aggregates( + filtered, + [agg], + resource, + false, + filtered.__ash_bindings__.root_binding + ) + + %Ash.Query.Calculation{} = calc -> + used_aggregates = Ash.Filter.used_aggregates(calc, []) + + with {:ok, filtered} <- AshSql.Join.join_all_relationships(filtered, calc, []) do + AshSql.Aggregate.add_aggregates( + filtered, + used_aggregates, + resource, + false, + filtered.__ash_bindings__.root_binding + ) + end + + _other -> + {:ok, filtered} + end + + AshSql.Join.join_all_relationships(filtered, ref) + else + {:ok, filtered} + end + + query = + AshSql.Aggregate.add_subquery_aggregate_select( + filtered, + agg.relationship_path |> Enum.drop(1), + %{agg | query: %{agg.query | filter: nil}}, + resource, + true, + first_relationship + ) + + repo = AshSql.dynamic_repo(resource, implementation, query) + + Map.merge( + result || %{}, + repo.one( + query, + AshSql.repo_opts(repo, query.__ash_bindings__.sql_behaviour, nil, nil, resource) + ) + ) + end) + end +end diff --git a/lib/aggregate_query.ex b/lib/aggregate_query.ex index ad612de..4f51c05 100644 --- a/lib/aggregate_query.ex +++ b/lib/aggregate_query.ex @@ -4,255 +4,33 @@ defmodule AshSql.AggregateQuery do @moduledoc false - import Ecto.Query, only: [from: 2, subquery: 1] def run_aggregate_query(original_query, aggregates, resource, implementation) do original_query = AshSql.Bindings.default_bindings(original_query, resource, implementation) - {can_group, cant_group} = - aggregates - |> Enum.split_with(&AshSql.Aggregate.can_group?(resource, &1, original_query)) - |> case do - {[one], cant_group} -> {[], [one | cant_group]} - {can_group, cant_group} -> {can_group, cant_group} - end - - {global_filter, can_group} = - AshSql.Aggregate.extract_shared_filters(can_group) - - query = - case global_filter do - {:ok, global_filter} -> - AshSql.Filter.filter(original_query, global_filter, resource) - - :error -> - {:ok, original_query} - end - - case query do - {:error, error} -> - {:error, error} - - {:ok, query} -> - query = - if query.distinct || query.limit do - query = - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(query), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - query - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - |> Map.put(:__ash_bindings__, query.__ash_bindings__) - - group_query = - Enum.reduce( - can_group, - query, - fn agg, query -> - first_relationship = - Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) - - AshSql.Aggregate.add_subquery_aggregate_select( - query, - agg.relationship_path |> Enum.drop(1), - agg, - resource, - false, - first_relationship - ) - end - ) - - result = - case can_group do - [] -> - %{} - - _ -> - repo = AshSql.dynamic_repo(resource, implementation, query) - repo.one(group_query, AshSql.repo_opts(repo, implementation, nil, nil, resource)) - end - - {:ok, add_single_aggs(result, resource, query, cant_group, implementation)} - end + strategy(original_query, resource).run_aggregate_query( + original_query, + aggregates, + resource, + implementation + ) end def add_single_aggs(result, resource, query, cant_group, implementation) do - Enum.reduce(cant_group, result, fn - %{kind: :exists} = agg, result -> - {:ok, filtered} = - case agg do - %{query: %{filter: filter}} when not is_nil(filter) -> - AshSql.Filter.filter(query, filter, resource) - - _ -> - {:ok, query} - end - - filtered = - if filtered.distinct || filtered.limit do - filtered = - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - - repo = AshSql.dynamic_repo(resource, implementation, filtered) - - Map.put( - result || %{}, - agg.name, - repo.exists?(filtered, AshSql.repo_opts(repo, implementation, nil, nil, resource)) - ) - - agg, result -> - {:ok, filtered} = - case agg do - %{query: %{filter: filter}} when not is_nil(filter) -> - AshSql.Filter.filter(query, filter, resource) - - _ -> - {:ok, query} - end - - filtered = - if filtered.distinct do - in_query = filtered |> Ecto.Query.exclude(:distinct) |> Ecto.Query.exclude(:select) - - dynamic = - Enum.reduce(Ash.Resource.Info.primary_key(resource), nil, fn key, dynamic -> - if dynamic do - Ecto.Query.dynamic( - [row], - ^dynamic and - field(parent_as(^query.__ash_bindings__.root_binding), ^key) == - field(row, ^key) - ) - else - Ecto.Query.dynamic( - [row], - field(parent_as(^query.__ash_bindings__.root_binding), ^key) == - field(row, ^key) - ) - end - end) - - in_query = - from(row in in_query, where: ^dynamic) - - in_query = Ecto.Query.exclude(in_query, :distinct) - - from(row in query.from.source, - as: ^query.__ash_bindings__.root_binding, - where: exists(in_query) - ) - else - filtered - end - - filtered = - if filtered.limit do - filtered = - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - - from(row in subquery(filtered), as: ^query.__ash_bindings__.root_binding, select: %{}) - else - filtered - |> Ecto.Query.exclude(:select) - |> Ecto.Query.exclude(:order_by) - |> Map.put(:windows, []) - |> Ecto.Query.select(%{}) - end - - first_relationship = - Ash.Resource.Info.relationship(resource, agg.relationship_path |> Enum.at(0)) - - filtered = AshSql.Bindings.default_bindings(filtered, resource, implementation) - - ref = - AshSql.Aggregate.aggregate_field_ref( - agg, - Ash.Resource.Info.related(resource, agg.relationship_path), - agg.relationship_path, - filtered, - first_relationship - ) - - {:ok, filtered} = - if ref do - {:ok, filtered} = - case ref.attribute do - %struct{} = agg when struct in [Ash.Query.Aggregate, Ash.Resource.Aggregate] -> - AshSql.Aggregate.add_aggregates( - filtered, - [agg], - resource, - false, - filtered.__ash_bindings__.root_binding - ) - - %Ash.Query.Calculation{} = calc -> - used_aggregates = Ash.Filter.used_aggregates(calc, []) - - with {:ok, filtered} <- AshSql.Join.join_all_relationships(filtered, calc, []) do - AshSql.Aggregate.add_aggregates( - filtered, - used_aggregates, - resource, - false, - filtered.__ash_bindings__.root_binding - ) - end - - _other -> - {:ok, filtered} - end - - AshSql.Join.join_all_relationships(filtered, ref) - else - {:ok, filtered} - end - - query = - AshSql.Aggregate.add_subquery_aggregate_select( - filtered, - agg.relationship_path |> Enum.drop(1), - %{agg | query: %{agg.query | filter: nil}}, - resource, - true, - first_relationship - ) - - repo = AshSql.dynamic_repo(resource, implementation, query) + AshSql.Aggregate.Lateral.Query.add_single_aggs( + result, + resource, + query, + cant_group, + implementation + ) + end - Map.merge( - result || %{}, - repo.one( - query, - AshSql.repo_opts(repo, query.__ash_bindings__.sql_behaviour, nil, nil, resource) - ) - ) - end) + defp strategy(query, resource) do + case query.__ash_bindings__.sql_behaviour.aggregate_strategy(resource) do + :lateral -> AshSql.Aggregate.Lateral.Query + :grouped -> AshSql.Aggregate.Grouped.Query + end end end diff --git a/lib/implementation.ex b/lib/implementation.ex index 496065a..5efd357 100644 --- a/lib/implementation.ex +++ b/lib/implementation.ex @@ -38,6 +38,7 @@ defmodule AshSql.Implementation do @callback require_extension_for_citext() :: {true, String.t()} | false @callback strpos_function() :: String.t() @callback type_expr(expr :: term, type :: term) :: term + @callback aggregate_strategy(Ash.Resource.t()) :: :lateral | :grouped @optional_callbacks determine_types: 3 @@ -58,6 +59,7 @@ defmodule AshSql.Implementation do def ilike?, do: true def equals_any?, do: true def storage_type(_, _), do: nil + def aggregate_strategy(_resource), do: :lateral def type_expr(expr, type) do type = @@ -85,6 +87,7 @@ defmodule AshSql.Implementation do require_ash_functions_for_or_and_and?: 0, require_extension_for_citext: 0, simple_join_first_aggregates: 1, + aggregate_strategy: 1, type_expr: 2, storage_type: 2, list_aggregate: 1, diff --git a/mix.exs b/mix.exs index 823966c..3cebd79 100644 --- a/mix.exs +++ b/mix.exs @@ -83,6 +83,7 @@ defmodule AshSql.MixProject do {:ash, ash_version("~> 3.24 and >= 3.24.5")}, {:ecto_sql, "~> 3.9"}, {:ecto, "~> 3.13 and >= 3.13.4"}, + {:jason, "~> 1.0"}, # dev/test dependencies {:igniter, "~> 0.5", only: [:dev, :test]}, {:simple_sat, "~> 0.1", only: [:dev, :test]},