From 0e37920ce26f6a45d66dcea3fc397dab3d7f3fe2 Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Fri, 14 Feb 2025 13:40:35 +0100 Subject: [PATCH 1/6] Initial suggestion --- lib/data_layer.ex | 28 ++++++- lib/data_layer/info.ex | 10 +++ .../migration_generator.ex | 35 +++++++- lib/migration_generator/operation.ex | 2 +- lib/migration_generator/phase.ex | 38 ++++++--- lib/partitioning.ex | 79 +++++++++++++++++++ .../partitioned_posts/20250214114101.json | 43 ++++++++++ .../20250214114101_partitioned_post.exs | 20 +++++ test/migration_generator_test.exs | 50 ++++++++++++ test/partition_test.exs | 15 ++++ test/support/domain.ex | 1 + test/support/resources/partitioned_post.ex | 28 +++++++ 12 files changed, 333 insertions(+), 16 deletions(-) create mode 100644 lib/partitioning.ex create mode 100644 priv/resource_snapshots/test_repo/partitioned_posts/20250214114101.json create mode 100644 priv/test_repo/migrations/20250214114101_partitioned_post.exs create mode 100644 test/partition_test.exs create mode 100644 test/support/resources/partitioned_post.ex diff --git a/lib/data_layer.ex b/lib/data_layer.ex index 274919e3..a6139582 100644 --- a/lib/data_layer.ex +++ b/lib/data_layer.ex @@ -252,6 +252,31 @@ defmodule AshPostgres.DataLayer do ] } + @partitioning %Spark.Dsl.Section{ + name: :partitioning, + describe: """ + A section for configuring the initial partitioning of the table + """, + examples: [ + """ + partitioning do + method :list + attribute :post + end + """ + ], + schema: [ + method: [ + type: {:one_of, [:range, :list, :hash]}, + doc: "Specifying what partitioning method to use" + ], + attribute: [ + type: :atom, + doc: "The attribute to partition on" + ] + ] + } + @postgres %Spark.Dsl.Section{ name: :postgres, describe: """ @@ -262,7 +287,8 @@ defmodule AshPostgres.DataLayer do @custom_statements, @manage_tenant, @references, - @check_constraints + @check_constraints, + @partitioning ], modules: [ :repo diff --git a/lib/data_layer/info.ex b/lib/data_layer/info.ex index 696439ae..9476b4e0 100644 --- a/lib/data_layer/info.ex +++ b/lib/data_layer/info.ex @@ -222,4 +222,14 @@ defmodule AshPostgres.DataLayer.Info do def manage_tenant_update?(resource) do Extension.get_opt(resource, [:postgres, :manage_tenant], :update?, false) end + + @doc "Partitioning method" + def partitioning_method(resource) do + Extension.get_opt(resource, [:postgres, :partitioning], :method, nil) + end + + @doc "Partitioning attribute" + def partitioning_attribute(resource) do + Extension.get_opt(resource, [:postgres, :partitioning], :attribute, nil) + end end diff --git a/lib/migration_generator/migration_generator.ex b/lib/migration_generator/migration_generator.ex index e0301956..87164f2c 100644 --- a/lib/migration_generator/migration_generator.ex +++ b/lib/migration_generator/migration_generator.ex @@ -1192,14 +1192,25 @@ defmodule AshPostgres.MigrationGenerator do defp group_into_phases( [ - %Operation.CreateTable{table: table, schema: schema, multitenancy: multitenancy} | rest + %Operation.CreateTable{ + table: table, + schema: schema, + multitenancy: multitenancy, + partitioning: partitioning + } + | rest ], nil, acc ) do group_into_phases( rest, - %Phase.Create{table: table, schema: schema, multitenancy: multitenancy}, + %Phase.Create{ + table: table, + schema: schema, + multitenancy: multitenancy, + partitioning: partitioning + }, acc ) end @@ -1801,7 +1812,8 @@ defmodule AshPostgres.MigrationGenerator do table: snapshot.table, schema: snapshot.schema, multitenancy: snapshot.multitenancy, - old_multitenancy: empty_snapshot.multitenancy + old_multitenancy: empty_snapshot.multitenancy, + partitioning: snapshot.partitioning } | acc ]) @@ -2836,7 +2848,8 @@ defmodule AshPostgres.MigrationGenerator do repo: AshPostgres.DataLayer.Info.repo(resource, :mutate), multitenancy: multitenancy(resource), base_filter: AshPostgres.DataLayer.Info.base_filter_sql(resource), - has_create_action: has_create_action?(resource) + has_create_action: has_create_action?(resource), + partitioning: partitioning(resource) } hash = @@ -2911,6 +2924,20 @@ defmodule AshPostgres.MigrationGenerator do end) end + defp partitioning(resource) do + method = AshPostgres.DataLayer.Info.partitioning_method(resource) + attribute = AshPostgres.DataLayer.Info.partitioning_attribute(resource) + + if not is_nil(method) and not is_nil(attribute) do + %{ + method: method, + attribute: attribute + } + else + nil + end + end + defp multitenancy(resource) do strategy = Ash.Resource.Info.multitenancy_strategy(resource) attribute = Ash.Resource.Info.multitenancy_attribute(resource) diff --git a/lib/migration_generator/operation.ex b/lib/migration_generator/operation.ex index 4b8c4eae..76b94563 100644 --- a/lib/migration_generator/operation.ex +++ b/lib/migration_generator/operation.ex @@ -131,7 +131,7 @@ defmodule AshPostgres.MigrationGenerator.Operation do defmodule CreateTable do @moduledoc false - defstruct [:table, :schema, :multitenancy, :old_multitenancy] + defstruct [:table, :schema, :multitenancy, :old_multitenancy, :partitioning] end defmodule AddAttribute do diff --git a/lib/migration_generator/phase.ex b/lib/migration_generator/phase.ex index b1e3e2b3..6e735501 100644 --- a/lib/migration_generator/phase.ex +++ b/lib/migration_generator/phase.ex @@ -3,24 +3,21 @@ defmodule AshPostgres.MigrationGenerator.Phase do defmodule Create do @moduledoc false - defstruct [:table, :schema, :multitenancy, operations: [], commented?: false] + defstruct [:table, :schema, :multitenancy, partitioning: nil, operations: [], commented?: false] import AshPostgres.MigrationGenerator.Operation.Helper, only: [as_atom: 1] - def up(%{schema: schema, table: table, operations: operations, multitenancy: multitenancy}) do + def up(%{schema: schema, table: table, operations: operations, multitenancy: multitenancy, partitioning: partitioning}) do if multitenancy.strategy == :context do - "create table(:#{as_atom(table)}, primary_key: false, prefix: prefix()) do\n" <> + arguments = arguments([prefix("prefix()"), options(partitioning: partitioning)]) + + "create table(:#{as_atom(table)}, primary_key: false#{arguments}) do\n" <> Enum.map_join(operations, "\n", fn operation -> operation.__struct__.up(operation) end) <> "\nend" else - opts = - if schema do - ", prefix: \"#{schema}\"" - else - "" - end + arguments = arguments([prefix(schema), options(partitioning: partitioning)]) - "create table(:#{as_atom(table)}, primary_key: false#{opts}) do\n" <> + "create table(:#{as_atom(table)}, primary_key: false#{arguments}) do\n" <> Enum.map_join(operations, "\n", fn operation -> operation.__struct__.up(operation) end) <> "\nend" end @@ -40,6 +37,27 @@ defmodule AshPostgres.MigrationGenerator.Phase do "drop table(:#{as_atom(table)}#{opts})" end end + + def arguments(["",""]), do: "" + def arguments(arguments), do: ", " <> Enum.join(Enum.reject(arguments, &is_nil(&1)), ",") + + def prefix(nil), do: nil + def prefix(schema), do: "prefix: #{schema}" + + def options(_options, _acc \\ []) + def options([], []), do: "" + def options([], acc), do: "options: \"#{Enum.join(acc, " ")}\"" + + def options([{:partitioning, %{method: method, attribute: attribute}} | rest], acc) do + option = "PARTITION BY #{String.upcase(Atom.to_string(method))} (#{attribute})" + + rest + |> options(acc ++ [option]) + end + + def options([_| rest], acc) do + options(rest, acc) + end end defmodule Alter do diff --git a/lib/partitioning.ex b/lib/partitioning.ex new file mode 100644 index 00000000..8775f1b3 --- /dev/null +++ b/lib/partitioning.ex @@ -0,0 +1,79 @@ +defmodule AshPostgres.Partitioning do + @moduledoc false + + @doc """ + Create a new partition for a resource + """ + def create_partition(resource, opts) do + repo = AshPostgres.DataLayer.Info.repo(resource) + + resource + |> AshPostgres.DataLayer.Info.partitioning_method() + |> case do + :range -> + create_range_partition(repo, resource, opts) + + :list -> + create_list_partition(repo, resource, opts) + + :hash -> + create_hash_partition(repo, resource, opts) + + unsupported_method -> + raise "Invalid partition method, got: #{unsupported_method}" + end + end + + @doc """ + Check if partition exists + """ + def exists?(resource, opts) do + repo = AshPostgres.DataLayer.Info.repo(resource) + key = Keyword.fetch!(opts, :key) + table = AshPostgres.DataLayer.Info.table(resource) + partition_name = table <> "_" <> "#{key}" + + partition_exists?(repo, resource, partition_name) + end + + # TBI + defp create_range_partition(repo, resource, opts) do + end + + defp create_list_partition(repo, resource, opts) do + key = Keyword.fetch!(opts, :key) + table = AshPostgres.DataLayer.Info.table(resource) + partition_name = table <> "_" <> "#{key}" + + if partition_exists?(repo, resource, partition_name) do + {:error, :allready_exists} + else + Ecto.Adapters.SQL.query( + repo, + "CREATE TABLE #{partition_name} PARTITION OF public.#{table} FOR VALUES IN (#{key})" + ) + + if partition_exists?(repo, resource, partition_name) do + :ok + else + {:error, "Unable to create partition"} + end + end + end + + # TBI + defp create_hash_partition(repo, resource, opts) do + end + + defp partition_exists?(repo, resource, parition_name) do + %Postgrex.Result{} = + result = + repo + |> Ecto.Adapters.SQL.query!( + "select table_name from information_schema.tables t where t.table_schema = 'public' and t.table_name = $1", + [parition_name] + ) + + result.num_rows > 0 + end +end diff --git a/priv/resource_snapshots/test_repo/partitioned_posts/20250214114101.json b/priv/resource_snapshots/test_repo/partitioned_posts/20250214114101.json new file mode 100644 index 00000000..e88c485e --- /dev/null +++ b/priv/resource_snapshots/test_repo/partitioned_posts/20250214114101.json @@ -0,0 +1,43 @@ +{ + "attributes": [ + { + "allow_nil?": false, + "default": "fragment(\"gen_random_uuid()\")", + "generated?": false, + "primary_key?": true, + "references": null, + "size": null, + "source": "id", + "type": "uuid" + }, + { + "allow_nil?": false, + "default": "1", + "generated?": false, + "primary_key?": true, + "references": null, + "size": null, + "source": "key", + "type": "bigint" + } + ], + "base_filter": null, + "check_constraints": [], + "custom_indexes": [], + "custom_statements": [], + "has_create_action": false, + "hash": "7FE5D9659135887A47FAE2729CEB0281FA8FF392EDB3B43426EAFD89A1518FEB", + "identities": [], + "multitenancy": { + "attribute": null, + "global": null, + "strategy": null + }, + "partitioning": { + "attribute": "key", + "method": "list" + }, + "repo": "Elixir.AshPostgres.TestRepo", + "schema": null, + "table": "partitioned_posts" +} \ No newline at end of file diff --git a/priv/test_repo/migrations/20250214114101_partitioned_post.exs b/priv/test_repo/migrations/20250214114101_partitioned_post.exs new file mode 100644 index 00000000..28fd2300 --- /dev/null +++ b/priv/test_repo/migrations/20250214114101_partitioned_post.exs @@ -0,0 +1,20 @@ +defmodule AshPostgres.TestRepo.Migrations.PartitionedPost do + @moduledoc """ + Updates resources based on their most recent snapshots. + + This file was autogenerated with `mix ash_postgres.generate_migrations` + """ + + use Ecto.Migration + + def up do + create table(:partitioned_posts, primary_key: false, options: "PARTITION BY LIST (key)") do + add(:id, :uuid, null: false, default: fragment("gen_random_uuid()"), primary_key: true) + add(:key, :bigint, null: false, default: 1, primary_key: true) + end + end + + def down do + drop(table(:partitioned_posts)) + end +end diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index b7f10688..4856e510 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -304,6 +304,56 @@ defmodule AshPostgres.MigrationGeneratorTest do end end + describe "creating initial snapshots for resources with partitioning" do + setup do + on_exit(fn -> + File.rm_rf!("test_snapshots_path") + File.rm_rf!("test_migration_path") + end) + + defposts do + postgres do + partitioning do + method(:list) + attribute(:title) + end + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true) + end + end + + defdomain([Post]) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + quiet: false, + format: false + ) + + :ok + end + + test "the migration sets up resources correctly" do + # the snapshot exists and contains valid json + assert File.read!(Path.wildcard("test_snapshots_path/test_repo/posts/*.json")) + |> Jason.decode!(keys: :atoms!) + + assert [file] = + Path.wildcard("test_migration_path/**/*_migrate_resources*.exs") + |> Enum.reject(&String.contains?(&1, "extensions")) + + file_contents = File.read!(file) + + # the migration creates the table with options specifing how to partition the table + assert file_contents =~ + ~S{create table(:posts, primary_key: false, options: "PARTITION BY LIST (title)") do} + end + end + describe "custom_indexes with `concurrently: true`" do setup do on_exit(fn -> diff --git a/test/partition_test.exs b/test/partition_test.exs new file mode 100644 index 00000000..07fe96fe --- /dev/null +++ b/test/partition_test.exs @@ -0,0 +1,15 @@ +defmodule AshPostgres.PartitionTest do + use AshPostgres.RepoCase, async: false + alias AshPostgres.Test.PartitionedPost + + test "seeding data works" do + assert false == AshPostgres.Partitioning.exists?(PartitionedPost, key: 1) + assert :ok == AshPostgres.Partitioning.create_partition(PartitionedPost, key: 1) + assert true == AshPostgres.Partitioning.exists?(PartitionedPost, key: 1) + + Ash.Seed.seed!(%PartitionedPost{key: 1}) + + assert :ok == AshPostgres.Partitioning.create_partition(PartitionedPost, key: 2) + Ash.Seed.seed!(%PartitionedPost{key: 2}) + end +end diff --git a/test/support/domain.ex b/test/support/domain.ex index c2d867a7..f682cb14 100644 --- a/test/support/domain.ex +++ b/test/support/domain.ex @@ -32,6 +32,7 @@ defmodule AshPostgres.Test.Domain do resource(AshPostgres.Test.PostFollower) resource(AshPostgres.Test.StatefulPostFollower) resource(AshPostgres.Test.PostWithEmptyUpdate) + resource(AshPostgres.Test.PartitionedPost) end authorization do diff --git a/test/support/resources/partitioned_post.ex b/test/support/resources/partitioned_post.ex new file mode 100644 index 00000000..c17df602 --- /dev/null +++ b/test/support/resources/partitioned_post.ex @@ -0,0 +1,28 @@ +defmodule AshPostgres.Test.PartitionedPost do + @moduledoc false + use Ash.Resource, + domain: AshPostgres.Test.Domain, + data_layer: AshPostgres.DataLayer + + postgres do + table "partitioned_posts" + repo AshPostgres.TestRepo + + partitioning do + method(:list) + attribute(:key) + end + end + + actions do + default_accept(:*) + + defaults([:read, :destroy]) + end + + attributes do + uuid_primary_key(:id, writable?: true) + + attribute(:key, :integer, allow_nil?: false, primary_key?: true, default: 1) + end +end From 37dd64368627a8543047cb64a269294593fe51c9 Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Mon, 17 Feb 2025 12:44:44 +0100 Subject: [PATCH 2/6] Fix primary key generation for multitenancy --- .gitignore | 1 + lib/migration_generator/operation.ex | 122 ++++++++++++------ .../tenants/composite_key/20250217095820.json | 40 ++++++ .../tenants/composite_key/20250217095828.json | 40 ++++++ .../20250217095820_migrate_resources5.exs | 20 +++ .../20250217095828_migrate_resources6.exs | 29 +++++ test/migration_generator_test.exs | 85 +++++++++++- test/multitenancy_test.exs | 11 +- test/support/multitenancy/domain.ex | 1 + .../resources/composite_key_post.ex | 26 ++++ test_snapshot_path/test_repo/extensions.json | 10 -- 11 files changed, 331 insertions(+), 54 deletions(-) create mode 100644 priv/resource_snapshots/test_repo/tenants/composite_key/20250217095820.json create mode 100644 priv/resource_snapshots/test_repo/tenants/composite_key/20250217095828.json create mode 100644 priv/test_repo/tenant_migrations/20250217095820_migrate_resources5.exs create mode 100644 priv/test_repo/tenant_migrations/20250217095828_migrate_resources6.exs create mode 100644 test/support/multitenancy/resources/composite_key_post.ex delete mode 100644 test_snapshot_path/test_repo/extensions.json diff --git a/.gitignore b/.gitignore index 4fe6d351..7d849d6b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ ash_postgres-*.tar test_migration_path test_snapshots_path +test_tenant_migration_path diff --git a/lib/migration_generator/operation.ex b/lib/migration_generator/operation.ex index 76b94563..7765fc48 100644 --- a/lib/migration_generator/operation.ex +++ b/lib/migration_generator/operation.ex @@ -1055,17 +1055,24 @@ defmodule AshPostgres.MigrationGenerator.Operation do @moduledoc false defstruct [:schema, :table, :keys, no_phase: true] - def up(%{schema: schema, table: table, keys: keys}) do + def up(%{schema: schema, table: table, keys: keys, multitenancy: multitenancy}) do keys = Enum.join(keys, ", ") - if schema do - """ - execute("ALTER TABLE \\\"#{schema}.#{table}\\\" ADD PRIMARY KEY (#{keys})") - """ - else - """ - execute("ALTER TABLE \\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") - """ + cond do + multitenancy.strategy == :context -> + """ + execute("ALTER TABLE \\\"\#{prefix()}\\\".\\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ + + schema -> + """ + execute("ALTER TABLE \\\"#{schema}.#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ + + true -> + """ + execute("ALTER TABLE \\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ end end @@ -1082,33 +1089,54 @@ defmodule AshPostgres.MigrationGenerator.Operation do "" end - def down(%{schema: schema, table: table, remove_old?: remove_old?, keys: keys}) do + def down(%{ + schema: schema, + table: table, + remove_old?: remove_old?, + keys: keys, + multitenancy: multitenancy + }) do keys = Enum.join(keys, ", ") - if schema do - remove_old = - if remove_old? do - """ - execute("ALTER TABLE \\\"#{schema}.#{table}\\\" DROP constraint #{table}_pkey") - """ - end + cond do + multitenancy.strategy == :context -> + remove_old = + if remove_old? do + """ + execute("ALTER TABLE \\\"\#{prefix()}\\\".\\\"#{table}\\\" DROP constraint #{table}_pkey") + """ + end - """ - #{remove_old} - execute("ALTER TABLE \\\"#{schema}.#{table}\\\" ADD PRIMARY KEY (#{keys})") - """ - else - remove_old = - if remove_old? do - """ - execute("ALTER TABLE \\\"#{table}\\\" DROP constraint #{table}_pkey") - """ - end + """ + #{remove_old} + execute("ALTER TABLE \\\"\#{prefix()}\\\".\\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ + + not is_nil(schema) -> + remove_old = + if remove_old? do + """ + execute("ALTER TABLE \\\"#{schema}.#{table}\\\" DROP constraint #{table}_pkey") + """ + end + + """ + #{remove_old} + execute("ALTER TABLE \\\"#{schema}.#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ + + true -> + remove_old = + if remove_old? do + """ + execute("ALTER TABLE \\\"#{table}\\\" DROP constraint #{table}_pkey") + """ + end - """ - #{remove_old} - execute("ALTER TABLE \\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") - """ + """ + #{remove_old} + execute("ALTER TABLE \\\"#{table}\\\" ADD PRIMARY KEY (#{keys})") + """ end end end @@ -1117,11 +1145,16 @@ defmodule AshPostgres.MigrationGenerator.Operation do @moduledoc false defstruct [:schema, :table, no_phase: true] - def up(%{schema: schema, table: table}) do - if schema do - "drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: \"#{schema}\")" - else - "drop constraint(#{inspect(table)}, \"#{table}_pkey\")" + def up(%{schema: schema, table: table, multitenancy: multitenancy}) do + cond do + multitenancy.strategy == :context -> + "drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: prefix())" + + schema -> + "drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: \"#{schema}\")" + + true -> + "drop constraint(#{inspect(table)}, \"#{table}_pkey\")" end end @@ -1138,7 +1171,7 @@ defmodule AshPostgres.MigrationGenerator.Operation do "" end - def down(%{schema: schema, table: table, commented?: commented?}) do + def down(%{schema: schema, table: table, commented?: commented?, multitenancy: multitenancy}) do comment = if commented? do """ @@ -1149,10 +1182,15 @@ defmodule AshPostgres.MigrationGenerator.Operation do "" end - if schema do - "#{comment}drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: \"#{schema}\")" - else - "#{comment}drop constraint(#{inspect(table)}, \"#{table}_pkey\")" + cond do + multitenancy.strategy == :context -> + "#{comment}drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: prefix())" + + schema -> + "#{comment}drop constraint(#{inspect(table)}, \"#{table}_pkey\", prefix: \"#{schema}\")" + + true -> + "#{comment}drop constraint(#{inspect(table)}, \"#{table}_pkey\")" end end end diff --git a/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095820.json b/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095820.json new file mode 100644 index 00000000..e78ef415 --- /dev/null +++ b/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095820.json @@ -0,0 +1,40 @@ +{ + "attributes": [ + { + "allow_nil?": false, + "default": "nil", + "generated?": true, + "primary_key?": true, + "references": null, + "size": null, + "source": "id", + "type": "bigint" + }, + { + "allow_nil?": false, + "default": "nil", + "generated?": false, + "primary_key?": false, + "references": null, + "size": null, + "source": "title", + "type": "text" + } + ], + "base_filter": null, + "check_constraints": [], + "custom_indexes": [], + "custom_statements": [], + "has_create_action": true, + "hash": "F547F05D353FC4B04CC604B8F2215A512BFB9FAD20B3C1DD2BCBF2455072D958", + "identities": [], + "multitenancy": { + "attribute": null, + "global": false, + "strategy": "context" + }, + "partitioning": null, + "repo": "Elixir.AshPostgres.TestRepo", + "schema": null, + "table": "composite_key" +} \ No newline at end of file diff --git a/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095828.json b/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095828.json new file mode 100644 index 00000000..0e511ce5 --- /dev/null +++ b/priv/resource_snapshots/test_repo/tenants/composite_key/20250217095828.json @@ -0,0 +1,40 @@ +{ + "attributes": [ + { + "allow_nil?": false, + "default": "nil", + "generated?": true, + "primary_key?": true, + "references": null, + "size": null, + "source": "id", + "type": "bigint" + }, + { + "allow_nil?": false, + "default": "nil", + "generated?": false, + "primary_key?": true, + "references": null, + "size": null, + "source": "title", + "type": "text" + } + ], + "base_filter": null, + "check_constraints": [], + "custom_indexes": [], + "custom_statements": [], + "has_create_action": true, + "hash": "0EA09E46F197BAF8034CBFC7CCEFE46D2CCE9927ACD0991B5E90D5463B9B4AEC", + "identities": [], + "multitenancy": { + "attribute": null, + "global": false, + "strategy": "context" + }, + "partitioning": null, + "repo": "Elixir.AshPostgres.TestRepo", + "schema": null, + "table": "composite_key" +} \ No newline at end of file diff --git a/priv/test_repo/tenant_migrations/20250217095820_migrate_resources5.exs b/priv/test_repo/tenant_migrations/20250217095820_migrate_resources5.exs new file mode 100644 index 00000000..5aa2decb --- /dev/null +++ b/priv/test_repo/tenant_migrations/20250217095820_migrate_resources5.exs @@ -0,0 +1,20 @@ +defmodule AshPostgres.TestRepo.TenantMigrations.MigrateResources5 do + @moduledoc """ + Updates resources based on their most recent snapshots. + + This file was autogenerated with `mix ash_postgres.generate_migrations` + """ + + use Ecto.Migration + + def up do + create table(:composite_key, primary_key: false, prefix: prefix()) do + add(:id, :bigserial, null: false, primary_key: true) + add(:title, :text, null: false) + end + end + + def down do + drop(table(:composite_key, prefix: prefix())) + end +end diff --git a/priv/test_repo/tenant_migrations/20250217095828_migrate_resources6.exs b/priv/test_repo/tenant_migrations/20250217095828_migrate_resources6.exs new file mode 100644 index 00000000..f5018509 --- /dev/null +++ b/priv/test_repo/tenant_migrations/20250217095828_migrate_resources6.exs @@ -0,0 +1,29 @@ +defmodule AshPostgres.TestRepo.TenantMigrations.MigrateResources6 do + @moduledoc """ + Updates resources based on their most recent snapshots. + + This file was autogenerated with `mix ash_postgres.generate_migrations` + """ + + use Ecto.Migration + + def up do + drop(constraint("composite_key", "composite_key_pkey", prefix: prefix())) + + alter table(:composite_key, prefix: prefix()) do + modify(:title, :text) + end + + execute("ALTER TABLE \"#{prefix()}\".\"composite_key\" ADD PRIMARY KEY (id, title)") + end + + def down do + drop(constraint("composite_key", "composite_key_pkey", prefix: prefix())) + + alter table(:composite_key, prefix: prefix()) do + modify(:title, :text) + end + + execute("ALTER TABLE \"#{prefix()}\".\"composite_key\" ADD PRIMARY KEY (id)") + end +end diff --git a/test/migration_generator_test.exs b/test/migration_generator_test.exs index 4856e510..956e6b92 100644 --- a/test/migration_generator_test.exs +++ b/test/migration_generator_test.exs @@ -542,6 +542,89 @@ defmodule AshPostgres.MigrationGeneratorTest do end end + describe "creating a multitenancy resource without composite key, adding it later" do + setup do + on_exit(fn -> + nil + File.rm_rf!("test_snapshots_path") + File.rm_rf!("test_migration_path") + File.rm_rf!("test_tenant_migration_path") + end) + + :ok + end + + test "create without composite key, then add extra key" do + defposts do + postgres do + schema("example") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true, allow_nil?: false) + end + + multitenancy do + strategy(:context) + end + end + + defdomain([Post]) + + send(self(), {:mix_shell_input, :yes?, true}) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + tenant_migration_path: "test_tenant_migration_path", + quiet: false, + format: false + ) + + defposts do + postgres do + schema("example") + end + + attributes do + uuid_primary_key(:id) + attribute(:title, :string, public?: true, primary_key?: true, allow_nil?: false) + end + + multitenancy do + strategy(:context) + end + end + + defdomain([Post]) + + send(self(), {:mix_shell_input, :yes?, true}) + + AshPostgres.MigrationGenerator.generate(Domain, + snapshot_path: "test_snapshots_path", + migration_path: "test_migration_path", + tenant_migration_path: "test_tenant_migration_path", + quiet: false, + format: false + ) + + assert [_file1, file2] = + Enum.sort(Path.wildcard("test_tenant_migration_path/**/*_migrate_resources*.exs")) + |> Enum.reject(&String.contains?(&1, "extensions")) + + contents = File.read!(file2) + + [up_side, down_side] = String.split(contents, "def down", parts: 2) + + assert up_side =~ + ~S[execute("ALTER TABLE \"#{prefix()}\".\"posts\" ADD PRIMARY KEY (id, title)")] + + assert down_side =~ + ~S[execute("ALTER TABLE \"#{prefix()}\".\"posts\" ADD PRIMARY KEY (id)")] + end + end + describe "creating follow up migrations with a schema" do setup do on_exit(fn -> @@ -1104,7 +1187,7 @@ defmodule AshPostgres.MigrationGeneratorTest do test "returns code(1) if snapshots and resources don't fit", %{domain: domain} do assert catch_exit( AshPostgres.MigrationGenerator.generate(domain, - snapshot_path: "test_snapshot_path", + snapshot_path: "test_snapshots_path", migration_path: "test_migration_path", check: true ) diff --git a/test/multitenancy_test.exs b/test/multitenancy_test.exs index afd5cbe5..80f367d2 100644 --- a/test/multitenancy_test.exs +++ b/test/multitenancy_test.exs @@ -2,7 +2,7 @@ defmodule AshPostgres.Test.MultitenancyTest do use AshPostgres.RepoCase, async: false require Ash.Query - alias AshPostgres.MultitenancyTest.{Org, Post, User} + alias AshPostgres.MultitenancyTest.{Org, Post, User, CompositeKeyPost} alias AshPostgres.Test.Post, as: GlobalPost setup do @@ -125,6 +125,15 @@ defmodule AshPostgres.Test.MultitenancyTest do ) end + test "composite key multitenancy works", %{org1: org1} do + CompositeKeyPost + |> Ash.Changeset.for_create(:create, %{title: "foo"}) + |> Ash.Changeset.set_tenant(org1) + |> Ash.create!() + + assert [_] = CompositeKeyPost |> Ash.Query.set_tenant(org1) |> Ash.read!() + end + test "loading attribute multitenant resources from context multitenant resources works" do org = Org diff --git a/test/support/multitenancy/domain.ex b/test/support/multitenancy/domain.ex index 2394c234..85f078da 100644 --- a/test/support/multitenancy/domain.ex +++ b/test/support/multitenancy/domain.ex @@ -9,6 +9,7 @@ defmodule AshPostgres.MultitenancyTest.Domain do resource(AshPostgres.MultitenancyTest.PostLink) resource(AshPostgres.MultitenancyTest.NonMultitenantPostLink) resource(AshPostgres.MultitenancyTest.CrossTenantPostLink) + resource(AshPostgres.MultitenancyTest.CompositeKeyPost) end authorization do diff --git a/test/support/multitenancy/resources/composite_key_post.ex b/test/support/multitenancy/resources/composite_key_post.ex new file mode 100644 index 00000000..bccd4836 --- /dev/null +++ b/test/support/multitenancy/resources/composite_key_post.ex @@ -0,0 +1,26 @@ +defmodule AshPostgres.MultitenancyTest.CompositeKeyPost do + @moduledoc false + use Ash.Resource, + domain: AshPostgres.MultitenancyTest.Domain, + data_layer: AshPostgres.DataLayer + + postgres do + table "composite_key" + repo AshPostgres.TestRepo + end + + multitenancy do + strategy(:context) + end + + actions do + default_accept(:*) + + defaults([:create, :read, :update, :destroy]) + end + + attributes do + integer_primary_key(:id) + attribute(:title, :string, public?: true, allow_nil?: false, primary_key?: true) + end +end diff --git a/test_snapshot_path/test_repo/extensions.json b/test_snapshot_path/test_repo/extensions.json deleted file mode 100644 index e084bbff..00000000 --- a/test_snapshot_path/test_repo/extensions.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "installed": [ - "ash-functions", - "uuid-ossp", - "pg_trgm", - "citext", - "demo-functions_v1" - ], - "ash_functions_version": 3 -} \ No newline at end of file From 4e08d72b1921a3369f3b5823d9c2cc74d83e2f6f Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Mon, 17 Feb 2025 12:45:14 +0100 Subject: [PATCH 3/6] Partitioning of resource --- lib/migration_generator/phase.ex | 30 ++++++++++++++------ lib/partitioning.ex | 47 +++++++++++++++++++++++--------- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/lib/migration_generator/phase.ex b/lib/migration_generator/phase.ex index 6e735501..618efc51 100644 --- a/lib/migration_generator/phase.ex +++ b/lib/migration_generator/phase.ex @@ -3,13 +3,26 @@ defmodule AshPostgres.MigrationGenerator.Phase do defmodule Create do @moduledoc false - defstruct [:table, :schema, :multitenancy, partitioning: nil, operations: [], commented?: false] + defstruct [ + :table, + :schema, + :multitenancy, + partitioning: nil, + operations: [], + commented?: false + ] import AshPostgres.MigrationGenerator.Operation.Helper, only: [as_atom: 1] - def up(%{schema: schema, table: table, operations: operations, multitenancy: multitenancy, partitioning: partitioning}) do + def up(%{ + schema: schema, + table: table, + operations: operations, + multitenancy: multitenancy, + partitioning: partitioning + }) do if multitenancy.strategy == :context do - arguments = arguments([prefix("prefix()"), options(partitioning: partitioning)]) + arguments = arguments([prefix(true), options(partitioning: partitioning)]) "create table(:#{as_atom(table)}, primary_key: false#{arguments}) do\n" <> Enum.map_join(operations, "\n", fn operation -> operation.__struct__.up(operation) end) <> @@ -38,14 +51,15 @@ defmodule AshPostgres.MigrationGenerator.Phase do end end - def arguments(["",""]), do: "" + def arguments([nil, nil]), do: "" def arguments(arguments), do: ", " <> Enum.join(Enum.reject(arguments, &is_nil(&1)), ",") - def prefix(nil), do: nil - def prefix(schema), do: "prefix: #{schema}" + def prefix(true), do: "prefix: prefix()" + def prefix(schema) when is_binary(schema) and schema != "", do: "prefix: \"#{schema}\"" + def prefix(_), do: nil def options(_options, _acc \\ []) - def options([], []), do: "" + def options([], []), do: nil def options([], acc), do: "options: \"#{Enum.join(acc, " ")}\"" def options([{:partitioning, %{method: method, attribute: attribute}} | rest], acc) do @@ -55,7 +69,7 @@ defmodule AshPostgres.MigrationGenerator.Phase do |> options(acc ++ [option]) end - def options([_| rest], acc) do + def options([_ | rest], acc) do options(rest, acc) end end diff --git a/lib/partitioning.ex b/lib/partitioning.ex index 8775f1b3..175ff48f 100644 --- a/lib/partitioning.ex +++ b/lib/partitioning.ex @@ -29,31 +29,33 @@ defmodule AshPostgres.Partitioning do """ def exists?(resource, opts) do repo = AshPostgres.DataLayer.Info.repo(resource) - key = Keyword.fetch!(opts, :key) - table = AshPostgres.DataLayer.Info.table(resource) - partition_name = table <> "_" <> "#{key}" + partition_name = partition_name(resource, opts) - partition_exists?(repo, resource, partition_name) + partition_exists?(repo, resource, partition_name, opts) end # TBI - defp create_range_partition(repo, resource, opts) do + defp create_range_partition(_repo, _resource, _opts) do end defp create_list_partition(repo, resource, opts) do key = Keyword.fetch!(opts, :key) table = AshPostgres.DataLayer.Info.table(resource) - partition_name = table <> "_" <> "#{key}" + partition_name = partition_name(resource, opts) + + schema = + Keyword.get(opts, :tenant) + |> tenant_schema(resource) - if partition_exists?(repo, resource, partition_name) do + if partition_exists?(repo, resource, partition_name, opts) do {:error, :allready_exists} else Ecto.Adapters.SQL.query( repo, - "CREATE TABLE #{partition_name} PARTITION OF public.#{table} FOR VALUES IN (#{key})" + "CREATE TABLE \"#{schema}\".\"#{partition_name}\" PARTITION OF \"#{schema}\".\"#{table}\" FOR VALUES IN ('#{key}')" ) - if partition_exists?(repo, resource, partition_name) do + if partition_exists?(repo, resource, partition_name, opts) do :ok else {:error, "Unable to create partition"} @@ -62,18 +64,37 @@ defmodule AshPostgres.Partitioning do end # TBI - defp create_hash_partition(repo, resource, opts) do + defp create_hash_partition(_repo, _resource, _opts) do end - defp partition_exists?(repo, resource, parition_name) do + defp partition_exists?(repo, resource, parition_name, opts) do + schema = + Keyword.get(opts, :tenant) + |> tenant_schema(resource) + %Postgrex.Result{} = result = repo |> Ecto.Adapters.SQL.query!( - "select table_name from information_schema.tables t where t.table_schema = 'public' and t.table_name = $1", - [parition_name] + "select table_name from information_schema.tables t where t.table_schema = $1 and t.table_name = $2", + [schema, parition_name] ) result.num_rows > 0 end + + defp partition_name(resource, opts) do + key = Keyword.fetch!(opts, :key) + table = AshPostgres.DataLayer.Info.table(resource) + "#{table}_#{key}" + end + + defp tenant_schema(tenant, resource) do + tenant + |> Ash.ToTenant.to_tenant(resource) + |> case do + nil -> "public" + tenant -> tenant + end + end end From 8b9ea15046f784306677de9735fabd718478dbf7 Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Mon, 17 Feb 2025 12:47:10 +0100 Subject: [PATCH 4/6] spelling --- lib/partitioning.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/partitioning.ex b/lib/partitioning.ex index 175ff48f..cabb3e46 100644 --- a/lib/partitioning.ex +++ b/lib/partitioning.ex @@ -48,7 +48,7 @@ defmodule AshPostgres.Partitioning do |> tenant_schema(resource) if partition_exists?(repo, resource, partition_name, opts) do - {:error, :allready_exists} + {:error, :already_exists} else Ecto.Adapters.SQL.query( repo, From aea852e82c30e90a6ca0eec16ebc5c798cb6a62c Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Thu, 20 Feb 2025 07:39:49 +0100 Subject: [PATCH 5/6] Rename methods --- lib/partitioning.ex | 26 ++++++++++++++++++++------ test/partition_test.exs | 4 ++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/lib/partitioning.ex b/lib/partitioning.ex index cabb3e46..5067c989 100644 --- a/lib/partitioning.ex +++ b/lib/partitioning.ex @@ -27,11 +27,25 @@ defmodule AshPostgres.Partitioning do @doc """ Check if partition exists """ - def exists?(resource, opts) do + def existing_partition?(resource, opts) do repo = AshPostgres.DataLayer.Info.repo(resource) - partition_name = partition_name(resource, opts) - partition_exists?(repo, resource, partition_name, opts) + resource + |> AshPostgres.DataLayer.Info.partitioning_method() + |> case do + :range -> + false + + :list -> + partition_name = partition_name(resource, opts) + schema_exists?(repo, resource, partition_name, opts) + + :hash -> + false + + unsupported_method -> + raise "Invalid partition method, got: #{unsupported_method}" + end end # TBI @@ -47,7 +61,7 @@ defmodule AshPostgres.Partitioning do Keyword.get(opts, :tenant) |> tenant_schema(resource) - if partition_exists?(repo, resource, partition_name, opts) do + if schema_exists?(repo, resource, partition_name, opts) do {:error, :already_exists} else Ecto.Adapters.SQL.query( @@ -55,7 +69,7 @@ defmodule AshPostgres.Partitioning do "CREATE TABLE \"#{schema}\".\"#{partition_name}\" PARTITION OF \"#{schema}\".\"#{table}\" FOR VALUES IN ('#{key}')" ) - if partition_exists?(repo, resource, partition_name, opts) do + if schema_exists?(repo, resource, partition_name, opts) do :ok else {:error, "Unable to create partition"} @@ -67,7 +81,7 @@ defmodule AshPostgres.Partitioning do defp create_hash_partition(_repo, _resource, _opts) do end - defp partition_exists?(repo, resource, parition_name, opts) do + defp schema_exists?(repo, resource, parition_name, opts) do schema = Keyword.get(opts, :tenant) |> tenant_schema(resource) diff --git a/test/partition_test.exs b/test/partition_test.exs index 07fe96fe..564e36f9 100644 --- a/test/partition_test.exs +++ b/test/partition_test.exs @@ -3,9 +3,9 @@ defmodule AshPostgres.PartitionTest do alias AshPostgres.Test.PartitionedPost test "seeding data works" do - assert false == AshPostgres.Partitioning.exists?(PartitionedPost, key: 1) + assert false == AshPostgres.Partitioning.existing_partition?(PartitionedPost, key: 1) assert :ok == AshPostgres.Partitioning.create_partition(PartitionedPost, key: 1) - assert true == AshPostgres.Partitioning.exists?(PartitionedPost, key: 1) + assert true == AshPostgres.Partitioning.existing_partition?(PartitionedPost, key: 1) Ash.Seed.seed!(%PartitionedPost{key: 1}) From 13af63c5aa3dc89ac79730c5c5c542c40084dbdc Mon Sep 17 00:00:00 2001 From: "morten.lund@maskon.no" Date: Fri, 21 Feb 2025 10:09:17 +0100 Subject: [PATCH 6/6] Merge branch 'main' of https://github.com/m0rt3nlund/ash_postgres into Partitioning-support