From 501e333811c9023a919f983713d8584503d83fa7 Mon Sep 17 00:00:00 2001 From: Giuliano Silvestro Date: Wed, 20 May 2026 15:23:12 +1000 Subject: [PATCH] Phase 3: usage metering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit usage_records — one metered observation per (deployment, resource, period, granularity). resource_kind is the BILLING kind (droplet_hours, spaces_gb_month, ...) so it joins straight to plan_items. sub_attribution jsonb column present from day one for forward-compat (sub-billing) but never written in phase 3. metering_config — per-billing-kind granularity (daily default; flip to hourly when fine-grained billing is wanted) + retention_days. ArcadiaCloud.Metering: - meter_day/1 — walks every deployment-attributed resource, maps its inventory kind to a billing kind, records that day's usage. Idempotent via the unique (deployment, resource, period, granularity) index. - usage_for_period/3 — SUM(qty) GROUP BY billing kind over a date range, returns the %{kind => qty} map the quote engine takes as :usage. Uniform accrual model so everything downstream is just SUM(qty): - droplet -> droplet_hours, 24/day when active (0 when off) - spaces_bucket / snapshot / droplet_backup -> *_gb_month, size/days so the month sums to GB-months - dns_zone -> dns_zones, 1/days so the month sums to the zone count MeteringWorker — Oban cron 01:10 UTC daily, meters the previous complete day. Smoke verified: a pilot deployment with 2 droplets + 2 DNS zones + 3 snapshots attributed; 10 days metered -> 70 usage records; aggregation gave droplet_hours 480 (2x24x10), dns_zones 0.65 (2x10/31), snapshot_gb_month 15.48; fed into the quote engine against Studio — all within allowance so $50 base, no overage (correct for a light partial month). Co-Authored-By: Claude Opus 4.7 (1M context) --- config/config.exs | 2 + lib/arcadia_cloud/metering.ex | 173 ++++++++++++++++++ lib/arcadia_cloud/metering/metering_config.ex | 25 +++ lib/arcadia_cloud/metering/usage_record.ex | 29 +++ lib/arcadia_cloud/sync/metering_worker.ex | 21 +++ .../20260520160000_create_metering.exs | 46 +++++ 6 files changed, 296 insertions(+) create mode 100644 lib/arcadia_cloud/metering.ex create mode 100644 lib/arcadia_cloud/metering/metering_config.ex create mode 100644 lib/arcadia_cloud/metering/usage_record.ex create mode 100644 lib/arcadia_cloud/sync/metering_worker.ex create mode 100644 priv/repo/migrations/20260520160000_create_metering.exs diff --git a/config/config.exs b/config/config.exs index 3b9bec8..f4d946c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -65,6 +65,8 @@ config :arcadia_cloud, Oban, {"41 * * * *", ArcadiaCloud.Sync.BackupsWorker}, # Drift sweep — offset past the :15 resource syncs so it sees fresh data {"20 * * * *", ArcadiaCloud.Sync.DriftDetectionWorker}, + # Metering — daily at 01:10 UTC, meters the previous complete day + {"10 1 * * *", ArcadiaCloud.Sync.MeteringWorker}, # Billing: hourly balance, daily invoice discovery {"7 * * * *", ArcadiaCloud.Sync.BalanceWorker}, {"23 2 * * *", ArcadiaCloud.Sync.BillingHistoryWorker} diff --git a/lib/arcadia_cloud/metering.ex b/lib/arcadia_cloud/metering.ex new file mode 100644 index 0000000..b2f4e69 --- /dev/null +++ b/lib/arcadia_cloud/metering.ex @@ -0,0 +1,173 @@ +defmodule ArcadiaCloud.Metering do + @moduledoc """ + Usage metering. Walks deployment-attributed cloud_resources and records + per-day usage into usage_records. The monthly invoice rollup (and the + quote engine) consume `usage_for_period/3`. + + Metering maps an inventory kind (droplet, spaces_bucket, ...) to a + BILLING kind (droplet_hours, spaces_gb_month, ...) — usage_records and + plan_items both speak billing kinds. + + Uniform accrual model: every kind contributes a daily quantity that + SUMS to the period total. droplet_hours sums to ~720/mo; gb_month kinds + record size/days_in_month so the month sums to GB-months; count kinds + (dns_zones) record 1/days_in_month so the month sums to the count. + Everything downstream is just SUM(qty). + """ + + import Ecto.Query, warn: false + + alias ArcadiaCloud.Repo + alias ArcadiaCloud.Cloud.CloudResource + alias ArcadiaCloud.Metering.{UsageRecord, MeteringConfig} + + @doc """ + Meter a single day for every billable resource. Idempotent — the + unique index on (deployment, resource, period_start, granularity) + means re-running upserts. Returns count of records written. + """ + def meter_day(%Date{} = date) do + period_start = DateTime.new!(date, ~T[00:00:00], "Etc/UTC") + days_in_month = Date.days_in_month(date) + + billable_resources() + |> Enum.reduce(0, fn resource, count -> + case daily_usage(resource, days_in_month) do + nil -> + count + + {billing_kind, qty} -> + upsert_record(resource, billing_kind, period_start, qty) + count + 1 + end + end) + end + + @doc """ + Total usage per billing kind for a deployment over a period. Returns a + map %{billing_kind => total_qty(float)} — the shape the quote engine + takes as `:usage`. + """ + def usage_for_period(deployment_id, %Date{} = from_date, %Date{} = to_date) do + from_dt = DateTime.new!(from_date, ~T[00:00:00], "Etc/UTC") + to_dt = DateTime.new!(to_date, ~T[23:59:59], "Etc/UTC") + + from(u in UsageRecord, + where: + u.deployment_id == ^deployment_id and u.period_start >= ^from_dt and + u.period_start <= ^to_dt, + group_by: u.resource_kind, + select: {u.resource_kind, sum(u.qty)} + ) + |> Repo.all() + |> Map.new(fn {kind, total} -> {kind, to_float(total)} end) + end + + def list_usage_records(deployment_id, opts \\ []) do + from(u in UsageRecord, + where: u.deployment_id == ^deployment_id, + order_by: [desc: u.period_start] + ) + |> maybe_limit(opts[:limit]) + |> Repo.all() + end + + # ---- config --------------------------------------------------------------- + + def get_granularity(resource_kind) do + case Repo.get_by(MeteringConfig, resource_kind: resource_kind) do + nil -> "daily" + config -> config.granularity + end + end + + def upsert_config(resource_kind, granularity, retention_days \\ 400) do + attrs = %{ + resource_kind: resource_kind, + granularity: granularity, + retention_days: retention_days + } + + case Repo.get_by(MeteringConfig, resource_kind: resource_kind) do + nil -> %MeteringConfig{} |> MeteringConfig.changeset(attrs) |> Repo.insert() + cfg -> cfg |> MeteringConfig.changeset(attrs) |> Repo.update() + end + end + + # ---- internals ------------------------------------------------------------ + + defp billable_resources do + from(r in CloudResource, + where: not is_nil(r.deployment_id) and is_nil(r.deleted_at) + ) + |> Repo.all() + end + + # inventory kind => {billing kind, daily qty} + defp daily_usage(%CloudResource{kind: "droplet", status: status}, _dim) do + hours = if status == "active", do: 24, else: 0 + {"droplet_hours", hours} + end + + defp daily_usage(%CloudResource{kind: "spaces_bucket"} = r, dim) do + {"spaces_gb_month", attr_gb(r) / dim} + end + + defp daily_usage(%CloudResource{kind: "snapshot"} = r, dim) do + {"snapshot_gb_month", attr_gb(r) / dim} + end + + defp daily_usage(%CloudResource{kind: "droplet_backup"} = r, dim) do + {"snapshot_gb_month", attr_gb(r) / dim} + end + + defp daily_usage(%CloudResource{kind: "dns_zone"}, dim) do + {"dns_zones", 1.0 / dim} + end + + defp daily_usage(_resource, _dim), do: nil + + defp attr_gb(%CloudResource{attrs: attrs}) do + (attrs && (attrs["size_gigabytes"] || attrs["disk_gb"])) || 0 + end + + defp upsert_record(resource, billing_kind, period_start, qty) do + granularity = get_granularity(billing_kind) + qty_dec = qty |> to_decimal() + + case Repo.get_by(UsageRecord, + deployment_id: resource.deployment_id, + resource_id: resource.id, + period_start: period_start, + granularity: granularity + ) do + nil -> + %UsageRecord{} + |> UsageRecord.changeset(%{ + deployment_id: resource.deployment_id, + resource_id: resource.id, + resource_kind: billing_kind, + period_start: period_start, + granularity: granularity, + qty: qty_dec + }) + |> Repo.insert!() + + existing -> + existing + |> UsageRecord.changeset(%{qty: qty_dec, resource_kind: billing_kind}) + |> Repo.update!() + end + end + + defp to_decimal(%Decimal{} = d), do: d + defp to_decimal(n) when is_integer(n), do: Decimal.new(n) + defp to_decimal(n) when is_float(n), do: Decimal.from_float(Float.round(n, 6)) + + defp to_float(nil), do: 0.0 + defp to_float(%Decimal{} = d), do: Decimal.to_float(d) + defp to_float(n) when is_number(n), do: n / 1 + + defp maybe_limit(q, nil), do: q + defp maybe_limit(q, n), do: from(u in q, limit: ^n) +end diff --git a/lib/arcadia_cloud/metering/metering_config.ex b/lib/arcadia_cloud/metering/metering_config.ex new file mode 100644 index 0000000..ee146f1 --- /dev/null +++ b/lib/arcadia_cloud/metering/metering_config.ex @@ -0,0 +1,25 @@ +defmodule ArcadiaCloud.Metering.MeteringConfig do + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + @granularities ~w(daily hourly) + + schema "metering_config" do + field :resource_kind, :string + field :granularity, :string, default: "daily" + field :retention_days, :integer, default: 400 + + timestamps(type: :utc_datetime) + end + + def changeset(config, attrs) do + config + |> cast(attrs, [:resource_kind, :granularity, :retention_days]) + |> validate_required([:resource_kind, :granularity]) + |> validate_inclusion(:granularity, @granularities) + |> unique_constraint(:resource_kind) + end +end diff --git a/lib/arcadia_cloud/metering/usage_record.ex b/lib/arcadia_cloud/metering/usage_record.ex new file mode 100644 index 0000000..b36c4ff --- /dev/null +++ b/lib/arcadia_cloud/metering/usage_record.ex @@ -0,0 +1,29 @@ +defmodule ArcadiaCloud.Metering.UsageRecord do + use Ecto.Schema + import Ecto.Changeset + + @primary_key {:id, :binary_id, autogenerate: true} + @foreign_key_type :binary_id + + schema "usage_records" do + field :resource_kind, :string + field :period_start, :utc_datetime + field :granularity, :string, default: "daily" + field :qty, :decimal, default: Decimal.new(0) + field :sub_attribution, :map + + belongs_to :deployment, ArcadiaCloud.Deployments.CloudDeployment + belongs_to :resource, ArcadiaCloud.Cloud.CloudResource + + timestamps(type: :utc_datetime, updated_at: false) + end + + @required ~w(deployment_id resource_kind period_start granularity qty)a + @optional ~w(resource_id sub_attribution)a + + def changeset(rec, attrs) do + rec + |> cast(attrs, @required ++ @optional) + |> validate_required(@required) + end +end diff --git a/lib/arcadia_cloud/sync/metering_worker.ex b/lib/arcadia_cloud/sync/metering_worker.ex new file mode 100644 index 0000000..03889b8 --- /dev/null +++ b/lib/arcadia_cloud/sync/metering_worker.ex @@ -0,0 +1,21 @@ +defmodule ArcadiaCloud.Sync.MeteringWorker do + @moduledoc """ + Daily metering job — meters the previous (complete) UTC day for every + deployment-attributed resource. Runs early each day so the day being + metered is fully over. + """ + + use Oban.Worker, queue: :metering, max_attempts: 3 + + require Logger + + alias ArcadiaCloud.Metering + + @impl Oban.Worker + def perform(_job) do + date = Date.add(Date.utc_today(), -1) + count = Metering.meter_day(date) + Logger.info("[metering] metered #{date}: #{count} usage records") + :ok + end +end diff --git a/priv/repo/migrations/20260520160000_create_metering.exs b/priv/repo/migrations/20260520160000_create_metering.exs new file mode 100644 index 0000000..2eca13b --- /dev/null +++ b/priv/repo/migrations/20260520160000_create_metering.exs @@ -0,0 +1,46 @@ +defmodule ArcadiaCloud.Repo.Migrations.CreateMetering do + use Ecto.Migration + + def change do + # Per-resource-kind metering granularity. Daily by default; flip a + # kind to hourly when fine-grained billing is wanted. + create table(:metering_config, primary_key: false) do + add :id, :binary_id, primary_key: true + add :resource_kind, :string, null: false + add :granularity, :string, null: false, default: "daily" + add :retention_days, :integer, null: false, default: 400 + + timestamps(type: :utc_datetime) + end + + create unique_index(:metering_config, [:resource_kind]) + + # One metered observation per (deployment, resource, period, granularity). + # resource_kind here is the BILLING kind (droplet_hours, spaces_gb_month, + # ...) so it joins straight to plan_items.resource_kind. + create table(:usage_records, primary_key: false) do + add :id, :binary_id, primary_key: true + add :deployment_id, + references(:cloud_deployments, type: :binary_id, on_delete: :delete_all), + null: false + add :resource_id, references(:cloud_resources, type: :binary_id, on_delete: :nilify_all) + add :resource_kind, :string, null: false + add :period_start, :utc_datetime, null: false + add :granularity, :string, null: false, default: "daily" + add :qty, :decimal, null: false, default: 0 + # forward-compat: tenants opting into sub-billing later can attribute + # usage to their own end-users without a schema migration. Never + # written in phase 3. + add :sub_attribution, :map + + timestamps(type: :utc_datetime, updated_at: false) + end + + create unique_index(:usage_records, [:deployment_id, :resource_id, :period_start, :granularity], + name: :usage_records_unique + ) + + create index(:usage_records, [:deployment_id, :period_start]) + create index(:usage_records, [:resource_kind]) + end +end