Two patterns added: 1. ProjectsWorker now does URN-discover for kinds without a dedicated sync worker (spaces_bucket, managed_db, k8s_cluster, etc.). For these, it inserts a minimal placeholder row when the URN points to something not yet in inventory. Kinds with dedicated workers (droplet, snapshot, volume, etc.) still get attribution-only — the worker is source of truth for richer attrs. Implemented by splitting attribute_or_discover/4 on a @dedicated_kinds whitelist. 2. New BackupsWorker pulls /v2/droplets/:id/backups for each active droplet. DO automated backups aren't in /v2/snapshots; they live per droplet. Cron: hourly at :41. Kind="droplet_backup". URN normalization extended for two more aliases DO emits: "volumesnapshot" → snapshot (was creating a duplicate row) "image" → snapshot (DO droplet snapshots show as do:image:id) Billing.find_resource/1 gets a kind-specific clause for droplet_backup that matches to the parent droplet by name, since invoice lines for backups read "<droplet-name> (Weekly Backup Services)" — the line is a per-droplet subscription, not a per-backup-snapshot fee. Live verified on the same April 2026 invoice: - 6 Spaces buckets discovered via URN (account has 6, only 1 visible in the invoice as the $5 subscription line — that's account-level so it can't tie to a specific bucket, expected). - 4 droplet backups discovered via BackupsWorker; the git.sky-ai.com backup line now matches (repo.sky-ai.com backup line can't match — that droplet was destroyed). - Of 16 unmatched lines: 11 are destroyed historic resources, 1 is GST, 1 is the account-level Spaces subscription, 3 are likely tiny snapshot name variances. Effectively ~100% of currently-existing billable resources match. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
149 lines
4.8 KiB
Elixir
149 lines
4.8 KiB
Elixir
defmodule ArcadiaCloud.Sync.ProjectsWorker do
|
|
@moduledoc """
|
|
Sync of DigitalOcean Projects → cloud_projects table.
|
|
|
|
Two-pass:
|
|
1. Upsert every DO project locally (purpose derives from name pattern).
|
|
2. For each known project, fetch its resource memberships and update
|
|
cloud_resources.cloud_project_id + tenant_id accordingly.
|
|
|
|
Tenant attribution: a DO project named `tenant-<uuid>` maps to that
|
|
tenant. `skyai-internal` is the platform tenant (tenant_id = nil).
|
|
Everything else has tenant_id = nil and is operator-classified later.
|
|
"""
|
|
|
|
use Oban.Worker, queue: :cloud_sync_full, max_attempts: 3
|
|
|
|
import Ecto.Query
|
|
|
|
alias ArcadiaCloud.Cloud
|
|
alias ArcadiaCloud.Cloud.CloudResource
|
|
alias ArcadiaCloud.DigitalOcean.Client
|
|
alias ArcadiaCloud.Repo
|
|
|
|
@impl Oban.Worker
|
|
def perform(_job) do
|
|
with {:ok, do_projects} <- Client.list_projects() do
|
|
Enum.each(do_projects, &sync_project/1)
|
|
attribute_resources(do_projects)
|
|
:ok
|
|
end
|
|
end
|
|
|
|
defp sync_project(do_project) do
|
|
Cloud.ensure_project(%{
|
|
provider: "digitalocean",
|
|
provider_id: do_project["id"],
|
|
name: do_project["name"],
|
|
purpose: derive_purpose(do_project["name"]),
|
|
metadata: %{
|
|
do_purpose: do_project["purpose"],
|
|
environment: do_project["environment"],
|
|
description: do_project["description"],
|
|
is_default: do_project["is_default"]
|
|
}
|
|
})
|
|
end
|
|
|
|
defp derive_purpose("skyai-internal"), do: "skyai-infra"
|
|
defp derive_purpose("tenant-" <> _rest), do: "tenant-workload"
|
|
defp derive_purpose(_), do: "shared-services"
|
|
|
|
# ---- attribution ----------------------------------------------------------
|
|
|
|
defp attribute_resources(do_projects) do
|
|
Enum.each(do_projects, fn do_project ->
|
|
local = Cloud.get_project_by_provider("digitalocean", do_project["id"])
|
|
|
|
if local do
|
|
case Client.list_project_resources(do_project["id"]) do
|
|
{:ok, resources} -> attribute_urns(resources, local)
|
|
_ -> :noop
|
|
end
|
|
end
|
|
end)
|
|
end
|
|
|
|
# Kinds with a dedicated sync worker — ProjectsWorker only updates attribution
|
|
# for these, never inserts (the worker is the source of truth for richer attrs).
|
|
@dedicated_kinds ~w(droplet volume snapshot floating_ip firewall load_balancer dns_zone)
|
|
|
|
defp attribute_urns(urns, %{id: project_id} = local) do
|
|
tenant_id = tenant_id_for(local)
|
|
|
|
Enum.each(urns, fn %{"urn" => urn} ->
|
|
case parse_urn(urn) do
|
|
{kind, provider_id} ->
|
|
attribute_or_discover(kind, provider_id, project_id, tenant_id)
|
|
|
|
_ ->
|
|
:skip
|
|
end
|
|
end)
|
|
end
|
|
|
|
# For kinds with a dedicated worker, just update attribution. For everything
|
|
# else (spaces_bucket, managed_db, k8s_cluster, etc.) insert a minimal
|
|
# placeholder so the resource shows up in inventory + cost matching.
|
|
defp attribute_or_discover(kind, provider_id, project_id, tenant_id) do
|
|
if kind in @dedicated_kinds do
|
|
update_resource_attribution(kind, provider_id, project_id, tenant_id)
|
|
else
|
|
ensure_via_urn(kind, provider_id, project_id, tenant_id)
|
|
end
|
|
end
|
|
|
|
defp update_resource_attribution(kind, provider_id, project_id, tenant_id) do
|
|
from(r in CloudResource,
|
|
where:
|
|
r.provider == "digitalocean" and r.kind == ^kind and r.provider_id == ^provider_id and
|
|
is_nil(r.deleted_at)
|
|
)
|
|
|> Repo.update_all(set: [cloud_project_id: project_id, tenant_id: tenant_id])
|
|
end
|
|
|
|
defp ensure_via_urn(kind, provider_id, project_id, tenant_id) do
|
|
now = DateTime.utc_now() |> DateTime.truncate(:second)
|
|
|
|
Cloud.upsert_resource(
|
|
%{
|
|
provider: "digitalocean",
|
|
provider_id: provider_id,
|
|
kind: kind,
|
|
name: provider_id,
|
|
status: "active",
|
|
cloud_project_id: project_id,
|
|
tenant_id: tenant_id,
|
|
attrs: %{discovered_via: "urn_membership"},
|
|
first_seen_at: now,
|
|
last_seen_at: now
|
|
},
|
|
source: "projects_urn"
|
|
)
|
|
end
|
|
|
|
# "do:droplet:567897199" → {"droplet", "567897199"}
|
|
defp parse_urn("do:" <> rest) do
|
|
case String.split(rest, ":", parts: 2) do
|
|
[kind, id] -> {normalize_kind(kind), id}
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_urn(_), do: nil
|
|
|
|
# DO URN uses singular common nouns; our cloud_resources.kind names some
|
|
# things more explicitly to disambiguate (e.g. dns_zone vs dns_record).
|
|
defp normalize_kind("domain"), do: "dns_zone"
|
|
defp normalize_kind("space"), do: "spaces_bucket"
|
|
defp normalize_kind("floatingip"), do: "floating_ip"
|
|
defp normalize_kind("loadbalancer"), do: "load_balancer"
|
|
defp normalize_kind("dbaas"), do: "managed_db"
|
|
defp normalize_kind("volumesnapshot"), do: "snapshot"
|
|
defp normalize_kind("image"), do: "snapshot"
|
|
defp normalize_kind(other), do: other
|
|
|
|
defp tenant_id_for(%{name: "tenant-" <> tenant_uuid}), do: tenant_uuid
|
|
defp tenant_id_for(_), do: nil
|
|
end
|