Phase 1 first chunk: inventory schema + DO droplet sync
Models: - cloud_projects: arcadia-cloud's mirror of DO Projects, indexed by (provider, provider_id); tenant_id + purpose classify each project. - cloud_resources: single unified resource table; kind-specific bits in attrs JSONB; first_seen_at / last_seen_at / stale_strike_count drive three-strike deletion. - cloud_resource_events: append-only audit (discovered, updated, deleted, drift_detected, tagged, restored). ArcadiaCloud.Cloud context owns the single upsert chokepoint that: - inserts new with `discovered` event - updates existing only when meaningful fields change - restores tombstoned rows seen again - bumps last_seen_at and resets strike count mark_stale/3 implements the three-strike rule. ArcadiaCloud.DigitalOcean.Client is a Req wrapper with auto-pagination. Per-purpose token resolution via .Tokens (phase 1: env DO_API_TOKEN; phase 2: vault). Per project_arcadia_cloud memory the long-term shape is one PAT per queue purpose for rate-limit isolation. ArcadiaCloud.Sync.Bootstrap ensures the skyai-internal DO Project exists on first sync, idempotent thereafter. ArcadiaCloud.Sync.DropletsWorker runs full droplet sync on the cloud_sync_full Oban queue. InventoryController wired to real data: platform_admin sees all, tenants see only their scope. Live smoke test against real DO: 5 droplets synced; skyai-internal project auto-created; events written; endpoint returns scoped results. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
51
lib/arcadia_cloud/sync/bootstrap.ex
Normal file
51
lib/arcadia_cloud/sync/bootstrap.ex
Normal file
@@ -0,0 +1,51 @@
|
||||
defmodule ArcadiaCloud.Sync.Bootstrap do
|
||||
@moduledoc """
|
||||
First-run bootstrap: ensure the `skyai-internal` DO Project exists and
|
||||
is registered in our `cloud_projects` table. Resources discovered before
|
||||
any tenant project exists land here by default.
|
||||
|
||||
Idempotent: safe to call on every sync.
|
||||
"""
|
||||
|
||||
alias ArcadiaCloud.Cloud
|
||||
alias ArcadiaCloud.DigitalOcean.Client
|
||||
|
||||
@internal_name "skyai-internal"
|
||||
@internal_purpose "skyai-infra"
|
||||
@do_purpose "Service or API"
|
||||
|
||||
def ensure_skyai_internal do
|
||||
case Cloud.skyai_internal_project() do
|
||||
%{} = project ->
|
||||
{:ok, project}
|
||||
|
||||
nil ->
|
||||
with {:ok, do_project} <- find_or_create_do_project() do
|
||||
{:ok, _local} =
|
||||
Cloud.ensure_project(%{
|
||||
provider: "digitalocean",
|
||||
provider_id: do_project["id"],
|
||||
name: @internal_name,
|
||||
purpose: @internal_purpose,
|
||||
metadata: %{
|
||||
do_purpose: do_project["purpose"],
|
||||
description: do_project["description"]
|
||||
}
|
||||
})
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp find_or_create_do_project do
|
||||
with {:ok, projects} <- Client.list_projects() do
|
||||
case Enum.find(projects, &(&1["name"] == @internal_name)) do
|
||||
nil ->
|
||||
Client.create_project(@internal_name, @do_purpose,
|
||||
"Sky AI internal infrastructure (auto-created by arcadia-cloud)")
|
||||
|
||||
existing ->
|
||||
{:ok, existing}
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
78
lib/arcadia_cloud/sync/droplets_worker.ex
Normal file
78
lib/arcadia_cloud/sync/droplets_worker.ex
Normal file
@@ -0,0 +1,78 @@
|
||||
defmodule ArcadiaCloud.Sync.DropletsWorker do
|
||||
@moduledoc """
|
||||
Full sync of DigitalOcean droplets.
|
||||
|
||||
Idempotent: re-runs are no-ops absent state changes. Three-strike deletion
|
||||
for resources that vanish from N consecutive syncs.
|
||||
|
||||
Tenant attribution is derived in two passes — phase 1 (this worker) tags
|
||||
all discovered resources to the `skyai-internal` project by default.
|
||||
Phase 1+ will resolve actual tenant projects via DO Projects API.
|
||||
"""
|
||||
|
||||
use Oban.Worker, queue: :cloud_sync_full, max_attempts: 3
|
||||
|
||||
alias ArcadiaCloud.Cloud
|
||||
alias ArcadiaCloud.DigitalOcean.Client
|
||||
alias ArcadiaCloud.Sync.Bootstrap
|
||||
|
||||
@kind "droplet"
|
||||
@provider "digitalocean"
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(_job) do
|
||||
sync_started_at = DateTime.utc_now() |> DateTime.truncate(:second)
|
||||
|
||||
with {:ok, _project} <- Bootstrap.ensure_skyai_internal(),
|
||||
{:ok, droplets} <- Client.list_droplets() do
|
||||
internal = Cloud.skyai_internal_project()
|
||||
|
||||
Enum.each(droplets, fn d ->
|
||||
Cloud.upsert_resource(normalize(d, internal, sync_started_at))
|
||||
end)
|
||||
|
||||
Cloud.mark_stale(@kind, sync_started_at)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
# ---- normalization --------------------------------------------------------
|
||||
|
||||
defp normalize(d, project, now) do
|
||||
%{
|
||||
provider: @provider,
|
||||
provider_id: to_string(d["id"]),
|
||||
kind: @kind,
|
||||
name: d["name"],
|
||||
region: get_in(d, ["region", "slug"]),
|
||||
status: normalize_status(d["status"]),
|
||||
size_slug: d["size_slug"],
|
||||
cloud_project_id: project && project.id,
|
||||
tags: d["tags"] || [],
|
||||
attrs: %{
|
||||
memory_mb: d["memory"],
|
||||
vcpus: d["vcpus"],
|
||||
disk_gb: d["disk"],
|
||||
image: take_image(d["image"]),
|
||||
networks: d["networks"],
|
||||
features: d["features"],
|
||||
do_created_at: d["created_at"]
|
||||
},
|
||||
first_seen_at: now,
|
||||
last_seen_at: now
|
||||
}
|
||||
end
|
||||
|
||||
defp normalize_status("active"), do: "active"
|
||||
defp normalize_status("off"), do: "off"
|
||||
defp normalize_status("new"), do: "provisioning"
|
||||
defp normalize_status("archive"), do: "archived"
|
||||
defp normalize_status(other) when is_binary(other), do: other
|
||||
defp normalize_status(_), do: "unknown"
|
||||
|
||||
defp take_image(nil), do: nil
|
||||
|
||||
defp take_image(image) when is_map(image) do
|
||||
Map.take(image, ["id", "name", "slug", "distribution"])
|
||||
end
|
||||
end
|
||||
Reference in New Issue
Block a user