fix(storage): make queue and object ownership durable

migrations/025_storage_object_lifecycle.sql

@@ -0,0 +1,101 @@
+ALTER TABLE tasks
+  ADD COLUMN deletion_started_at TIMESTAMPTZ,
+  ADD COLUMN deletion_reason VARCHAR(32);
+
+CREATE INDEX tasks_deletion_pending
+  ON tasks(deletion_started_at)
+  WHERE deletion_started_at IS NOT NULL;
+
+CREATE TABLE storage_objects (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  task_id UUID NOT NULL,
+  task_file_id UUID,
+  object_kind VARCHAR(32) NOT NULL,
+  state VARCHAR(20) NOT NULL DEFAULT 'staging',
+  backend VARCHAR(16) NOT NULL,
+  storage_endpoint_id UUID REFERENCES storage_endpoints(id) ON DELETE RESTRICT,
+  object_key TEXT NOT NULL,
+  storage_etag TEXT,
+  size_bytes BIGINT,
+  lease_owner UUID,
+  lease_until TIMESTAMPTZ,
+  delete_attempts INTEGER NOT NULL DEFAULT 0,
+  next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  last_error TEXT,
+  published_at TIMESTAMPTZ,
+  deleted_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT storage_objects_kind_check
+    CHECK (object_kind IN ('result', 'zip_attempt', 'input', 'input_dir', 'legacy_zip')),
+  CONSTRAINT storage_objects_state_check
+    CHECK (state IN ('staging', 'published', 'delete_pending', 'deleted')),
+  CONSTRAINT storage_objects_backend_check
+    CHECK (backend IN ('s3', 'local', 'local_dir')),
+  CONSTRAINT storage_objects_attempts_check
+    CHECK (delete_attempts >= 0),
+  CONSTRAINT storage_objects_size_check
+    CHECK (size_bytes IS NULL OR size_bytes >= 0),
+  CONSTRAINT storage_objects_endpoint_check
+    CHECK (
+      (backend = 's3' AND storage_endpoint_id IS NOT NULL)
+      OR (backend IN ('local', 'local_dir') AND storage_endpoint_id IS NULL)
+    ),
+  CONSTRAINT storage_objects_lease_pair_check
+    CHECK ((lease_owner IS NULL) = (lease_until IS NULL))
+);
+
+CREATE UNIQUE INDEX storage_objects_locator_unique
+  ON storage_objects(
+    backend,
+    COALESCE(storage_endpoint_id, '00000000-0000-0000-0000-000000000000'::uuid),
+    object_key
+  );
+
+CREATE INDEX storage_objects_cleanup_ready
+  ON storage_objects(next_attempt_at, created_at)
+  WHERE state IN ('staging', 'delete_pending');
+
+CREATE INDEX storage_objects_task
+  ON storage_objects(task_id, state);
+
+INSERT INTO storage_objects (
+  task_id, task_file_id, object_kind, state,
+  backend, storage_endpoint_id, object_key, storage_etag,
+  size_bytes, published_at
+)
+SELECT
+  file.task_id,
+  file.id,
+  'result',
+  'published',
+  file.storage_backend,
+  file.storage_endpoint_id,
+  COALESCE(file.storage_key, file.storage_path),
+  file.storage_etag,
+  file.compressed_size,
+  COALESCE(file.completed_at, file.created_at)
+FROM task_files AS file
+WHERE file.status = 'completed'
+  AND COALESCE(file.storage_key, file.storage_path) IS NOT NULL
+ON CONFLICT DO NOTHING;
+
+INSERT INTO storage_objects (
+  task_id, object_kind, state,
+  backend, storage_endpoint_id, object_key, storage_etag,
+  size_bytes, published_at
+)
+SELECT
+  task.id,
+  'zip_attempt',
+  'published',
+  task.zip_storage_backend,
+  task.zip_storage_endpoint_id,
+  task.zip_storage_key,
+  task.zip_storage_etag,
+  task.zip_size,
+  COALESCE(task.completed_at, task.created_at)
+FROM tasks AS task
+WHERE task.zip_storage_backend IS NOT NULL
+  AND task.zip_storage_key IS NOT NULL
+ON CONFLICT DO NOTHING;