fix(storage): make queue and object ownership durable

migrations/024_task_queue_outbox.sql

@@ -0,0 +1,28 @@
+CREATE TABLE task_queue_outbox (
+  task_id UUID PRIMARY KEY REFERENCES tasks(id) ON DELETE CASCADE,
+  status VARCHAR(20) NOT NULL DEFAULT 'pending',
+  attempts INTEGER NOT NULL DEFAULT 0,
+  next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  lease_owner UUID,
+  lease_until TIMESTAMPTZ,
+  last_error TEXT,
+  delivered_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT task_queue_outbox_status_check
+    CHECK (status IN ('pending', 'delivering', 'delivered', 'dead')),
+  CONSTRAINT task_queue_outbox_attempts_check
+    CHECK (attempts >= 0),
+  CONSTRAINT task_queue_outbox_lease_pair_check
+    CHECK ((lease_owner IS NULL) = (lease_until IS NULL))
+);
+
+CREATE INDEX task_queue_outbox_ready
+  ON task_queue_outbox(next_attempt_at, created_at)
+  WHERE status IN ('pending', 'delivering');
+
+INSERT INTO task_queue_outbox (task_id)
+SELECT id
+FROM tasks
+WHERE status = 'pending'
+ON CONFLICT (task_id) DO NOTHING;

migrations/025_storage_object_lifecycle.sql

@@ -0,0 +1,101 @@
+ALTER TABLE tasks
+  ADD COLUMN deletion_started_at TIMESTAMPTZ,
+  ADD COLUMN deletion_reason VARCHAR(32);
+
+CREATE INDEX tasks_deletion_pending
+  ON tasks(deletion_started_at)
+  WHERE deletion_started_at IS NOT NULL;
+
+CREATE TABLE storage_objects (
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+  task_id UUID NOT NULL,
+  task_file_id UUID,
+  object_kind VARCHAR(32) NOT NULL,
+  state VARCHAR(20) NOT NULL DEFAULT 'staging',
+  backend VARCHAR(16) NOT NULL,
+  storage_endpoint_id UUID REFERENCES storage_endpoints(id) ON DELETE RESTRICT,
+  object_key TEXT NOT NULL,
+  storage_etag TEXT,
+  size_bytes BIGINT,
+  lease_owner UUID,
+  lease_until TIMESTAMPTZ,
+  delete_attempts INTEGER NOT NULL DEFAULT 0,
+  next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  last_error TEXT,
+  published_at TIMESTAMPTZ,
+  deleted_at TIMESTAMPTZ,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT storage_objects_kind_check
+    CHECK (object_kind IN ('result', 'zip_attempt', 'input', 'input_dir', 'legacy_zip')),
+  CONSTRAINT storage_objects_state_check
+    CHECK (state IN ('staging', 'published', 'delete_pending', 'deleted')),
+  CONSTRAINT storage_objects_backend_check
+    CHECK (backend IN ('s3', 'local', 'local_dir')),
+  CONSTRAINT storage_objects_attempts_check
+    CHECK (delete_attempts >= 0),
+  CONSTRAINT storage_objects_size_check
+    CHECK (size_bytes IS NULL OR size_bytes >= 0),
+  CONSTRAINT storage_objects_endpoint_check
+    CHECK (
+      (backend = 's3' AND storage_endpoint_id IS NOT NULL)
+      OR (backend IN ('local', 'local_dir') AND storage_endpoint_id IS NULL)
+    ),
+  CONSTRAINT storage_objects_lease_pair_check
+    CHECK ((lease_owner IS NULL) = (lease_until IS NULL))
+);
+
+CREATE UNIQUE INDEX storage_objects_locator_unique
+  ON storage_objects(
+    backend,
+    COALESCE(storage_endpoint_id, '00000000-0000-0000-0000-000000000000'::uuid),
+    object_key
+  );
+
+CREATE INDEX storage_objects_cleanup_ready
+  ON storage_objects(next_attempt_at, created_at)
+  WHERE state IN ('staging', 'delete_pending');
+
+CREATE INDEX storage_objects_task
+  ON storage_objects(task_id, state);
+
+INSERT INTO storage_objects (
+  task_id, task_file_id, object_kind, state,
+  backend, storage_endpoint_id, object_key, storage_etag,
+  size_bytes, published_at
+)
+SELECT
+  file.task_id,
+  file.id,
+  'result',
+  'published',
+  file.storage_backend,
+  file.storage_endpoint_id,
+  COALESCE(file.storage_key, file.storage_path),
+  file.storage_etag,
+  file.compressed_size,
+  COALESCE(file.completed_at, file.created_at)
+FROM task_files AS file
+WHERE file.status = 'completed'
+  AND COALESCE(file.storage_key, file.storage_path) IS NOT NULL
+ON CONFLICT DO NOTHING;
+
+INSERT INTO storage_objects (
+  task_id, object_kind, state,
+  backend, storage_endpoint_id, object_key, storage_etag,
+  size_bytes, published_at
+)
+SELECT
+  task.id,
+  'zip_attempt',
+  'published',
+  task.zip_storage_backend,
+  task.zip_storage_endpoint_id,
+  task.zip_storage_key,
+  task.zip_storage_etag,
+  task.zip_size,
+  COALESCE(task.completed_at, task.created_at)
+FROM tasks AS task
+WHERE task.zip_storage_backend IS NOT NULL
+  AND task.zip_storage_key IS NOT NULL
+ON CONFLICT DO NOTHING;

migrations/026_idempotency_operation_leases.sql

@@ -0,0 +1,11 @@
+ALTER TABLE idempotency_keys
+  ADD COLUMN lease_owner UUID,
+  ADD COLUMN lease_until TIMESTAMPTZ;
+
+ALTER TABLE idempotency_keys
+  ADD CONSTRAINT idempotency_keys_lease_pair_check
+  CHECK ((lease_owner IS NULL) = (lease_until IS NULL));
+
+CREATE INDEX idempotency_keys_stale_operations
+  ON idempotency_keys(lease_until)
+  WHERE response_status = 0;