354 lines
14 KiB
Python
354 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import socket
|
|
import threading
|
|
import time
|
|
from datetime import datetime
|
|
|
|
import database
|
|
from app_logger import get_logger
|
|
from flask import jsonify, session
|
|
from routes.admin_api import admin_api_bp
|
|
from routes.decorators import admin_required
|
|
from services.request_metrics import get_request_metrics_snapshot
|
|
from services.slow_sql_metrics import get_slow_sql_metrics_snapshot
|
|
from services.time_utils import BEIJING_TZ, get_beijing_now
|
|
|
|
logger = get_logger("app")
|
|
|
|
_ADMIN_STATS_CACHE_TTL = max(1.0, float(os.environ.get("ADMIN_STATS_CACHE_TTL_SECONDS", "5")))
|
|
_admin_stats_cache: dict[str, object] = {"expires_at_monotonic": 0.0, "data": None}
|
|
_admin_stats_cache_lock = threading.Lock()
|
|
|
|
_DOCKER_STATS_CACHE_TTL = max(2.0, float(os.environ.get("ADMIN_DOCKER_STATS_CACHE_TTL_SECONDS", "5")))
|
|
_docker_stats_cache: dict[str, object] = {"expires_at_monotonic": 0.0, "data": None}
|
|
_docker_stats_cache_lock = threading.Lock()
|
|
|
|
_REQUEST_METRICS_CACHE_TTL = max(1.0, float(os.environ.get("ADMIN_REQUEST_METRICS_CACHE_TTL_SECONDS", "3")))
|
|
_request_metrics_cache: dict[str, object] = {"expires_at_monotonic": 0.0, "data": None}
|
|
_request_metrics_cache_lock = threading.Lock()
|
|
|
|
_SLOW_SQL_METRICS_CACHE_TTL = max(1.0, float(os.environ.get("ADMIN_SLOW_SQL_METRICS_CACHE_TTL_SECONDS", "3")))
|
|
_slow_sql_metrics_cache: dict[str, object] = {"expires_at_monotonic": 0.0, "data": None}
|
|
_slow_sql_metrics_cache_lock = threading.Lock()
|
|
|
|
|
|
def _get_system_stats_cached() -> dict:
|
|
now = time.monotonic()
|
|
with _admin_stats_cache_lock:
|
|
expires_at = float(_admin_stats_cache.get("expires_at_monotonic") or 0.0)
|
|
cached_data = _admin_stats_cache.get("data")
|
|
if isinstance(cached_data, dict) and now < expires_at:
|
|
return dict(cached_data)
|
|
|
|
fresh_data = database.get_system_stats() or {}
|
|
|
|
with _admin_stats_cache_lock:
|
|
_admin_stats_cache["data"] = dict(fresh_data)
|
|
_admin_stats_cache["expires_at_monotonic"] = now + _ADMIN_STATS_CACHE_TTL
|
|
|
|
return dict(fresh_data)
|
|
|
|
|
|
def _get_request_metrics_cached() -> dict:
|
|
now = time.monotonic()
|
|
with _request_metrics_cache_lock:
|
|
expires_at = float(_request_metrics_cache.get("expires_at_monotonic") or 0.0)
|
|
cached_data = _request_metrics_cache.get("data")
|
|
if isinstance(cached_data, dict) and now < expires_at:
|
|
return dict(cached_data)
|
|
|
|
fresh_data = get_request_metrics_snapshot() or {}
|
|
|
|
with _request_metrics_cache_lock:
|
|
_request_metrics_cache["data"] = dict(fresh_data)
|
|
_request_metrics_cache["expires_at_monotonic"] = now + _REQUEST_METRICS_CACHE_TTL
|
|
|
|
return dict(fresh_data)
|
|
|
|
|
|
def _get_slow_sql_metrics_cached() -> dict:
|
|
now = time.monotonic()
|
|
with _slow_sql_metrics_cache_lock:
|
|
expires_at = float(_slow_sql_metrics_cache.get("expires_at_monotonic") or 0.0)
|
|
cached_data = _slow_sql_metrics_cache.get("data")
|
|
if isinstance(cached_data, dict) and now < expires_at:
|
|
return dict(cached_data)
|
|
|
|
fresh_data = get_slow_sql_metrics_snapshot() or {}
|
|
|
|
with _slow_sql_metrics_cache_lock:
|
|
_slow_sql_metrics_cache["data"] = dict(fresh_data)
|
|
_slow_sql_metrics_cache["expires_at_monotonic"] = now + _SLOW_SQL_METRICS_CACHE_TTL
|
|
|
|
return dict(fresh_data)
|
|
|
|
|
|
@admin_api_bp.route("/stats", methods=["GET"])
|
|
@admin_required
|
|
def get_system_stats():
|
|
"""获取系统统计"""
|
|
stats = _get_system_stats_cached()
|
|
stats["admin_username"] = session.get("admin_username", "admin")
|
|
return jsonify(stats)
|
|
|
|
|
|
@admin_api_bp.route("/request_metrics", methods=["GET"])
|
|
@admin_required
|
|
def get_request_metrics():
|
|
"""获取请求级监控指标"""
|
|
try:
|
|
metrics = _get_request_metrics_cached()
|
|
return jsonify(metrics)
|
|
except Exception as e:
|
|
logger.exception(f"获取请求级监控指标失败: {e}")
|
|
return jsonify({"error": "获取请求级监控指标失败"}), 500
|
|
|
|
|
|
@admin_api_bp.route("/slow_sql_metrics", methods=["GET"])
|
|
@admin_required
|
|
def get_slow_sql_metrics():
|
|
"""获取慢 SQL 监控指标"""
|
|
try:
|
|
metrics = _get_slow_sql_metrics_cached()
|
|
return jsonify(metrics)
|
|
except Exception as e:
|
|
logger.exception(f"获取慢 SQL 监控指标失败: {e}")
|
|
return jsonify({"error": "获取慢 SQL 监控指标失败"}), 500
|
|
|
|
|
|
@admin_api_bp.route("/browser_pool/stats", methods=["GET"])
|
|
@admin_required
|
|
def get_browser_pool_stats():
|
|
"""获取截图线程池状态"""
|
|
try:
|
|
from browser_pool_worker import get_browser_worker_pool
|
|
|
|
pool = get_browser_worker_pool()
|
|
stats = pool.get_stats() or {}
|
|
|
|
worker_details = []
|
|
for worker in stats.get("workers") or []:
|
|
last_ts = float(worker.get("last_active_ts") or 0)
|
|
last_active_at = None
|
|
if last_ts > 0:
|
|
try:
|
|
last_active_at = datetime.fromtimestamp(last_ts, tz=BEIJING_TZ).strftime("%Y-%m-%d %H:%M:%S")
|
|
except Exception:
|
|
last_active_at = None
|
|
|
|
created_ts = worker.get("browser_created_at")
|
|
created_at = None
|
|
if created_ts:
|
|
try:
|
|
created_at = datetime.fromtimestamp(float(created_ts), tz=BEIJING_TZ).strftime("%Y-%m-%d %H:%M:%S")
|
|
except Exception:
|
|
created_at = None
|
|
|
|
worker_details.append(
|
|
{
|
|
"worker_id": worker.get("worker_id"),
|
|
"idle": bool(worker.get("idle")),
|
|
"has_browser": bool(worker.get("has_browser")),
|
|
"total_tasks": int(worker.get("total_tasks") or 0),
|
|
"failed_tasks": int(worker.get("failed_tasks") or 0),
|
|
"browser_use_count": int(worker.get("browser_use_count") or 0),
|
|
"browser_created_at": created_at,
|
|
"browser_created_ts": created_ts,
|
|
"last_active_at": last_active_at,
|
|
"last_active_ts": last_ts,
|
|
"thread_alive": bool(worker.get("thread_alive")),
|
|
}
|
|
)
|
|
|
|
total_workers = len(worker_details) if worker_details else int(stats.get("pool_size") or 0)
|
|
return jsonify(
|
|
{
|
|
"total_workers": total_workers,
|
|
"active_workers": int(stats.get("busy_workers") or 0),
|
|
"idle_workers": int(stats.get("idle_workers") or 0),
|
|
"queue_size": int(stats.get("queue_size") or 0),
|
|
"workers": worker_details,
|
|
"summary": {
|
|
"total_tasks": int(stats.get("total_tasks") or 0),
|
|
"failed_tasks": int(stats.get("failed_tasks") or 0),
|
|
"success_rate": stats.get("success_rate"),
|
|
},
|
|
"server_time_cst": get_beijing_now().strftime("%Y-%m-%d %H:%M:%S"),
|
|
}
|
|
)
|
|
except Exception as e:
|
|
logger.exception(f"[AdminAPI] 获取截图线程池状态失败: {e}")
|
|
return jsonify({"error": "获取截图线程池状态失败"}), 500
|
|
|
|
|
|
def _format_duration(seconds: int) -> str:
|
|
total = max(0, int(seconds or 0))
|
|
days = total // 86400
|
|
hours = (total % 86400) // 3600
|
|
minutes = (total % 3600) // 60
|
|
if days > 0:
|
|
return f"{days}天{hours}小时{minutes}分钟"
|
|
if hours > 0:
|
|
return f"{hours}小时{minutes}分钟"
|
|
return f"{minutes}分钟"
|
|
|
|
|
|
def _fill_host_service_stats(docker_status: dict) -> None:
|
|
import psutil
|
|
|
|
process = psutil.Process(os.getpid())
|
|
memory_info = process.memory_info()
|
|
virtual_memory = psutil.virtual_memory()
|
|
|
|
rss_bytes = float(memory_info.rss or 0)
|
|
total_bytes = float(virtual_memory.total or 0)
|
|
memory_percent = (rss_bytes / total_bytes * 100.0) if total_bytes > 0 else 0.0
|
|
|
|
docker_status.update(
|
|
{
|
|
"running": True,
|
|
"status": "Host Service",
|
|
"container_name": f"host:{socket.gethostname()}",
|
|
"uptime": _format_duration(int(time.time() - float(process.create_time() or time.time()))),
|
|
"memory_usage": f"{rss_bytes / 1024 / 1024:.2f} MB",
|
|
"memory_limit": f"{total_bytes / 1024 / 1024 / 1024:.2f} GB" if total_bytes > 0 else "N/A",
|
|
"memory_percent": f"{memory_percent:.2f}%",
|
|
"cpu_percent": f"{max(0.0, float(process.cpu_percent(interval=0.1))):.2f}%",
|
|
}
|
|
)
|
|
|
|
|
|
@admin_api_bp.route("/docker_stats", methods=["GET"])
|
|
@admin_required
|
|
def get_docker_stats():
|
|
"""获取容器运行状态（非容器部署时返回当前服务进程状态）"""
|
|
now = time.monotonic()
|
|
with _docker_stats_cache_lock:
|
|
expires_at = float(_docker_stats_cache.get("expires_at_monotonic") or 0.0)
|
|
cached_data = _docker_stats_cache.get("data")
|
|
if isinstance(cached_data, dict) and now < expires_at:
|
|
return jsonify(dict(cached_data))
|
|
|
|
docker_status = {
|
|
"running": False,
|
|
"container_name": "N/A",
|
|
"uptime": "N/A",
|
|
"memory_usage": "N/A",
|
|
"memory_limit": "N/A",
|
|
"memory_percent": "N/A",
|
|
"cpu_percent": "N/A",
|
|
"status": "Unknown",
|
|
}
|
|
|
|
try:
|
|
if os.path.exists("/.dockerenv"):
|
|
docker_status["running"] = True
|
|
|
|
try:
|
|
with open("/etc/hostname", "r", encoding="utf-8") as f:
|
|
docker_status["container_name"] = f.read().strip() or "N/A"
|
|
except Exception as e:
|
|
logger.debug(f"读取容器名称失败: {e}")
|
|
|
|
try:
|
|
if os.path.exists("/sys/fs/cgroup/memory.current"):
|
|
with open("/sys/fs/cgroup/memory.current", "r", encoding="utf-8") as f:
|
|
mem_total = int(f.read().strip())
|
|
|
|
cache = 0
|
|
if os.path.exists("/sys/fs/cgroup/memory.stat"):
|
|
with open("/sys/fs/cgroup/memory.stat", "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
if line.startswith("inactive_file "):
|
|
cache = int(line.split()[1])
|
|
break
|
|
|
|
mem_bytes = max(0, mem_total - cache)
|
|
docker_status["memory_usage"] = f"{mem_bytes / 1024 / 1024:.2f} MB"
|
|
|
|
if os.path.exists("/sys/fs/cgroup/memory.max"):
|
|
with open("/sys/fs/cgroup/memory.max", "r", encoding="utf-8") as f:
|
|
limit_str = f.read().strip()
|
|
if limit_str != "max":
|
|
limit_bytes = int(limit_str)
|
|
if limit_bytes > 0:
|
|
docker_status["memory_limit"] = f"{limit_bytes / 1024 / 1024 / 1024:.2f} GB"
|
|
docker_status["memory_percent"] = f"{mem_bytes / limit_bytes * 100:.2f}%"
|
|
elif os.path.exists("/sys/fs/cgroup/memory/memory.usage_in_bytes"):
|
|
with open("/sys/fs/cgroup/memory/memory.usage_in_bytes", "r", encoding="utf-8") as f:
|
|
mem_bytes = int(f.read().strip())
|
|
docker_status["memory_usage"] = f"{mem_bytes / 1024 / 1024:.2f} MB"
|
|
|
|
with open("/sys/fs/cgroup/memory/memory.limit_in_bytes", "r", encoding="utf-8") as f:
|
|
limit_bytes = int(f.read().strip())
|
|
if 0 < limit_bytes < 1e18:
|
|
docker_status["memory_limit"] = f"{limit_bytes / 1024 / 1024 / 1024:.2f} GB"
|
|
docker_status["memory_percent"] = f"{mem_bytes / limit_bytes * 100:.2f}%"
|
|
except Exception as e:
|
|
logger.debug(f"读取容器内存信息失败: {e}")
|
|
|
|
try:
|
|
if os.path.exists("/sys/fs/cgroup/cpu.stat"):
|
|
usage1 = 0
|
|
with open("/sys/fs/cgroup/cpu.stat", "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
if line.startswith("usage_usec"):
|
|
usage1 = int(line.split()[1])
|
|
break
|
|
|
|
time.sleep(0.1)
|
|
|
|
usage2 = 0
|
|
with open("/sys/fs/cgroup/cpu.stat", "r", encoding="utf-8") as f:
|
|
for line in f:
|
|
if line.startswith("usage_usec"):
|
|
usage2 = int(line.split()[1])
|
|
break
|
|
|
|
cpu_percent = (usage2 - usage1) / 0.1 / 1e6 * 100
|
|
docker_status["cpu_percent"] = f"{max(0.0, cpu_percent):.2f}%"
|
|
elif os.path.exists("/sys/fs/cgroup/cpu/cpuacct.usage"):
|
|
with open("/sys/fs/cgroup/cpu/cpuacct.usage", "r", encoding="utf-8") as f:
|
|
usage1 = int(f.read().strip())
|
|
|
|
time.sleep(0.1)
|
|
|
|
with open("/sys/fs/cgroup/cpu/cpuacct.usage", "r", encoding="utf-8") as f:
|
|
usage2 = int(f.read().strip())
|
|
|
|
cpu_percent = (usage2 - usage1) / 0.1 / 1e9 * 100
|
|
docker_status["cpu_percent"] = f"{max(0.0, cpu_percent):.2f}%"
|
|
except Exception as e:
|
|
logger.debug(f"读取容器CPU信息失败: {e}")
|
|
|
|
try:
|
|
with open("/proc/uptime", "r", encoding="utf-8") as f:
|
|
system_uptime = float(f.read().split()[0])
|
|
|
|
with open("/proc/1/stat", "r", encoding="utf-8") as f:
|
|
stat = f.read().split()
|
|
starttime_jiffies = int(stat[21])
|
|
|
|
clk_tck = os.sysconf(os.sysconf_names["SC_CLK_TCK"])
|
|
uptime_seconds = int(system_uptime - (starttime_jiffies / clk_tck))
|
|
docker_status["uptime"] = _format_duration(uptime_seconds)
|
|
except Exception as e:
|
|
logger.debug(f"获取容器运行时间失败: {e}")
|
|
|
|
docker_status["status"] = "Running"
|
|
else:
|
|
_fill_host_service_stats(docker_status)
|
|
except Exception as e:
|
|
logger.exception(f"获取容器/服务状态失败: {e}")
|
|
docker_status["status"] = f"Error: {e}"
|
|
|
|
with _docker_stats_cache_lock:
|
|
_docker_stats_cache["data"] = dict(docker_status)
|
|
_docker_stats_cache["expires_at_monotonic"] = now + _DOCKER_STATS_CACHE_TTL
|
|
|
|
return jsonify(docker_status)
|