461 lines
17 KiB
Python
461 lines
17 KiB
Python
"""匠厂数据管理展示元数据（_jiangchang_tables / _jiangchang_columns）。
|
||
|
||
SQLite 无原生表/字段 COMMENT；中文展示名称通过这两张元数据表提供给宿主「数据管理」页面。
|
||
字段在界面上的顺序由 CREATE TABLE / PRAGMA table_info(cid) 决定，**不**使用 display_order 排序。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from typing import Iterable
|
||
|
||
INTERNAL_METADATA_TABLES = frozenset({"_jiangchang_tables", "_jiangchang_columns"})
|
||
LEGACY_METADATA_TABLES = frozenset({"_schema_meta"})
|
||
METADATA_TABLES = INTERNAL_METADATA_TABLES | LEGACY_METADATA_TABLES
|
||
|
||
PLATFORMS_TABLE = "platforms"
|
||
ACCOUNTS_TABLE = "accounts"
|
||
CREDENTIALS_TABLE = "credentials"
|
||
ACCOUNT_LEASES_TABLE = "account_leases"
|
||
|
||
DATETIME_UNIX_SECONDS = "datetime_unix_seconds"
|
||
STANDARD_TIMESTAMP_COLUMNS = frozenset({"created_at", "updated_at"})
|
||
|
||
# Data-management lookup: host resolves options_json object into select options.
|
||
PLATFORM_KEY_LOOKUP_JSON = json.dumps(
|
||
{
|
||
"ref_table": "platforms",
|
||
"value_column": "platform_key",
|
||
"label_column": "display_name",
|
||
"filter_column": "enabled",
|
||
"filter_value": 1,
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
ACCOUNT_ID_LOOKUP_JSON = json.dumps(
|
||
{
|
||
"ref_table": "accounts",
|
||
"value_column": "id",
|
||
"label_column": "account_label",
|
||
},
|
||
ensure_ascii=False,
|
||
)
|
||
|
||
USER_BUSINESS_TABLES = (
|
||
PLATFORMS_TABLE,
|
||
ACCOUNTS_TABLE,
|
||
CREDENTIALS_TABLE,
|
||
ACCOUNT_LEASES_TABLE,
|
||
)
|
||
|
||
_JIANGCHANG_TABLES_DDL = """
|
||
CREATE TABLE IF NOT EXISTS _jiangchang_tables (
|
||
table_name TEXT PRIMARY KEY,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT,
|
||
sort_order INTEGER,
|
||
visible INTEGER NOT NULL DEFAULT 1,
|
||
readonly INTEGER NOT NULL DEFAULT 0
|
||
)
|
||
"""
|
||
|
||
_JIANGCHANG_COLUMNS_DDL = """
|
||
CREATE TABLE IF NOT EXISTS _jiangchang_columns (
|
||
table_name TEXT NOT NULL,
|
||
column_name TEXT NOT NULL,
|
||
display_name TEXT NOT NULL,
|
||
description TEXT,
|
||
display_order INTEGER,
|
||
visible INTEGER NOT NULL DEFAULT 1,
|
||
searchable INTEGER NOT NULL DEFAULT 1,
|
||
editable INTEGER NOT NULL DEFAULT 1,
|
||
display_type TEXT,
|
||
options_json TEXT,
|
||
PRIMARY KEY (table_name, column_name)
|
||
)
|
||
"""
|
||
|
||
|
||
def _col(
|
||
column_name: str,
|
||
display_name: str,
|
||
*,
|
||
description: str | None = None,
|
||
visible: int = 1,
|
||
searchable: int = 1,
|
||
editable: int = 1,
|
||
display_type: str | None = None,
|
||
options_json: str | None = None,
|
||
) -> dict[str, object]:
|
||
return {
|
||
"column_name": column_name,
|
||
"display_name": display_name,
|
||
"description": description,
|
||
"visible": visible,
|
||
"searchable": searchable,
|
||
"editable": editable,
|
||
"display_type": display_type,
|
||
"options_json": options_json,
|
||
}
|
||
|
||
|
||
def _ts(column_name: str, display_name: str, description: str) -> dict[str, object]:
|
||
return _col(
|
||
column_name,
|
||
display_name,
|
||
description=description,
|
||
searchable=0,
|
||
editable=0,
|
||
display_type=DATETIME_UNIX_SECONDS,
|
||
)
|
||
|
||
|
||
PLATFORMS_TABLE_METADATA = {
|
||
"table_name": PLATFORMS_TABLE,
|
||
"display_name": "平台信息",
|
||
"description": "已注册平台（物流、LLM、内容等）及默认登录策略",
|
||
"sort_order": 1,
|
||
"visible": 1,
|
||
"readonly": 0,
|
||
}
|
||
|
||
PLATFORMS_COLUMN_METADATA: tuple[dict[str, object], ...] = (
|
||
_col("platform_key", "平台键", description="平台唯一标识（英文 kebab-case）", editable=0),
|
||
_col("display_name", "展示名称", description="平台在界面上的显示名称"),
|
||
_col("domain", "业务域", description="logistics / llm / content / dev / sim 等"),
|
||
_col("provider_code", "供应商代码", description="供应商或平台代码，可空"),
|
||
_col("default_url", "默认链接", description="平台默认入口 URL"),
|
||
_col("aliases_json", "别名列表", description="平台别名 JSON 数组", editable=0),
|
||
_col("capabilities_json", "能力声明", description='能力 JSON，如 {"web":true,"api_key":false}', editable=0),
|
||
_col("enabled", "是否启用", description="1=启用，0=停用"),
|
||
_col("default_auth_strategy", "默认登录策略", description="新建账号时的默认 auth_strategy"),
|
||
_ts("created_at", "创建时间", "记录创建时间（Unix 秒级时间戳）"),
|
||
_ts("updated_at", "更新时间", "记录最后更新时间（Unix 秒级时间戳）"),
|
||
)
|
||
|
||
ACCOUNTS_TABLE_METADATA = {
|
||
"table_name": ACCOUNTS_TABLE,
|
||
"display_name": "账号信息",
|
||
"description": "网页/RPA 账号身份与浏览器 Profile 路径",
|
||
"sort_order": 2,
|
||
"visible": 1,
|
||
"readonly": 0,
|
||
}
|
||
|
||
ACCOUNTS_COLUMN_METADATA: tuple[dict[str, object], ...] = (
|
||
_col("id", "编号", description="账号唯一编号", editable=0),
|
||
_col(
|
||
"platform_key",
|
||
"平台",
|
||
description="所属平台键（来自平台信息表）",
|
||
display_type="lookup",
|
||
options_json=PLATFORM_KEY_LOOKUP_JSON,
|
||
),
|
||
_col("account_label", "账号名称", description="人可读的账号名称"),
|
||
_col("login_id", "登录账号", description="手机号、邮箱、用户名等登录标识"),
|
||
_col("login_id_type", "登录标识类型", description="phone / email / username / unknown 等"),
|
||
_col("tenant_id", "租户编号", description="租户或客户标识，可空"),
|
||
_col("environment", "环境", description="production / simulator / staging 等"),
|
||
_col("role", "角色", description="booking / tracking / api / default 等"),
|
||
_col("provider_code", "供应商代码", description="冗余供应商码，便于查询"),
|
||
_col("profile_dir", "浏览器配置目录", description="Playwright 持久化 Profile 路径", editable=0),
|
||
_col("url", "入口链接", description="账号默认入口 URL"),
|
||
_col("status", "状态", description="active / disabled / needs_review 等"),
|
||
_col("session_status", "会话状态", description="unknown / likely_valid / needs_login 等"),
|
||
_ts("last_used_at", "最后使用时间", "最近一次被业务技能使用的时间"),
|
||
_ts("last_login_check_at", "最后登录检查时间", "最近一次登录状态检查时间"),
|
||
_col("last_error_code", "最近错误码", description="最近一次错误代码", editable=0),
|
||
_col("last_error_message", "最近错误信息", description="最近一次错误说明", editable=0),
|
||
_col("extra_json", "扩展信息", description="扩展 JSON 配置", editable=0),
|
||
_col("auth_strategy", "登录方式", description="九种登录策略之一"),
|
||
_col("session_persistent", "持久会话", description="是否需要持久化浏览器会话（0/1）"),
|
||
_col("device_fingerprint", "设备指纹", description="device_bound 策略专用，可空"),
|
||
_ts("created_at", "创建时间", "记录创建时间（Unix 秒级时间戳）"),
|
||
_ts("updated_at", "更新时间", "记录最后更新时间（Unix 秒级时间戳）"),
|
||
)
|
||
|
||
CREDENTIALS_TABLE_METADATA = {
|
||
"table_name": CREDENTIALS_TABLE,
|
||
"display_name": "凭证信息",
|
||
"description": "API Key、Token、加密密码等凭据元数据（不含明文）",
|
||
"sort_order": 3,
|
||
"visible": 1,
|
||
"readonly": 0,
|
||
}
|
||
|
||
CREDENTIALS_COLUMN_METADATA: tuple[dict[str, object], ...] = (
|
||
_col("id", "编号", description="凭据唯一编号", editable=0),
|
||
_col(
|
||
"account_id",
|
||
"账号编号",
|
||
description="关联的 accounts.id，可空",
|
||
display_type="lookup",
|
||
options_json=ACCOUNT_ID_LOOKUP_JSON,
|
||
),
|
||
_col(
|
||
"platform_key",
|
||
"平台",
|
||
description="所属平台键（来自平台信息表）",
|
||
display_type="lookup",
|
||
options_json=PLATFORM_KEY_LOOKUP_JSON,
|
||
),
|
||
_col("credential_label", "凭据名称", description="人可读的凭据名称"),
|
||
_col("credential_type", "凭据类型", description="api_key / password / bearer_token 等"),
|
||
_col("secret_storage", "密钥存储方式", description="none / env / windows_credential / local_encrypted", editable=0),
|
||
_col("secret_ref", "密钥引用", description="环境变量名或凭据管理器 target", editable=0, searchable=0),
|
||
_col("secret_mask", "密钥掩码", description="打码后的 secret 展示", editable=0, searchable=0),
|
||
_col(
|
||
"secret_ciphertext",
|
||
"密钥密文",
|
||
description="local_encrypted 时的 Fernet 密文，不可手工编辑",
|
||
editable=0,
|
||
searchable=0,
|
||
),
|
||
_ts("expires_at", "过期时间", "凭据过期时间，可空"),
|
||
_col("status", "状态", description="active / disabled / expired 等"),
|
||
_ts("last_used_at", "最后使用时间", "最近一次使用时间"),
|
||
_col("extra_json", "扩展信息", description="扩展 JSON 配置", editable=0),
|
||
_ts("created_at", "创建时间", "记录创建时间（Unix 秒级时间戳）"),
|
||
_ts("updated_at", "更新时间", "记录最后更新时间（Unix 秒级时间戳）"),
|
||
)
|
||
|
||
ACCOUNT_LEASES_TABLE_METADATA = {
|
||
"table_name": ACCOUNT_LEASES_TABLE,
|
||
"display_name": "账号租约",
|
||
"description": "RPA 并发租约，防止多任务同时使用同一浏览器 Profile",
|
||
"sort_order": 4,
|
||
"visible": 1,
|
||
"readonly": 1,
|
||
}
|
||
|
||
ACCOUNT_LEASES_COLUMN_METADATA: tuple[dict[str, object], ...] = (
|
||
_col("id", "编号", description="租约唯一编号", editable=0),
|
||
_col("account_id", "账号编号", description="关联的 accounts.id", editable=0),
|
||
_col("lease_token", "租约令牌", description="用于 release 的随机 token", editable=0, searchable=1),
|
||
_col("holder", "持有者", description="持有租约的业务技能或任务标识", editable=0),
|
||
_col("purpose", "用途", description="租约用途说明", editable=0),
|
||
_ts("expires_at", "过期时间", "租约过期时间"),
|
||
_ts("heartbeat_at", "心跳时间", "最近一次心跳时间"),
|
||
_ts("released_at", "释放时间", "主动释放时间，可空"),
|
||
_col("status", "状态", description="active / released / expired", editable=0),
|
||
_ts("created_at", "创建时间", "记录创建时间（Unix 秒级时间戳）"),
|
||
_ts("updated_at", "更新时间", "记录最后更新时间（Unix 秒级时间戳）"),
|
||
)
|
||
|
||
_TABLE_SEEDS: tuple[tuple[dict[str, object], tuple[dict[str, object], ...]], ...] = (
|
||
(PLATFORMS_TABLE_METADATA, PLATFORMS_COLUMN_METADATA),
|
||
(ACCOUNTS_TABLE_METADATA, ACCOUNTS_COLUMN_METADATA),
|
||
(CREDENTIALS_TABLE_METADATA, CREDENTIALS_COLUMN_METADATA),
|
||
(ACCOUNT_LEASES_TABLE_METADATA, ACCOUNT_LEASES_COLUMN_METADATA),
|
||
)
|
||
|
||
|
||
def init_display_metadata_tables(cursor) -> None:
|
||
cursor.execute(_JIANGCHANG_TABLES_DDL)
|
||
cursor.execute(_JIANGCHANG_COLUMNS_DDL)
|
||
|
||
|
||
def upsert_table_metadata(
|
||
cursor,
|
||
*,
|
||
table_name: str,
|
||
display_name: str,
|
||
description: str | None = None,
|
||
sort_order: int | None = None,
|
||
visible: int = 1,
|
||
readonly: int = 0,
|
||
) -> None:
|
||
cursor.execute(
|
||
"""
|
||
INSERT INTO _jiangchang_tables (
|
||
table_name, display_name, description, sort_order, visible, readonly
|
||
)
|
||
VALUES (?, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(table_name) DO UPDATE SET
|
||
display_name = excluded.display_name,
|
||
description = excluded.description,
|
||
sort_order = excluded.sort_order,
|
||
visible = excluded.visible,
|
||
readonly = excluded.readonly
|
||
""",
|
||
(table_name, display_name, description, sort_order, visible, readonly),
|
||
)
|
||
|
||
|
||
def upsert_column_metadata(
|
||
cursor,
|
||
*,
|
||
table_name: str,
|
||
column_name: str,
|
||
display_name: str,
|
||
description: str | None = None,
|
||
visible: int = 1,
|
||
searchable: int = 1,
|
||
editable: int = 1,
|
||
display_type: str | None = None,
|
||
options_json: str | None = None,
|
||
) -> None:
|
||
cursor.execute(
|
||
"""
|
||
INSERT INTO _jiangchang_columns (
|
||
table_name, column_name, display_name, description,
|
||
display_order, visible, searchable, editable, display_type, options_json
|
||
)
|
||
VALUES (?, ?, ?, ?, NULL, ?, ?, ?, ?, ?)
|
||
ON CONFLICT(table_name, column_name) DO UPDATE SET
|
||
display_name = excluded.display_name,
|
||
description = excluded.description,
|
||
display_order = NULL,
|
||
visible = excluded.visible,
|
||
searchable = excluded.searchable,
|
||
editable = excluded.editable,
|
||
display_type = excluded.display_type,
|
||
options_json = excluded.options_json
|
||
""",
|
||
(
|
||
table_name,
|
||
column_name,
|
||
display_name,
|
||
description,
|
||
visible,
|
||
searchable,
|
||
editable,
|
||
display_type,
|
||
options_json,
|
||
),
|
||
)
|
||
|
||
|
||
def seed_column_metadata_batch(
|
||
cursor,
|
||
*,
|
||
table_name: str,
|
||
columns: Iterable[dict[str, object]],
|
||
valid_column_names: Iterable[str] | None = None,
|
||
) -> None:
|
||
for column in columns:
|
||
upsert_column_metadata(
|
||
cursor,
|
||
table_name=table_name,
|
||
column_name=str(column["column_name"]),
|
||
display_name=str(column["display_name"]),
|
||
description=str(column["description"]) if column.get("description") else None,
|
||
visible=int(column.get("visible", 1)),
|
||
searchable=int(column.get("searchable", 1)),
|
||
editable=int(column.get("editable", 1)),
|
||
display_type=str(column["display_type"]) if column.get("display_type") else None,
|
||
options_json=str(column["options_json"]) if column.get("options_json") else None,
|
||
)
|
||
if valid_column_names is not None:
|
||
prune_stale_column_metadata(cursor, table_name, valid_column_names)
|
||
|
||
|
||
def prune_stale_column_metadata(cursor, table_name: str, valid_column_names: Iterable[str]) -> None:
|
||
valid = set(valid_column_names)
|
||
cursor.execute(
|
||
"SELECT column_name FROM _jiangchang_columns WHERE table_name = ?",
|
||
(table_name,),
|
||
)
|
||
for (column_name,) in cursor.fetchall():
|
||
if str(column_name) not in valid:
|
||
cursor.execute(
|
||
"DELETE FROM _jiangchang_columns WHERE table_name = ? AND column_name = ?",
|
||
(table_name, column_name),
|
||
)
|
||
|
||
|
||
def get_table_readonly(cursor, table_name: str) -> int | None:
|
||
cursor.execute(
|
||
"SELECT readonly FROM _jiangchang_tables WHERE table_name = ?",
|
||
(table_name,),
|
||
)
|
||
row = cursor.fetchone()
|
||
if not row:
|
||
return None
|
||
return int(row[0] or 0)
|
||
|
||
|
||
def get_column_editable(cursor, table_name: str, column_name: str) -> int | None:
|
||
cursor.execute(
|
||
"""
|
||
SELECT editable FROM _jiangchang_columns
|
||
WHERE table_name = ? AND column_name = ?
|
||
""",
|
||
(table_name, column_name),
|
||
)
|
||
row = cursor.fetchone()
|
||
if not row:
|
||
return None
|
||
return int(row[0] or 0)
|
||
|
||
|
||
def list_editable_business_columns(cursor, table_name: str) -> list[str]:
|
||
physical = list_physical_column_names(cursor, table_name)
|
||
cursor.execute(
|
||
"""
|
||
SELECT column_name, editable
|
||
FROM _jiangchang_columns
|
||
WHERE table_name = ?
|
||
""",
|
||
(table_name,),
|
||
)
|
||
editable_set = {
|
||
str(column_name)
|
||
for column_name, editable in cursor.fetchall()
|
||
if editable is not None and int(editable) == 1
|
||
}
|
||
return [name for name in physical if name in editable_set]
|
||
|
||
|
||
def table_allows_manual_create(cursor, table_name: str) -> bool:
|
||
readonly = get_table_readonly(cursor, table_name)
|
||
if readonly is None or int(readonly) != 0:
|
||
return False
|
||
return bool(list_editable_business_columns(cursor, table_name))
|
||
|
||
|
||
def seed_business_table_display_metadata(cursor, table_meta: dict[str, object], columns: tuple[dict[str, object], ...]) -> None:
|
||
table_name = str(table_meta["table_name"])
|
||
upsert_table_metadata(
|
||
cursor,
|
||
table_name=table_name,
|
||
display_name=str(table_meta["display_name"]),
|
||
description=str(table_meta["description"]) if table_meta.get("description") else None,
|
||
sort_order=int(table_meta["sort_order"]) if table_meta.get("sort_order") is not None else None,
|
||
visible=int(table_meta.get("visible", 1)),
|
||
readonly=int(table_meta.get("readonly", 0)),
|
||
)
|
||
seed_column_metadata_batch(
|
||
cursor,
|
||
table_name=table_name,
|
||
columns=columns,
|
||
valid_column_names=[str(column["column_name"]) for column in columns],
|
||
)
|
||
|
||
|
||
def init_display_metadata(cursor) -> None:
|
||
init_display_metadata_tables(cursor)
|
||
for table_meta, columns in _TABLE_SEEDS:
|
||
seed_business_table_display_metadata(cursor, table_meta, columns)
|
||
|
||
|
||
def iter_user_tables(cursor) -> Iterable[str]:
|
||
cursor.execute(
|
||
"SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name"
|
||
)
|
||
for (name,) in cursor.fetchall():
|
||
if name in METADATA_TABLES:
|
||
continue
|
||
yield str(name)
|
||
|
||
|
||
def _quote_identifier(name: str) -> str:
|
||
return '"' + name.replace('"', '""') + '"'
|
||
|
||
|
||
def list_physical_column_names(cursor, table_name: str) -> list[str]:
|
||
cursor.execute(f"PRAGMA table_info({_quote_identifier(table_name)})")
|
||
rows = cursor.fetchall()
|
||
rows.sort(key=lambda row: int(row[0]))
|
||
return [str(row[1]) for row in rows]
|