mirror of
https://github.com/stardrophere/InsightRadar.git
synced 2026-06-06 03:07:50 +08:00
stardrophere
360 lines
15 KiB
Python
360 lines
15 KiB
Python
# models.py
|
|
from datetime import datetime, timezone, time
|
|
from typing import Optional, Any
|
|
import enum
|
|
|
|
from sqlalchemy import (
|
|
String, Integer, BigInteger, Text, Boolean, DateTime, Time,
|
|
Float, JSON, ForeignKey, Enum, UniqueConstraint, Index
|
|
)
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
|
|
|
|
|
# ==========================================
|
|
# 0. 全局基类、枚举定义与动态类型
|
|
# ==========================================
|
|
|
|
class Base(DeclarativeBase):
|
|
"""
|
|
SQLAlchemy 2.0 声明式基类
|
|
所有的表模型都必须继承这个基类。
|
|
"""
|
|
pass
|
|
|
|
|
|
# 让代码在 SQLite 环境下自动降级为 Integer 以保证自增正常工作，
|
|
# 而在生产环境部署到 PostgreSQL 或 MySQL 时，依然会使用容量更大的 BigInteger。
|
|
BigIntType = BigInteger().with_variant(Integer, "sqlite")
|
|
|
|
|
|
class SourceType(str, enum.Enum):
|
|
"""信息源的抓取方式"""
|
|
HOT_TREND = "HOT_TREND" # 热搜榜单类
|
|
RSS_FEED = "RSS_FEED" # 传统RSS订阅
|
|
API = "API" # 接口抓取类
|
|
|
|
|
|
class TargetType(str, enum.Enum):
|
|
"""
|
|
多态目标类型 (Polymorphic Target)
|
|
用于标记一条评论或一个标签到底是挂载在哪个实体下的。
|
|
"""
|
|
EVENT = "EVENT" # 挂载在单个热搜事件下
|
|
TREND = "TREND" # 挂载在宏观趋势下
|
|
ARTICLE = "ARTICLE" # 挂载在具体新闻文章下
|
|
|
|
|
|
class TaskStatus(str, enum.Enum):
|
|
"""后台任务状态"""
|
|
SUCCESS = "SUCCESS"
|
|
ERROR = "ERROR"
|
|
|
|
|
|
class GenderType(str, enum.Enum):
|
|
"""用户性别枚举"""
|
|
MALE = "MALE"
|
|
FEMALE = "FEMALE"
|
|
OTHER = "OTHER"
|
|
UNKNOWN = "UNKNOWN"
|
|
|
|
|
|
def utcnow():
|
|
"""
|
|
获取带UTC时区的当前时间 (最佳实践)
|
|
服务器内部和数据库统一存储 UTC 时间，只在前端展示时转为用户本地时区，避免时区错乱。
|
|
"""
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
# ==========================================
|
|
# 模块一：信息源管理
|
|
# ==========================================
|
|
class InfoSource(Base):
|
|
"""
|
|
抓取源配置表
|
|
充当爬虫的“任务清单”，后台可以随时开关特定的信息源，而不需要重启代码。
|
|
"""
|
|
__tablename__ = "info_sources"
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
|
source_name: Mapped[str] = mapped_column(String(100), comment="信息源名称")
|
|
source_type: Mapped[SourceType] = mapped_column(Enum(SourceType))
|
|
home_url: Mapped[Optional[str]] = mapped_column(String(255))
|
|
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块二：AI 语义聚类中枢 (大事件池)
|
|
# ==========================================
|
|
class UnifiedEvent(Base):
|
|
"""
|
|
AI 统一事件表
|
|
核心业务逻辑：比如微博热搜叫“苹果发布会”，知乎热搜叫“iPhone 16 测评”，
|
|
它们在子表(TrendingEvent)是两条记录，但通过 AI 语义向量对比后，
|
|
会将它们统一挂载到这个表的一个 UnifiedEvent ID 下，实现跨平台事件聚合。
|
|
"""
|
|
__tablename__ = "unified_events"
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
unified_title: Mapped[str] = mapped_column(String(255), comment="AI统一标题")
|
|
ai_comprehensive_summary: Mapped[Optional[str]] = mapped_column(Text, comment="AI全局深度总结")
|
|
|
|
center_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="中心向量") # 用于高维空间相似度计算
|
|
hot_score: Mapped[int] = mapped_column(Integer, default=0, comment="聚合热度得分")
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块三：内容存储库 (热搜 & 新闻子节点)
|
|
# ==========================================
|
|
class TrendingEvent(Base):
|
|
"""
|
|
各平台热搜数据明细表
|
|
"""
|
|
__tablename__ = "trending_events"
|
|
__table_args__ = (
|
|
# 联合唯一索引：同一个来源(比如微博)的同一条外部ID(MD5)只能存在一条记录，防重插核心保障
|
|
UniqueConstraint("source_id", "external_id", name="idx_unique_external_trend"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
|
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
|
|
|
|
external_id: Mapped[str] = mapped_column(String(32), comment="32位MD5哈希指纹防重")
|
|
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
|
|
|
|
icon_url: Mapped[Optional[str]] = mapped_column(String(500))
|
|
current_headline: Mapped[str] = mapped_column(String(255))
|
|
event_url: Mapped[Optional[str]] = mapped_column(String(500))
|
|
app_link: Mapped[Optional[str]] = mapped_column(String(500))
|
|
current_ranking: Mapped[Optional[int]] = mapped_column(Integer)
|
|
brief_snippet: Mapped[Optional[str]] = mapped_column(Text)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
class NewsArticle(Base):
|
|
"""新闻文章明细表 (与 TrendingEvent 类似，但侧重长文本阅读)"""
|
|
__tablename__ = "news_articles"
|
|
__table_args__ = (
|
|
UniqueConstraint("source_id", "external_id", name="idx_unique_external_article"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
|
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
|
|
|
|
external_id: Mapped[str] = mapped_column(String(32))
|
|
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
|
|
|
|
cover_image_url: Mapped[Optional[str]] = mapped_column(String(500))
|
|
article_title: Mapped[str] = mapped_column(String(255))
|
|
article_url: Mapped[Optional[str]] = mapped_column(String(500))
|
|
author_name: Mapped[Optional[str]] = mapped_column(String(100))
|
|
original_summary: Mapped[Optional[str]] = mapped_column(Text)
|
|
publish_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块四：热度与轨迹追踪
|
|
# ==========================================
|
|
class HeadlineRevision(Base):
|
|
"""
|
|
标题修订历史表
|
|
用于记录平台方暗戳戳修改热搜词条的行为（例如公关介入改标题）。
|
|
"""
|
|
__tablename__ = "headline_revisions"
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
|
|
previous_headline: Mapped[str] = mapped_column(String(255))
|
|
revised_headline: Mapped[str] = mapped_column(String(255))
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class RankingLog(Base):
|
|
"""
|
|
热搜排名时间序列化日志
|
|
每一次抓取都会生成一条记录，可以用于前端绘制热搜“排名起伏折线图”。
|
|
"""
|
|
__tablename__ = "ranking_logs"
|
|
__table_args__ = (
|
|
# 针对时间序列查询优化的复合索引，加速类似 "查询某事件在过去24小时内的排名变化" 的操作
|
|
Index("idx_event_time", "event_id", "observed_at"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
|
|
ranking_position: Mapped[int] = mapped_column(Integer)
|
|
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块五：多态话题与多态评论
|
|
# ==========================================
|
|
# 【设计模式】：多态设计
|
|
# 通过 target_type (存表名/类型) + target_id (存主键ID) 的组合，
|
|
# 让这两个表既能挂载在"单一热搜"下，也能挂载在"新闻文章"下，甚至挂在"统一大事件"下，避免了建立无数个外键的冗余。
|
|
|
|
class ExtractedTopic(Base):
|
|
"""AI 提取的核心话题标签表"""
|
|
__tablename__ = "extracted_topics"
|
|
__table_args__ = (
|
|
Index("idx_topic_keyword", "topic_keyword"),
|
|
# 多态查询索引，加速 target_type + target_id 的组合查询
|
|
Index("idx_polymorphic_topics", "target_type", "target_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
|
target_id: Mapped[int] = mapped_column(BigIntType)
|
|
topic_keyword: Mapped[str] = mapped_column(String(100))
|
|
relevance_score: Mapped[Optional[float]] = mapped_column(Float)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class DiscussionComment(Base):
|
|
"""全平台统一评论表"""
|
|
__tablename__ = "discussion_comments"
|
|
__table_args__ = (
|
|
Index("idx_polymorphic_comments", "target_type", "target_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
|
target_id: Mapped[int] = mapped_column(BigIntType)
|
|
|
|
commenter_name: Mapped[Optional[str]] = mapped_column(String(100))
|
|
comment_content: Mapped[str] = mapped_column(Text)
|
|
likes_count: Mapped[int] = mapped_column(Integer, default=0)
|
|
external_comment_id: Mapped[Optional[str]] = mapped_column(String(32))
|
|
comment_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块六：用户画像与多渠道高可用推送系统
|
|
# ==========================================
|
|
class AppUser(Base):
|
|
"""系统核心用户表"""
|
|
__tablename__ = "app_users"
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
email: Mapped[str] = mapped_column(String(150), unique=True, index=True)
|
|
password_hash: Mapped[Optional[str]] = mapped_column(String(255))
|
|
|
|
nickname: Mapped[Optional[str]] = mapped_column(String(100))
|
|
avatar_url: Mapped[Optional[str]] = mapped_column(String(500))
|
|
gender: Mapped[GenderType] = mapped_column(Enum(GenderType), default=GenderType.UNKNOWN)
|
|
|
|
# 预留的 JSON 字段，可以存放未来灵活变化的用户配置，避免频繁修改表结构
|
|
metadata_: Mapped[Optional[Any]] = mapped_column("metadata", JSON, comment="自定义扩展偏好")
|
|
|
|
timezone: Mapped[str] = mapped_column(String(50), default="Asia/Shanghai")
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
class UserPushEndpoint(Base):
|
|
"""
|
|
多渠道推送端点配置表
|
|
一个用户可能绑定了邮箱(EMAIL)和微信(WECHAT)，支持配置降级重试(priority_level)。
|
|
"""
|
|
__tablename__ = "user_push_endpoints"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "channel_type", name="idx_unique_user_channel"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
|
channel_type: Mapped[str] = mapped_column(String(50), comment="如 EMAIL, WECHAT")
|
|
channel_account: Mapped[str] = mapped_column(String(255))
|
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
priority_level: Mapped[int] = mapped_column(Integer, default=1, comment="1最高，降级重试")
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
|
|
|
|
|
class UserTopicPreference(Base):
|
|
"""用户订阅的兴趣标签库"""
|
|
__tablename__ = "user_topic_preferences"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "interested_keyword", name="idx_unique_preference"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
|
interested_keyword: Mapped[str] = mapped_column(String(100))
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class UserDeliverySchedule(Base):
|
|
"""用户勿扰/定时推送时间表"""
|
|
__tablename__ = "user_delivery_schedules"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "delivery_time", name="idx_unique_schedule"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
|
delivery_time: Mapped[time] = mapped_column(Time, comment="如 08:30:00")
|
|
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
class DeliveryHistory(Base):
|
|
"""
|
|
推送历史防刷表
|
|
核心用途：一旦给某个用户推送过某条新闻/事件，就记录在这里。
|
|
下次再触发推荐时，检查这个表，防止给同一个用户反复发送相同的内容。
|
|
"""
|
|
__tablename__ = "delivery_history"
|
|
__table_args__ = (
|
|
UniqueConstraint("user_id", "target_type", "target_id", name="idx_prevent_duplicate_push"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
|
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
|
target_id: Mapped[int] = mapped_column(BigIntType)
|
|
status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
|
|
|
|
|
# ==========================================
|
|
# 模块七：系统任务监控
|
|
# ==========================================
|
|
class DataSyncTask(Base):
|
|
"""
|
|
数据同步健康度监控表
|
|
这就是爬虫脚本每次运行都要写入记录的地方，用于后台 Dashboard 监控爬虫健康状态和错误堆栈。
|
|
"""
|
|
__tablename__ = "data_sync_tasks"
|
|
|
|
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
|
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
|
items_fetched: Mapped[int] = mapped_column(Integer, default=0)
|
|
task_status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
|
|
error_trace: Mapped[Optional[str]] = mapped_column(Text)
|
|
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|