login+ai cluster

This commit is contained in:
stardrophere
2026-03-11 01:33:21 +08:00
parent 9fa07cfb07
commit 8ed819a580
39 changed files with 3392 additions and 610 deletions
+13
View File
@@ -0,0 +1,13 @@
# app/api/dependencies.py
from app.database import SessionLocal
def get_db():
"""
FastAPI 依赖注入:为每个 HTTP 请求提供独立的数据库会话。
请求处理完成后自动关闭,防止连接泄漏。
"""
db = SessionLocal()
try:
yield db
finally:
db.close()
+238
View File
@@ -0,0 +1,238 @@
import os
from datetime import timedelta
from typing import Tuple
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy.orm import Session
from app.api.dependencies import get_db
from app.core.security import (
create_access_token,
generate_verification_code,
hash_password,
hash_verification_code,
verify_password,
verify_verification_code,
)
from app.models.models import AppUser, EmailVerificationCode, VerificationPurpose, utcnow
from app.schemas.auth_schema import (
AuthTokenResponse,
LoginCodeSendRequest,
LoginRequest,
LoginWithCodeRequest,
MessageResponse,
RegisterCodeSendRequest,
RegisterRequest,
UserProfileResponse,
)
from app.utils.email_utils import send_html_email
router = APIRouter()
REGISTER_CODE_EXPIRE_MINUTES = int(os.getenv("REGISTER_CODE_EXPIRE_MINUTES", "10"))
LOGIN_CODE_EXPIRE_MINUTES = int(os.getenv("LOGIN_CODE_EXPIRE_MINUTES", "10"))
def _normalize_email(email: str) -> str:
return email.strip().lower()
def _build_verification_email(code: str, purpose_text: str, expire_minutes: int) -> str:
return f"""
<div style="font-family: Arial, sans-serif; line-height: 1.6; color: #222;">
<h2 style="margin-bottom: 12px;">InsightRadar Email Verification</h2>
<p>Your {purpose_text} verification code is:</p>
<p style="font-size: 28px; font-weight: bold; letter-spacing: 4px; color: #0b57d0;">{code}</p>
<p>The code is valid for {expire_minutes} minutes. Do not share it with others.</p>
</div>
"""
def _invalidate_unused_codes(db: Session, email: str, purpose: VerificationPurpose) -> None:
db.query(EmailVerificationCode).filter(
EmailVerificationCode.email == email,
EmailVerificationCode.purpose == purpose,
EmailVerificationCode.is_used.is_(False),
).update({EmailVerificationCode.is_used: True}, synchronize_session=False)
db.commit()
def _create_code_record(
db: Session,
*,
email: str,
purpose: VerificationPurpose,
expire_minutes: int,
) -> Tuple[EmailVerificationCode, str]:
code = generate_verification_code()
now = utcnow()
code_record = EmailVerificationCode(
email=email,
purpose=purpose,
code_hash=hash_verification_code(code),
expires_at=now + timedelta(minutes=expire_minutes),
)
db.add(code_record)
db.commit()
return code_record, code
def _build_auth_response(user: AppUser) -> AuthTokenResponse:
token, expires_in = create_access_token(user_id=user.id, email=user.email)
return AuthTokenResponse(
access_token=token,
expires_in=expires_in,
user=UserProfileResponse.model_validate(user),
)
@router.post("/register/send-code", response_model=MessageResponse)
async def send_register_code(payload: RegisterCodeSendRequest, db: Session = Depends(get_db)):
email = _normalize_email(payload.email)
existing_user = db.query(AppUser).filter(AppUser.email == email).first()
if existing_user:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email is already registered")
_invalidate_unused_codes(db, email, VerificationPurpose.REGISTER)
code_record, code = _create_code_record(
db,
email=email,
purpose=VerificationPurpose.REGISTER,
expire_minutes=REGISTER_CODE_EXPIRE_MINUTES,
)
email_sent = await send_html_email(
to_email=email,
subject="InsightRadar Registration Code",
html_content=_build_verification_email(code, "registration", REGISTER_CODE_EXPIRE_MINUTES),
)
if not email_sent:
code_record.is_used = True
db.add(code_record)
db.commit()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to send verification code",
)
return MessageResponse(message="Verification code sent")
@router.post("/login/send-code", response_model=MessageResponse)
async def send_login_code(payload: LoginCodeSendRequest, db: Session = Depends(get_db)):
email = _normalize_email(payload.email)
user = db.query(AppUser).filter(AppUser.email == email).first()
if not user:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Email is not registered")
_invalidate_unused_codes(db, email, VerificationPurpose.LOGIN)
code_record, code = _create_code_record(
db,
email=email,
purpose=VerificationPurpose.LOGIN,
expire_minutes=LOGIN_CODE_EXPIRE_MINUTES,
)
email_sent = await send_html_email(
to_email=email,
subject="InsightRadar Login Code",
html_content=_build_verification_email(code, "login", LOGIN_CODE_EXPIRE_MINUTES),
)
if not email_sent:
code_record.is_used = True
db.add(code_record)
db.commit()
raise HTTPException(
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
detail="Failed to send verification code",
)
return MessageResponse(message="Verification code sent")
@router.post(
"/register",
response_model=AuthTokenResponse,
status_code=status.HTTP_201_CREATED,
)
async def register(payload: RegisterRequest, db: Session = Depends(get_db)):
email = _normalize_email(payload.email)
existing_user = db.query(AppUser).filter(AppUser.email == email).first()
if existing_user:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email is already registered")
now = utcnow()
code_record = db.query(EmailVerificationCode).filter(
EmailVerificationCode.email == email,
EmailVerificationCode.purpose == VerificationPurpose.REGISTER,
EmailVerificationCode.is_used.is_(False),
EmailVerificationCode.expires_at >= now,
).order_by(EmailVerificationCode.created_at.desc()).first()
if not code_record:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Verification code does not exist or expired")
if not verify_verification_code(payload.verification_code, code_record.code_hash):
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid verification code")
nickname = payload.nickname or email.split("@")[0]
user = AppUser(
email=email,
password_hash=hash_password(payload.password),
nickname=nickname,
metadata_={"email_verified_at": now.isoformat()},
)
code_record.is_used = True
db.add(user)
db.add(code_record)
db.commit()
db.refresh(user)
return _build_auth_response(user)
@router.post("/login", response_model=AuthTokenResponse)
async def login(payload: LoginRequest, db: Session = Depends(get_db)):
email = _normalize_email(payload.email)
user = db.query(AppUser).filter(AppUser.email == email).first()
if not user or not user.password_hash:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or password")
if not verify_password(payload.password, user.password_hash):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or password")
return _build_auth_response(user)
@router.post("/login/code", response_model=AuthTokenResponse)
async def login_with_code(payload: LoginWithCodeRequest, db: Session = Depends(get_db)):
email = _normalize_email(payload.email)
user = db.query(AppUser).filter(AppUser.email == email).first()
if not user:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or verification code")
now = utcnow()
code_record = db.query(EmailVerificationCode).filter(
EmailVerificationCode.email == email,
EmailVerificationCode.purpose == VerificationPurpose.LOGIN,
EmailVerificationCode.is_used.is_(False),
EmailVerificationCode.expires_at >= now,
).order_by(EmailVerificationCode.created_at.desc()).first()
if not code_record:
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Verification code does not exist or expired")
if not verify_verification_code(payload.verification_code, code_record.code_hash):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or verification code")
code_record.is_used = True
db.add(code_record)
db.commit()
return _build_auth_response(user)
+69
View File
@@ -0,0 +1,69 @@
# app/api/endpoints/events.py
from fastapi import APIRouter, Depends, Query
from sqlalchemy.orm import Session
from datetime import timedelta
from typing import List
from app.api.dependencies import get_db
from app.models.models import UnifiedEvent, TrendingEvent, InfoSource, RankingLog, utcnow
# 导入你上传的 Schema
from app.schemas.event_schema import UnifiedEventResponse, PlatformTrendResponse
router = APIRouter()
@router.get("/unified", response_model=List[UnifiedEventResponse])
def list_unified_events(
min_hot: int = Query(5, description="热度过滤阈值"),
hours: int = Query(24, description="查询过去 X 小时的数据"),
db: Session = Depends(get_db)
):
"""
获取聚合大事件列表,完全适配前端 template.html 所需的数据结构
"""
# 计算时间水位线
time_limit = utcnow() - timedelta(hours=hours)
# 1. 查询大事件(按热度降序,且满足时间范围)
events = db.query(UnifiedEvent).filter(
UnifiedEvent.hot_score >= min_hot,
UnifiedEvent.created_at >= time_limit
).order_by(UnifiedEvent.hot_score.desc()).all()
results = []
for ev in events:
# 2. 联表查询:获取该大事件下关联的所有平台及其具体热搜信息
trends = db.query(TrendingEvent, InfoSource.source_name).join(
InfoSource, TrendingEvent.source_id == InfoSource.id
).filter(TrendingEvent.unified_event_id == ev.id).all()
platform_list = []
for trend, s_name in trends:
# 3. 获取排名历史轨迹 (用于前端渲染)
# 这里的排序顺序 asc 保证了数组从旧到新
logs = db.query(RankingLog.ranking_position).filter(
RankingLog.event_id == trend.id,
RankingLog.observed_at >= time_limit
).order_by(RankingLog.observed_at.asc()).all()
# 组装符合 PlatformTrendResponse 结构的字典
platform_list.append(PlatformTrendResponse(
source_id=trend.source_id,
platform_name=s_name,
headline=trend.current_headline,
url=trend.event_url,
current_ranking=trend.current_ranking,
ranking_history=[log[0] for log in logs]
))
# 4. 组装符合 UnifiedEventResponse 结构的字典
results.append(UnifiedEventResponse(
event_id=ev.id,
unified_title=ev.unified_title if ev.unified_title else "暂无标题",
summary=ev.ai_comprehensive_summary,
hot_score=ev.hot_score,
created_at=ev.created_at,
platforms=platform_list
))
return results
+1 -1
View File
@@ -4,7 +4,7 @@ from sqlalchemy.orm import Session
from typing import List
from app.database import get_db
from app.schemas.schemas import (
from app.schemas.source_schema import (
InfoSourceCreate, InfoSourceUpdate, InfoSourceResponse, PaginatedResponse
)
from app.crud import crud_source
View File
+6 -2
View File
@@ -1,8 +1,12 @@
# app/api/router.py
from fastapi import APIRouter
from app.api.endpoints import sources
from app.api.endpoints import auth, sources, events
api_router = APIRouter()
# 信息源管理
api_router.include_router(sources.router, prefix="/sources", tags=["信息源管理"])
api_router.include_router(sources.router, prefix="/sources", tags=["信息源管理"])
# 注册大事件相关的路由
api_router.include_router(events.router, prefix="/events", tags=["Unified Events"])
api_router.include_router(auth.router, prefix="/auth", tags=["Auth"])
+79
View File
@@ -0,0 +1,79 @@
import base64
import hashlib
import hmac
import json
import os
import secrets
import time
from typing import Tuple
PASSWORD_HASH_ITERATIONS = int(os.getenv("PASSWORD_HASH_ITERATIONS", "120000"))
AUTH_SECRET_KEY = os.getenv("AUTH_SECRET_KEY", "change-this-secret-in-env")
AUTH_TOKEN_EXPIRE_MINUTES = int(os.getenv("AUTH_TOKEN_EXPIRE_MINUTES", "10080"))
def hash_password(password: str) -> str:
salt = secrets.token_hex(16)
digest = hashlib.pbkdf2_hmac(
"sha256",
password.encode("utf-8"),
salt.encode("utf-8"),
PASSWORD_HASH_ITERATIONS,
)
return (
f"pbkdf2_sha256${PASSWORD_HASH_ITERATIONS}${salt}$"
f"{base64.urlsafe_b64encode(digest).decode('utf-8')}"
)
def verify_password(plain_password: str, password_hash: str) -> bool:
try:
algorithm, iterations, salt, expected = password_hash.split("$", 3)
if algorithm != "pbkdf2_sha256":
return False
digest = hashlib.pbkdf2_hmac(
"sha256",
plain_password.encode("utf-8"),
salt.encode("utf-8"),
int(iterations),
)
calculated = base64.urlsafe_b64encode(digest).decode("utf-8")
return hmac.compare_digest(calculated, expected)
except Exception:
return False
def generate_verification_code(length: int = 6) -> str:
return "".join(secrets.choice("0123456789") for _ in range(length))
def hash_verification_code(code: str) -> str:
return hashlib.sha256(code.encode("utf-8")).hexdigest()
def verify_verification_code(code: str, expected_hash: str) -> bool:
return hmac.compare_digest(hash_verification_code(code), expected_hash)
def _urlsafe_b64encode(raw: bytes) -> str:
return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=")
def create_access_token(user_id: int, email: str) -> Tuple[str, int]:
expires_in = AUTH_TOKEN_EXPIRE_MINUTES * 60
payload = {
"sub": str(user_id),
"email": email,
"exp": int(time.time()) + expires_in,
}
payload_bytes = json.dumps(payload, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
encoded_payload = _urlsafe_b64encode(payload_bytes)
signature = hmac.new(
AUTH_SECRET_KEY.encode("utf-8"),
encoded_payload.encode("utf-8"),
hashlib.sha256,
).digest()
token = f"{encoded_payload}.{_urlsafe_b64encode(signature)}"
return token, expires_in
+1 -1
View File
@@ -3,7 +3,7 @@ from sqlalchemy.orm import Session
from typing import List, Optional
from app.models.models import InfoSource
from app.schemas.schemas import InfoSourceCreate, InfoSourceUpdate
from app.schemas.source_schema import InfoSourceCreate, InfoSourceUpdate
def get(db: Session, source_id: int) -> Optional[InfoSource]:
+15 -1
View File
@@ -6,6 +6,7 @@ from dotenv import load_dotenv
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from app.services.fetcher_service import fetch_and_save_trending_data
from app.services.summary_service import generate_unified_summaries
from app.database import engine
from app.models.models import Base
@@ -14,6 +15,7 @@ from app.api.router import api_router
load_dotenv()
CRAWL_INTERVAL = int(os.getenv("CRAWL_INTERVAL_MINUTES", 10))
SUMMARY_INTERVAL = int(os.getenv("SUMMARY_INTERVAL_MINUTES", 30))
scheduler = AsyncIOScheduler()
@@ -36,15 +38,27 @@ async def lifespan(app: FastAPI):
id='trending_fetch_job',
replace_existing=True
)
# 平台摘要
scheduler.add_job(
generate_unified_summaries,
'interval',
minutes=SUMMARY_INTERVAL,
id='ai_summary_job',
replace_existing=True
)
scheduler.start()
print(f"定时抓取任务已启动,每 {CRAWL_INTERVAL} 分钟执行一次")
print(f"AI 摘要生成任务已启动,每 {SUMMARY_INTERVAL} 分钟执行一次")
# 为了测试方便,启动时立即执行一次
await fetch_and_save_trending_data()
await generate_unified_summaries()
yield # 此时 FastAPI 开始接受请求
# 3. 优雅关闭
# 优雅关闭
scheduler.shutdown()
print("定时任务已安全关闭")
+194 -111
View File
@@ -29,35 +29,40 @@ BigIntType = BigInteger().with_variant(Integer, "sqlite")
class SourceType(str, enum.Enum):
"""信息源的抓取方式"""
HOT_TREND = "HOT_TREND" # 热搜榜单类
RSS_FEED = "RSS_FEED" # 传统RSS订阅
API = "API" # 接口抓取
HOT_TREND = "HOT_TREND" # 热搜榜单类 (如微博热搜)
RSS_FEED = "RSS_FEED" # 传统RSS订阅 (如36氪、纽约时报)
API = "API" # 接口直接接入
class TargetType(str, enum.Enum):
"""
多态目标类型 (Polymorphic Target)
用于标记一条评论或一个标签到底是挂载在哪个实体下的。
用于标记一条评论、标签或推送记录,到底是挂载在哪个实体下的。
"""
EVENT = "EVENT" # 挂载在单个热搜事件下
TREND = "TREND" # 挂载在宏观趋势
ARTICLE = "ARTICLE" # 挂载在具体新闻文章下
EVENT = "EVENT" # 挂载在AI聚合后的大事件下
TREND = "TREND" # 挂载在单个平台的热搜条目
ARTICLE = "ARTICLE" # 挂载在具体的长篇新闻文章下
class TaskStatus(str, enum.Enum):
"""后台任务状态"""
SUCCESS = "SUCCESS"
ERROR = "ERROR"
"""后台爬虫/推送任务的执行状态"""
SUCCESS = "SUCCESS" # 执行成功
ERROR = "ERROR" # 发生报错
class GenderType(str, enum.Enum):
"""用户性别枚举"""
"""用户性别枚举,常用于给AI提供Prompt背景信息以生成个性化摘要"""
MALE = "MALE"
FEMALE = "FEMALE"
OTHER = "OTHER"
UNKNOWN = "UNKNOWN"
class VerificationPurpose(str, enum.Enum):
REGISTER = "REGISTER"
LOGIN = "LOGIN"
def utcnow():
"""
获取带UTC时区的当前时间 (最佳实践)
@@ -77,10 +82,14 @@ class InfoSource(Base):
__tablename__ = "info_sources"
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
source_name: Mapped[str] = mapped_column(String(100), comment="信息源名称")
source_type: Mapped[SourceType] = mapped_column(Enum(SourceType))
home_url: Mapped[Optional[str]] = mapped_column(String(255))
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
# 信息源的展示名称,例如 "微博热搜", "今日头条"
source_name: Mapped[str] = mapped_column(String(100), comment="信息源中文名称")
# 抓取类型,决定爬虫调用哪个解析逻辑
source_type: Mapped[SourceType] = mapped_column(Enum(SourceType), comment="抓取方式枚举")
# 极其重要:原意存官网链接,但实际开发中常借用来存放 API的专属标识(如 'weibo', 'toutiao')
home_url: Mapped[Optional[str]] = mapped_column(String(255), comment="官网链接或API的平台标识ID")
# 爬虫开关:如果某平台封禁了我们,可以直接置为False,爬虫将自动跳过该平台
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True, comment="是否开启定时抓取")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
@@ -91,19 +100,22 @@ class InfoSource(Base):
# ==========================================
class UnifiedEvent(Base):
"""
AI 统一事件表
核心业务逻辑:比如微博热搜叫“苹果发布会”知乎热搜叫“iPhone 16 测评”,
它们在子表(TrendingEvent)是两条记录,但通过 AI 语义向量对比后,
会将它们统一挂载到这个表的一个 UnifiedEvent ID 下,实现跨平台事件聚合。
AI 统一事件表 (核心大脑)
逻辑:微博的“苹果发布会”知乎“iPhone 16 测评”,通过语义相似度碰撞后,统一归入此表的一行记录中。
"""
__tablename__ = "unified_events"
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
unified_title: Mapped[str] = mapped_column(String(255), comment="AI统一标题")
ai_comprehensive_summary: Mapped[Optional[str]] = mapped_column(Text, comment="AI全局深度总结")
# 经过AI润色去重后的中立、客观的标准大标题
unified_title: Mapped[str] = mapped_column(String(255), comment="AI生成的客观统一大标题")
# AI阅读子新闻后生成的千字长文摘要,直接用于早报推送
ai_comprehensive_summary: Mapped[Optional[str]] = mapped_column(Text, comment="AI综合全网子新闻生成的深度总结")
center_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="中心向量") # 用于高维空间相似度计算
hot_score: Mapped[int] = mapped_column(Integer, default=0, comment="聚合热度得分")
# [高阶字段] 将文本转化成高维浮点数向量,爬虫抓到新新闻时,跟这个向量算余弦相似度来判断是不是同一个事件
center_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="该事件簇的中心语义向量")
# 事件热度值:挂载的平台越多、相关评论越多,分数越高,用于首页的热榜排序
hot_score: Mapped[int] = mapped_column(Integer, default=0, comment="聚合热度得分(分数越高排名越靠前)")
last_summarized_trends_count: Mapped[int] = mapped_column(Integer, default=0, comment="用于判断是否需要重新调用LLM")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
@@ -115,51 +127,71 @@ class UnifiedEvent(Base):
class TrendingEvent(Base):
"""
各平台热搜数据明细表
存放从爬虫直接拉取下来的最原始的热搜数据。
"""
__tablename__ = "trending_events"
__table_args__ = (
# 联合唯一索引:同一个来源(比如微博)的同一条外部ID(MD5)只能存一条记录,防重插核心保障
# 联合唯一索引:同一个来源的同一个哈希只能存一条,完美实现 UPSERT (去重更新)
UniqueConstraint("source_id", "external_id", name="idx_unique_external_trend"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
# 关联:这条热搜是从哪个平台(InfoSource)抓来的
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="所属信息源ID")
# 关联:它被AI归类到了哪个大事件(UnifiedEvent)之下 (可为空,表示未归类的新鲜事)
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"),
comment="所属的聚合大事件ID")
external_id: Mapped[str] = mapped_column(String(32), comment="32位MD5哈希指纹防重")
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
# 极其核心:将第三方易变的标题/URL,强制压平为32位不变的 MD5 字符串,用作唯一防重指纹
external_id: Mapped[str] = mapped_column(String(32), comment="通过平台ID+原始ID生成的32位MD5防重指纹")
# 这条特定热搜标题的独立语义向量,用于和 unified_events 做碰撞
title_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="标题的语义向量")
icon_url: Mapped[Optional[str]] = mapped_column(String(500))
current_headline: Mapped[str] = mapped_column(String(255))
event_url: Mapped[Optional[str]] = mapped_column(String(500))
app_link: Mapped[Optional[str]] = mapped_column(String(500))
current_ranking: Mapped[Optional[int]] = mapped_column(Integer)
brief_snippet: Mapped[Optional[str]] = mapped_column(Text)
# 爬虫抓下来的热搜配图、带'爆'或'热'字的角标链接
icon_url: Mapped[Optional[str]] = mapped_column(String(500), comment="热榜附带的小图标或配图链接")
# 最新标题 (注意:小编随时可能改标题,所以绝不能放入唯一索引)
current_headline: Mapped[str] = mapped_column(String(255), comment="当前最新的热搜标题")
# 该热点在PC/H5端的访问链接
event_url: Mapped[Optional[str]] = mapped_column(String(500), comment="浏览器访问链接")
# 该热点专门用于手机App唤醒的 DeepLink (如 sinaweibo://...)
app_link: Mapped[Optional[str]] = mapped_column(String(500), comment="移动端App唤醒专属链接")
# 本次抓取时,它在平台上的名次 (如 1, 2, 3)
current_ranking: Mapped[Optional[int]] = mapped_column(Integer, comment="当前最新排名(可能随时上下浮动)")
# 有些平台在热搜底下会配一句话简介
brief_snippet: Mapped[Optional[str]] = mapped_column(Text, comment="平台自带的一句话简介或导语")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
class NewsArticle(Base):
"""新闻文章明细表 (与 TrendingEvent 类似,但侧重长文本阅读)"""
"""
新闻文章明细表 (长篇资讯)
与 TrendingEvent 类似,但它主要用来存放 36氪、纽约时报等长篇正文,用于提供深度的阅读素材。
"""
__tablename__ = "news_articles"
__table_args__ = (
UniqueConstraint("source_id", "external_id", name="idx_unique_external_article"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="所属信息源ID")
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"),
comment="深度文章也可归入大事件分析")
external_id: Mapped[str] = mapped_column(String(32))
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
external_id: Mapped[str] = mapped_column(String(32), comment="RSS原文<guid>生成的MD5防重指纹")
title_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="新闻标题/摘要的语义向量")
cover_image_url: Mapped[Optional[str]] = mapped_column(String(500))
article_title: Mapped[str] = mapped_column(String(255))
article_url: Mapped[Optional[str]] = mapped_column(String(500))
author_name: Mapped[Optional[str]] = mapped_column(String(100))
original_summary: Mapped[Optional[str]] = mapped_column(Text)
publish_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
# 新闻文章的封面大图,很适合前端做瀑布流展示
cover_image_url: Mapped[Optional[str]] = mapped_column(String(500), comment="新闻封面大图链接")
article_title: Mapped[str] = mapped_column(String(255), comment="新闻原文标题")
article_url: Mapped[Optional[str]] = mapped_column(String(500), comment="新闻原文链接")
# 作者或发布机构 (如 "澎湃新闻", "虎嗅作者X")
author_name: Mapped[Optional[str]] = mapped_column(String(100), comment="作者或发布机构名称")
# RSS原文中附带的长摘要,甚至是完整的 HTML 格式正文
original_summary: Mapped[Optional[str]] = mapped_column(Text, comment="原文自带的长摘要或正文片段")
# 新闻在平台上的真实发布时间 (可能比我们爬取的时间要早几天)
publish_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), comment="新闻实际发布的时间")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
@@ -171,33 +203,40 @@ class NewsArticle(Base):
class HeadlineRevision(Base):
"""
标题修订历史表
用于记录平台方暗戳戳修改热搜词条的行为(例如公关介入改标题)
当系统通过哈希发现某条新闻是老熟人,但标题发生了改变时,会自动往这里插一条记录
常用于公关监测(看看谁半夜偷偷改了标题)。
"""
__tablename__ = "headline_revisions"
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
previous_headline: Mapped[str] = mapped_column(String(255))
revised_headline: Mapped[str] = mapped_column(String(255))
# 属于哪一条被修改的热搜
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"), comment="关联的热搜ID")
previous_headline: Mapped[str] = mapped_column(String(255), comment="修改前的旧标题")
revised_headline: Mapped[str] = mapped_column(String(255), comment="修改后的新标题")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
comment="系统发现被修改的时间")
class RankingLog(Base):
"""
热搜排名时间序列化日志
一次抓取都会生成一条记录,可以用于前端绘制热搜“排名起伏折线图”
次爬虫运行(例如每10分钟),都会往这里塞一堆数据,记录某热搜当时的具体名次
前端可以通过这张表画出非常漂亮的“名次起伏折线图(K线图)”。
"""
__tablename__ = "ranking_logs"
__table_args__ = (
# 针对时间序列查询优化的复合索引,加速类似 "查询某事件在过去24小时内的排名变化" 的操作
# 复合索引,加速 "查询某事件在某段时间内的走势"
Index("idx_event_time", "event_id", "observed_at"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
ranking_position: Mapped[int] = mapped_column(Integer)
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"), comment="关联的热搜ID")
# 当时它在第几名
ranking_position: Mapped[int] = mapped_column(Integer, comment="当时抓取时的排名名次")
# 爬虫看到它的那一瞬间的时间
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
comment="观察到该名次的准确时间")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
@@ -205,44 +244,49 @@ class RankingLog(Base):
# ==========================================
# 模块五:多态话题与多态评论
# ==========================================
# 【设计模式】:多态设计
# 通过 target_type (存表名/类型) + target_id (存主键ID) 的组合,
# 让这两个表既能挂载在"单一热搜"下,也能挂载在"新闻文章"下,甚至挂在"统一大事件"下,避免了建立无数个外键的冗余。
class ExtractedTopic(Base):
"""AI 提取的核心话题标签表"""
"""
AI 提取的核心话题标签表
设计模式(多态):一条标签("AI")既能打在大事件上,也能打在单篇文章上。
"""
__tablename__ = "extracted_topics"
__table_args__ = (
Index("idx_topic_keyword", "topic_keyword"),
# 多态查询索引,加速 target_type + target_id 的组合查询
Index("idx_polymorphic_topics", "target_type", "target_id"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
target_id: Mapped[int] = mapped_column(BigIntType)
topic_keyword: Mapped[str] = mapped_column(String(100))
relevance_score: Mapped[Optional[float]] = mapped_column(Float)
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="挂载目标的类型(大事件/热点/文章)")
target_id: Mapped[int] = mapped_column(BigIntType, comment="对应的具体主键ID")
# 提取出的标签词,例如 "自动驾驶", "马斯克"
topic_keyword: Mapped[str] = mapped_column(String(100), comment="提取出的核心关键词汇")
# AI 认为这个词和这篇文章的相关程度(0~100),方便以后做精准度过滤
relevance_score: Mapped[Optional[float]] = mapped_column(Float, comment="AI计算的相关度得分")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class DiscussionComment(Base):
"""全平台统一评论表"""
"""
全平台统一评论库
不论是微博网友的短评,还是新闻网站的长评,全部扔进这张多态表。
"""
__tablename__ = "discussion_comments"
__table_args__ = (
Index("idx_polymorphic_comments", "target_type", "target_id"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
target_id: Mapped[int] = mapped_column(BigIntType)
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="被评论内容的类型")
target_id: Mapped[int] = mapped_column(BigIntType, comment="被评论内容的主键ID")
commenter_name: Mapped[Optional[str]] = mapped_column(String(100))
comment_content: Mapped[str] = mapped_column(Text)
likes_count: Mapped[int] = mapped_column(Integer, default=0)
external_comment_id: Mapped[Optional[str]] = mapped_column(String(32))
comment_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
commenter_name: Mapped[Optional[str]] = mapped_column(String(100), comment="发评人昵称")
comment_content: Mapped[str] = mapped_column(Text, comment="评论正文内容")
# 这条评论本身获得的点赞数,可用于筛选出“神评论”一并推送给用户
likes_count: Mapped[int] = mapped_column(Integer, default=0, comment="评论被点赞的数量")
# 防复抓:用第三方平台原生评论ID做的MD5哈希
external_comment_id: Mapped[Optional[str]] = mapped_column(String(32), comment="第三方评论ID的MD5指纹")
comment_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), comment="评论实际发布时间")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
@@ -251,29 +295,53 @@ class DiscussionComment(Base):
# 模块六:用户画像与多渠道高可用推送系统
# ==========================================
class AppUser(Base):
"""系统核心用户表"""
"""
系统核心用户表
不仅存放密码,还包含用户的基础画像,供大模型做个性化阅读推荐。
"""
__tablename__ = "app_users"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
email: Mapped[str] = mapped_column(String(150), unique=True, index=True)
password_hash: Mapped[Optional[str]] = mapped_column(String(255))
email: Mapped[str] = mapped_column(String(150), unique=True, index=True, comment="主账号邮箱")
password_hash: Mapped[Optional[str]] = mapped_column(String(255), comment="密码哈希(第三方登录可为空)")
nickname: Mapped[Optional[str]] = mapped_column(String(100))
avatar_url: Mapped[Optional[str]] = mapped_column(String(500))
gender: Mapped[GenderType] = mapped_column(Enum(GenderType), default=GenderType.UNKNOWN)
nickname: Mapped[Optional[str]] = mapped_column(String(100), comment="用户展示昵称")
avatar_url: Mapped[Optional[str]] = mapped_column(String(500), comment="用户头像地址")
gender: Mapped[GenderType] = mapped_column(Enum(GenderType), default=GenderType.UNKNOWN,
comment="用户性别(用于AI调整行文语气)")
# 预留的 JSON 字段,可以存放未来灵活变化的用户配置,避免频繁修改表结构
metadata_: Mapped[Optional[Any]] = mapped_column("metadata", JSON, comment="自定义扩展偏好")
# 极其强大:一个万能收纳箱!前端未来想加任何诸如“夜间模式”、“字体变大”的开关,
# 全部丢进这个 JSON 字段即可,从此免去手动修改后端表结构的麻烦。
metadata_: Mapped[Optional[Any]] = mapped_column("metadata", JSON,
comment="JSON扩展字段: 存放灵活多变的前端用户偏好设置")
timezone: Mapped[str] = mapped_column(String(50), default="Asia/Shanghai")
# 时区对于定时推送系统极其重要!保证纽约的用户和北京的用户都能在早晨8点收到新闻。
timezone: Mapped[str] = mapped_column(String(50), default="Asia/Shanghai", comment="用户所在地时区")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
class EmailVerificationCode(Base):
__tablename__ = "email_verification_codes"
__table_args__ = (
Index("idx_email_code_lookup", "email", "purpose", "is_used", "expires_at"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
email: Mapped[str] = mapped_column(String(150), index=True, nullable=False)
purpose: Mapped[VerificationPurpose] = mapped_column(Enum(VerificationPurpose), nullable=False)
code_hash: Mapped[str] = mapped_column(String(64), nullable=False)
is_used: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
class UserPushEndpoint(Base):
"""
多渠道推送端点配置表
一个用户可能绑定了邮箱(EMAIL)和微信(WECHAT),支持配置降级重试(priority_level)。
多渠道推送端点配置表 (高可用解耦设计)
一个用户可以配置好几个推送渠道(邮箱、微信、钉钉),
万一主渠道今天报错了,系统会自动按优先级(priority)降级寻找备用渠道重发。
"""
__tablename__ = "user_push_endpoints"
__table_args__ = (
@@ -281,63 +349,76 @@ class UserPushEndpoint(Base):
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
channel_type: Mapped[str] = mapped_column(String(50), comment="如 EMAIL, WECHAT")
channel_account: Mapped[str] = mapped_column(String(255))
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
priority_level: Mapped[int] = mapped_column(Integer, default=1, comment="1最高,降级重试")
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
# 填入大写的纯字符串,如 EMAIL, WECHAT_BOT, TELEGRAM
channel_type: Mapped[str] = mapped_column(String(50), comment="推送渠道类型标识")
# 具体的发送目标地址
channel_account: Mapped[str] = mapped_column(String(255), comment="具体的接收账号(邮箱号/微信号/Webhook)")
is_active: Mapped[bool] = mapped_column(Boolean, default=True, comment="用户是否临时关闭了该渠道")
# 高可用容灾:比如 1 代表必须先发微信,如果报错了,再去找 priority=2 的邮箱补发
priority_level: Mapped[int] = mapped_column(Integer, default=1, comment="推送优先级(1最高,用于错误降级重试)")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
class UserTopicPreference(Base):
"""用户订阅的兴趣标签库"""
"""
用户订阅的兴趣标签库
当这里的标签和 ExtractedTopic 表里的标签匹配上时,就会触发相关新闻的推送。
"""
__tablename__ = "user_topic_preferences"
__table_args__ = (
# 联合防抖限制:防止用户在界面卡顿时连点两次,订阅了两个同样的词
UniqueConstraint("user_id", "interested_keyword", name="idx_unique_preference"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
interested_keyword: Mapped[str] = mapped_column(String(100))
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
interested_keyword: Mapped[str] = mapped_column(String(100), comment="用户填写的感兴趣标签(如'马斯克')")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class UserDeliverySchedule(Base):
"""用户勿扰/定时推送时间表"""
"""
用户专属的定时推送时间表
如果用户设定了早上 08:30,后台的定时任务就会在每天 08:30 精准地把匹配到的聚合新闻发出去。
"""
__tablename__ = "user_delivery_schedules"
__table_args__ = (
UniqueConstraint("user_id", "delivery_time", name="idx_unique_schedule"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
delivery_time: Mapped[time] = mapped_column(Time, comment="如 08:30:00")
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
delivery_time: Mapped[time] = mapped_column(Time, comment="每天期望收到推送的具体时间")
is_active: Mapped[bool] = mapped_column(Boolean, default=True, comment="是否启用此时段")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
class DeliveryHistory(Base):
"""
推送历史防刷表
核心用途:一旦给某个用户推送过某条新闻/事件,就记录在这里
下次再触发推荐时,检查这个表,防止给同一个用户反复发送相同的内容
推送历史防刷表 (推送系统的绝对底线)
核心业务逻辑:一旦大事件或者文章通过某个渠道成功发给用户,就记一笔帐
明天如果这条旧新闻还在热搜上,系统查一下这个表,发现发过了,直接抛弃,绝不轰炸用户
"""
__tablename__ = "delivery_history"
__table_args__ = (
# 终极去重约束:一个用户,针对同一篇新闻,永远只允许存在一条记录
UniqueConstraint("user_id", "target_type", "target_id", name="idx_prevent_duplicate_push"),
)
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
target_id: Mapped[int] = mapped_column(BigIntType)
status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="接收推送的用户")
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="推送出去的具体内容类型")
target_id: Mapped[int] = mapped_column(BigIntType, comment="推送内容的主键ID")
# 记录这次推送是彻底成功了,还是由于渠道网络问题失败了
status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), comment="最终推送结果状态")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
comment="记录或实际推送的准确时间")
# ==========================================
@@ -345,15 +426,17 @@ class DeliveryHistory(Base):
# ==========================================
class DataSyncTask(Base):
"""
数据同步健康度监控表
这就是爬虫脚本每次运行都要写入记录的地方,用于后台 Dashboard 监控爬虫健康状态和错误堆栈
数据同步健康度监控表 (运维巡检专用)
爬虫每跑完一个平台的轮询,就在这里打卡上报
方便后台画出爬虫成功率的饼图,一旦 error_trace 堆积,能迅速报警排查。
"""
__tablename__ = "data_sync_tasks"
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
items_fetched: Mapped[int] = mapped_column(Integer, default=0)
task_status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
error_trace: Mapped[Optional[str]] = mapped_column(Text)
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="本次运行爬取的哪个源")
items_fetched: Mapped[int] = mapped_column(Integer, default=0, comment="本次爬虫成功插入或更新的新闻条数")
task_status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), comment="该平台的宏观抓取状态")
# 如果代码意外崩溃、或是遭遇403/502,把 Python的 traceback 堆栈原封不动存进这里
error_trace: Mapped[Optional[str]] = mapped_column(Text, comment="若失败则保存完整报错堆栈")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, comment="任务执行的发生时间")
+14
View File
@@ -0,0 +1,14 @@
SUMMARY_SYSTEM_PROMPT = "你是一个输出严格 JSON 格式的后台引擎。"
SUMMARY_USER_PROMPT_TEMPLATE = """
你是一个专业的新闻聚合编辑。请根据以下同一个大事件在不同平台的热搜标题,
为该事件生成一个客观、吸睛的【统一大标题】,以及一段【多平台视角的综合摘要】。
要求:
1. 摘要结构类似:"该事件在多平台发酵。微博侧重讨论...,知乎硬核解析...,科技媒体关注..."
2. 提炼出各平台的讨论侧重点,不要简单罗列标题。
3. 必须以严格的 JSON 格式返回,只包含 "unified_title""ai_comprehensive_summary" 两个字段,不要有多余的说明。
各平台热搜标题数据:
{platform_data_text}
"""
+56
View File
@@ -0,0 +1,56 @@
from datetime import datetime
from typing import Optional
from pydantic import BaseModel, ConfigDict, Field
from app.models.models import GenderType
EMAIL_PATTERN = r"^[^@\s]+@[^@\s]+\.[^@\s]+$"
class RegisterCodeSendRequest(BaseModel):
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
class LoginCodeSendRequest(BaseModel):
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
class RegisterRequest(BaseModel):
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
password: str = Field(..., min_length=8, max_length=128)
verification_code: str = Field(..., pattern=r"^\d{6}$")
nickname: Optional[str] = Field(default=None, max_length=100)
class LoginRequest(BaseModel):
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
password: str = Field(..., min_length=8, max_length=128)
class LoginWithCodeRequest(BaseModel):
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
verification_code: str = Field(..., pattern=r"^\d{6}$")
class UserProfileResponse(BaseModel):
id: int
email: str
nickname: Optional[str]
avatar_url: Optional[str]
gender: GenderType
created_at: datetime
model_config = ConfigDict(from_attributes=True)
class AuthTokenResponse(BaseModel):
access_token: str
token_type: str = "bearer"
expires_in: int
user: UserProfileResponse
class MessageResponse(BaseModel):
message: str
+23
View File
@@ -0,0 +1,23 @@
# app/schemas/event_schema.py
from pydantic import BaseModel
from typing import List, Optional
from datetime import datetime
class PlatformTrendResponse(BaseModel):
source_id: int
platform_name: str # 平台名称,如 "微博热搜"
headline: str # 平台对应的具体热搜标题
url: Optional[str] # 跳转链接
current_ranking: Optional[int] # 当前排名
ranking_history: List[int] # 排名历史轨迹,如 [50, 45, 20, 5, 1],供 ApexCharts 渲染
class UnifiedEventResponse(BaseModel):
event_id: int
unified_title: str # AI 生成的统一大标题
summary: Optional[str] # AI 生成的摘要
hot_score: int # 总热度值
created_at: datetime # 事件发现时间
platforms: List[PlatformTrendResponse] # 挂载的各个平台子热搜
# tags: List[str] = [] # 如果后续打通了 ExtractedTopic,可以在这里返回标签
@@ -1,4 +1,4 @@
# app/schemas/schemas.py
# app/schemas/source_schema.py
from pydantic import BaseModel, ConfigDict, Field
from typing import Optional
from datetime import datetime
+197 -154
View File
@@ -1,62 +1,222 @@
# app/services/fetcher_service.py
import os
import hashlib
import httpx
from dotenv import load_dotenv
from datetime import timedelta
import httpx
import json
import numpy as np
from dotenv import load_dotenv
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
from app.database import SessionLocal
from app.models.models import (
InfoSource, TrendingEvent, NewsArticle, DataSyncTask, TaskStatus,
HeadlineRevision, RankingLog, SourceType, utcnow
HeadlineRevision, RankingLog, SourceType, utcnow, UnifiedEvent
)
# ==========================================
# 环境变量与全局配置
# ==========================================
# 加载环境变量
load_dotenv()
# 从环境变量获取 API 基础地址,如果没有配置则提供默认回退地址
hf_token = os.getenv("HF_TOKEN")
SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", 0.72))
API_BASE_URL = os.getenv("API_BASE_URL", "https://newsnow.busiyi.world/api/s")
EMBEDDING_MODEL_PATH = os.getenv("EMBEDDING_MODEL_PATH", "")
print("正在加载 BAAI/bge-m3 向量模型...")
# 全局单例
embedder_model = SentenceTransformer(EMBEDDING_MODEL_PATH, local_files_only=True, device="cuda")
print("模型加载完成。")
def generate_md5(text: str) -> str:
"""
生成32位MD5哈希值
维护说明:
各个平台(微博、知乎、微信等)返回的原始 ID 格式千奇百怪(有长数字、有UUID、有URL)。
为了方便数据库建立统一的高性能唯一索引(UniqueConstraint),
我们统一将其转为长度固定的 32 位 MD5 字符串作为 external_id。
"""
"""生成32位MD5哈希值作为全局唯一指纹"""
return hashlib.md5(text.encode('utf-8')).hexdigest()
def generate_embedding_json(text: str) -> str:
"""辅助函数:调用大模型生成向量,并序列化为 JSON 字符串"""
raw_vec = embedder_model.encode([text], normalize_embeddings=True)[0]
truncated_vec = [round(float(x), 5) for x in raw_vec]
return json.dumps(truncated_vec, separators=(',', ':'))
def match_or_create_unified_event(db, title: str, embedding_json: str) -> int:
"""
辅助函数:大事件聚类中枢。
拿着新计算的向量去数据库里碰,碰到了就返回老 ID,碰不到就建新的。
"""
# 提取刚算出来的向量
new_vec = np.array(json.loads(embedding_json))
# 只取最近 3 天的活跃大事件进行比对
three_days_ago = utcnow() - timedelta(days=3)
recent_events = db.query(UnifiedEvent).filter(
UnifiedEvent.created_at >= three_days_ago
).order_by(UnifiedEvent.created_at.desc()).limit(200).all()
if recent_events:
valid_events = [ev for ev in recent_events if ev.center_embedding]
if valid_events:
event_vectors = [json.loads(ev.center_embedding) for ev in valid_events]
# 批量矩阵计算相似度
sim_scores = cosine_similarity([new_vec], event_vectors)[0]
max_idx = np.argmax(sim_scores)
if sim_scores[max_idx] >= SIMILARITY_THRESHOLD:
matched_event = valid_events[max_idx]
matched_event.hot_score += 1
return matched_event.id
# 没匹配到,创建一个新的统一大事件
new_unified = UnifiedEvent(
unified_title=title,
center_embedding=embedding_json,
hot_score=1 # 初始热度
)
db.add(new_unified)
db.flush() # 获取自增的主键 ID
return new_unified.id
def process_hot_trend_item(db, source, item, index: int, external_id: str):
"""
处理【热搜/短新闻】的业务逻辑,现已加入 AI 聚类功能
"""
title = item.get("title")
item_url = item.get("url", "")
existing_event = db.query(TrendingEvent).filter(
TrendingEvent.source_id == source.id,
TrendingEvent.external_id == external_id
).first()
event_to_log = None
# 核心逻辑:查重后再决定是否调用模型
if existing_event:
# 场景 A1:老熟人
if existing_event.current_headline != title:
# 标题被暗改,此时需要重新算一次 Embedding
new_embedding_json = generate_embedding_json(title)
revision = HeadlineRevision(
event_id=existing_event.id,
previous_headline=existing_event.current_headline,
revised_headline=title
)
db.add(revision)
existing_event.current_headline = title
existing_event.title_embedding = new_embedding_json # 更新为新标题的语义向量
# 注:这里不改变它所属的 unified_event_id,因为大体还是同一件事
existing_event.current_ranking = index
existing_event.event_url = item_url
event_to_log = existing_event
else:
# 场景 A2:这是一条彻底的全新热搜
# 1. 计算向量
new_embedding_json = generate_embedding_json(title)
# 2. 扔进聚类中枢找归宿
matched_event_id = match_or_create_unified_event(db, title, new_embedding_json)
# 3. 落库
new_event = TrendingEvent(
source_id=source.id,
external_id=external_id,
current_headline=title,
event_url=item_url,
current_ranking=index,
title_embedding=new_embedding_json, # 存入向量
unified_event_id=matched_event_id # 挂载到大事件下
)
db.add(new_event)
db.flush()
event_to_log = new_event
# 强制记录排名轨迹
rank_log = RankingLog(
event_id=event_to_log.id,
ranking_position=index
)
db.add(rank_log)
def process_rss_feed_item(db, source, item, external_id: str):
"""
处理【长文章/传统订阅】分支的核心业务逻辑 (写入 NewsArticle 表)
"""
title = item.get("title")
item_url = item.get("url", "")
existing_article = db.query(NewsArticle).filter(
NewsArticle.source_id == source.id,
NewsArticle.external_id == external_id
).first()
if existing_article:
# 文章若存在,仅更新基础字段
existing_article.article_title = title
existing_article.article_url = item_url
else:
# 全新文章入库
new_article = NewsArticle(
source_id=source.id,
external_id=external_id,
article_title=title,
article_url=item_url,
)
db.add(new_article)
def process_source_data(db, source, items: list) -> int:
"""
数据清洗与路由分发层:
遍历 API 返回的 items,生成唯一指纹,并路由到不同的处理模块。
返回成功处理的条目数量。
"""
saved_count = 0
platform_id = source.home_url
for index, item in enumerate(items, 1):
title = item.get("title")
if not title:
continue
item_url = item.get("url", "")
# ID 兜底策略:接口ID -> URL -> Title
raw_id = item.get("id") or item_url or title
external_id = generate_md5(f"{platform_id}_{raw_id}")
# 核心路由分流
if source.source_type in (SourceType.HOT_TREND, SourceType.API):
process_hot_trend_item(db, source, item, index, external_id)
elif source.source_type == SourceType.RSS_FEED:
process_rss_feed_item(db, source, item, external_id)
saved_count += 1
return saved_count
async def fetch_and_save_trending_data():
"""
核心定时任务:从数据库读取信息源 -> 抓取API -> 解析 -> 根据业务类型分流存入对应的数据库表
执行流程:
1. 查询所有配置为“已启用”的信息源 (is_enabled == True)。
2. 伪装 HTTP 请求头,规避目标服务器的反爬机制。
3. 遍历解析数据,生成 MD5 唯一指纹进行全局去重。
4. 核心路由分流:
- 若源为 HOT_TREND/API,按热搜逻辑处理(记录名次轨迹、标题变更)。
- 若源为 RSS_FEED,按长文章逻辑处理(忽略名次,直接落库)。
5. 严格的事务管理:成功则统一提交,报错则回滚业务数据并独立提交错误日志。
调度层:负责网络请求、数据库事务管理和异常监控隔离。
"""
print(f"[{utcnow()}] 开始执行定时抓取任务...")
# 使用上下文管理器确保数据库连接池正确获取和归还连接
with SessionLocal() as db:
# 1. 动态获取抓取源。
# 优势:在后台修改数据库的信息源开关,下一次定时任务立刻生效,无需重启服务。
# 获取启用的信息源
sources = db.query(InfoSource).filter(InfoSource.is_enabled == True).all()
if not sources:
print("没有找到启用的信息源,任务结束。")
return
# 2. 伪装成真实的浏览器 HTTP 请求头
# 维护注意:如果抓取接口返回 403 Forbidden,通常是这里的反爬策略失效了,需要更新 User-Agent 或 Cookie
# 伪装请求头,规避反爬
custom_headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
"Accept": "application/json, text/plain, */*",
@@ -64,157 +224,40 @@ async def fetch_and_save_trending_data():
"Origin": "https://newsnow.busiyi.world"
}
# 复用异步 HTTP 客户端,比每次循环新建 Client 性能更高
async with httpx.AsyncClient(timeout=15.0, headers=custom_headers) as client:
for source in sources:
# platform_id 对应第三方接口的入参标识,如 "weibo", "zhihu" 等
platform_id = source.home_url
if not platform_id:
continue
# ==========================================
# 【技术债预警 / TODO】
# 目前无论 source_type 是什么,都统一请求了这个 JSON API。
# 未来如果加入了真正的外部 RSS 订阅源(返回的是 XML 格式),
# 这里需要增加判断逻辑:如果是 RSS_FEED,应当使用 feedparser 库去解析 XML,而不是用 httpx 获取 JSON。
# ==========================================
url = f"{API_BASE_URL}?id={platform_id}&latest"
# 初始化本次特定信息源抓取任务的系统监控日志
# 初始化监控日志
task_log = DataSyncTask(source_id=source.id, items_fetched=0)
try:
# 发起请求并校验 HTTP 状态码 (非 2xx 会抛出异常进入 except 块)
# 发起网络请求
response = await client.get(url)
response.raise_for_status()
data_json = response.json()
items = data_json.get("items", [])
saved_count = 0
for index, item in enumerate(items, 1):
title = item.get("title")
if not title:
continue
# 调用数据处理层
saved_count = process_source_data(db, source, items)
item_url = item.get("url", "")
# 3. ID 兜底与去重策略
# 优先用接口自带的 ID -> 没有则用 URL 代替 -> 最差情况用 title 兜底
raw_id = item.get("id") or item_url or title
# 组合“平台标识+原始ID”算出全局唯一的 MD5 外部标识
external_id = generate_md5(f"{platform_id}_{raw_id}")
# ==========================================
# 4. 核心数据分流路由
# 根据信息源的业务类型,将数据推入不同的物理表
# ==========================================
if source.source_type in (SourceType.HOT_TREND, SourceType.API):
# --------------------------------------------------
# 分支 A:热搜/短新闻逻辑 -> 写入 TrendingEvent 表
# --------------------------------------------------
existing_event = db.query(TrendingEvent).filter(
TrendingEvent.source_id == source.id,
TrendingEvent.external_id == external_id
).first()
event_to_log = None # 临时指针,用于后续绑定排名轨迹
if existing_event:
# 场景 A1:该热搜已经在数据库中
# 监控并记录“标题暗改”(常见于热搜公关介入)
if existing_event.current_headline != title:
revision = HeadlineRevision(
event_id=existing_event.id,
previous_headline=existing_event.current_headline,
revised_headline=title
)
db.add(revision)
existing_event.current_headline = title # 覆盖为主表最新标题
# 更新当前的实时排名和 URL
existing_event.current_ranking = index
existing_event.event_url = item_url
event_to_log = existing_event
else:
# 场景 A2:发现全新热搜
new_event = TrendingEvent(
source_id=source.id,
external_id=external_id,
current_headline=title,
event_url=item_url,
current_ranking=index,
)
db.add(new_event)
# db.flush() 是关键:它将数据推给数据库生成了自增的主键 ID,但尚未最终 commit。
# 拿到合法的 event_to_log.id
db.flush()
event_to_log = new_event
# 排名轨迹强制记录
# 只要抓到了热搜(无论新旧),必须打点记录当前名次,用于前端绘制排名趋势图
rank_log = RankingLog(
event_id=event_to_log.id,
ranking_position=index
)
db.add(rank_log)
elif source.source_type == SourceType.RSS_FEED:
# --------------------------------------------------
# 分支 B:长文章/传统订阅逻辑 -> 写入 NewsArticle 表
# --------------------------------------------------
existing_article = db.query(NewsArticle).filter(
NewsArticle.source_id == source.id,
NewsArticle.external_id == external_id
).first()
if existing_article:
# 文章如果已存在,通常只需要更新基础字段(文章一般不涉及排名起伏)
existing_article.article_title = title
existing_article.article_url = item_url
# 预留位置:如果以后接口返回了摘要,可以在这里 update existing_article.original_summary
else:
# 全新文章入库
new_article = NewsArticle(
source_id=source.id,
external_id=external_id,
article_title=title,
article_url=item_url,
# original_summary=item.get("desc", ""),
# author_name=item.get("author", "")
)
db.add(new_article)
saved_count += 1
# --------------------------------------------------
# 5. 业务事务成功提交
# --------------------------------------------------
# 只有当前平台(source)的所有 item 都顺畅走完,才标记成功
# 业务事务成功提交
task_log.items_fetched = saved_count
task_log.task_status = TaskStatus.SUCCESS
db.add(task_log)
# 统一将当前信息源爬取的所有业务数据持久化到硬盘
db.commit()
print(f"[{source.source_name}] ({source.source_type}) 成功抓取并更新了 {saved_count} 条数据")
except Exception as e:
# --------------------------------------------------
# 6. 异常拦截与错误隔离机制
# --------------------------------------------------
# 回滚本次抓取的全部脏数据,
# 异常拦截与错误隔离
db.rollback()
# 错误日志记下来
task_log.task_status = TaskStatus.ERROR
task_log.error_trace = str(e)
db.add(task_log)
db.commit()
print(f"[{source.source_name}] 抓取失败: {e}")
print(f"[{source.source_name}] 抓取失败: {e}")
+104
View File
@@ -0,0 +1,104 @@
# app/services/summary_service.py
import os
import json
from datetime import timedelta
from openai import AsyncOpenAI
from app.database import SessionLocal
from app.models.models import UnifiedEvent, TrendingEvent, InfoSource, utcnow
from app.prompts.summary_prompts import (
SUMMARY_SYSTEM_PROMPT,
SUMMARY_USER_PROMPT_TEMPLATE,
)
HOT_SCORE_THRESHOLD = int(os.getenv("HOT_SCORE_THRESHOLD", 3))
AI_API_KEY = os.getenv("AI_API_KEY", '')
# 1. 初始化异步客户端 (全局复用)
deepseek_client = AsyncOpenAI(
api_key=AI_API_KEY,
base_url="https://api.deepseek.com"
)
async def call_llm_for_summary(platform_data_text: str) -> dict:
"""调用 DeepSeek 生成统一标题和多平台视角摘要"""
prompt = SUMMARY_USER_PROMPT_TEMPLATE.format(
platform_data_text=platform_data_text
)
# await
response = await deepseek_client.chat.completions.create(
model="deepseek-chat",
messages=[
{"role": "system", "content": SUMMARY_SYSTEM_PROMPT},
{"role": "user", "content": prompt}
],
response_format={"type": "json_object"},
temperature=1
)
result_text = response.choices[0].message.content
return json.loads(result_text)
async def generate_unified_summaries():
"""定时任务:扫描高热度事件并生成/更新摘要"""
print(f"[{utcnow()}] 开始执行 DeepSeek 摘要生成任务...")
with SessionLocal() as db:
recent_threshold = utcnow() - timedelta(days=3)
# 必须满足:热度达标 AND (当前热度 > 上次生成摘要时的热度) AND 近期活跃
events = db.query(UnifiedEvent).filter(
UnifiedEvent.hot_score >= HOT_SCORE_THRESHOLD,
UnifiedEvent.hot_score > UnifiedEvent.last_summarized_trends_count,
UnifiedEvent.created_at >= recent_threshold
).all()
if not events:
print("当前没有需要更新摘要的大事件,任务结束。")
return
for event in events:
# 联合查询获取该事件在各平台的子新闻
trends = db.query(TrendingEvent, InfoSource.source_name) \
.join(InfoSource, TrendingEvent.source_id == InfoSource.id) \
.filter(TrendingEvent.unified_event_id == event.id) \
.all()
if not trends:
continue
# 按平台归类标题并去重
platform_dict = {}
for trend_record, source_name in trends:
if source_name not in platform_dict:
platform_dict[source_name] = set()
platform_dict[source_name].add(trend_record.current_headline)
# 组装给大模型的 Prompt 数据
prompt_lines = [f"{platform}】: {', '.join(headlines)}" for platform, headlines in platform_dict.items()]
platform_data_text = "\n".join(prompt_lines)
try:
# 调用封装好的异步函数
llm_result = await call_llm_for_summary(platform_data_text)
if "unified_title" in llm_result:
event.unified_title = llm_result["unified_title"]
if "ai_comprehensive_summary" in llm_result:
event.ai_comprehensive_summary = llm_result["ai_comprehensive_summary"]
# 成功后更新水位线
# 将最后一次总结时的热搜数量,更新为当前最新的 hot_score
event.last_summarized_trends_count = event.hot_score
print(f"成功更新大事件 ID {event.id} 的深度摘要 (当前热度: {event.hot_score})。")
except Exception as e:
print(f"大事件 ID {event.id} 摘要生成失败: {e}")
continue
# 提交事务
db.commit()
+49
View File
@@ -0,0 +1,49 @@
# app/utils/email_utils.py
import os
from email.message import EmailMessage
import aiosmtplib
from dotenv import load_dotenv
import asyncio
load_dotenv()
SMTP_HOST = os.getenv("SMTP_HOST", "smtp.qiye.aliyun.com")
SMTP_PORT = int(os.getenv("SMTP_PORT", 465))
SMTP_USER = os.getenv("SMTP_USER", "noreply@yourdomain.com")
SMTP_PASS = os.getenv("SMTP_PASS", "your_password")
async def send_html_email(
to_email: str,
subject: str,
html_content: str,
sender_name: str = "AI 新闻早报",
sender_email: str = None
) -> bool:
"""底层纯异步发送邮件工具"""
# 如果未指定发送者邮箱,默认使用环境配置中的认证邮箱
if sender_email is None:
sender_email = SMTP_USER
message = EmailMessage()
# 动态拼接 From 字段
message["From"] = f"{sender_name} <{sender_email}>"
message["To"] = to_email
message["Subject"] = subject
# 设定内容为 HTML
message.set_content(html_content, subtype="html")
try:
await aiosmtplib.send(
message,
hostname=SMTP_HOST,
port=SMTP_PORT,
username=SMTP_USER,
password=SMTP_PASS,
use_tls=True,
)
return True
except Exception as e:
print(f"邮件发送至 {to_email} 失败: {str(e)}")
return False