mirror of
https://github.com/stardrophere/InsightRadar.git
synced 2026-06-05 23:32:49 +08:00
login+ai cluster
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
# app/api/dependencies.py
|
||||
from app.database import SessionLocal
|
||||
|
||||
def get_db():
|
||||
"""
|
||||
FastAPI 依赖注入:为每个 HTTP 请求提供独立的数据库会话。
|
||||
请求处理完成后自动关闭,防止连接泄漏。
|
||||
"""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,238 @@
|
||||
import os
|
||||
from datetime import timedelta
|
||||
from typing import Tuple
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.api.dependencies import get_db
|
||||
from app.core.security import (
|
||||
create_access_token,
|
||||
generate_verification_code,
|
||||
hash_password,
|
||||
hash_verification_code,
|
||||
verify_password,
|
||||
verify_verification_code,
|
||||
)
|
||||
from app.models.models import AppUser, EmailVerificationCode, VerificationPurpose, utcnow
|
||||
from app.schemas.auth_schema import (
|
||||
AuthTokenResponse,
|
||||
LoginCodeSendRequest,
|
||||
LoginRequest,
|
||||
LoginWithCodeRequest,
|
||||
MessageResponse,
|
||||
RegisterCodeSendRequest,
|
||||
RegisterRequest,
|
||||
UserProfileResponse,
|
||||
)
|
||||
from app.utils.email_utils import send_html_email
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
REGISTER_CODE_EXPIRE_MINUTES = int(os.getenv("REGISTER_CODE_EXPIRE_MINUTES", "10"))
|
||||
LOGIN_CODE_EXPIRE_MINUTES = int(os.getenv("LOGIN_CODE_EXPIRE_MINUTES", "10"))
|
||||
|
||||
|
||||
def _normalize_email(email: str) -> str:
|
||||
return email.strip().lower()
|
||||
|
||||
|
||||
def _build_verification_email(code: str, purpose_text: str, expire_minutes: int) -> str:
|
||||
return f"""
|
||||
<div style="font-family: Arial, sans-serif; line-height: 1.6; color: #222;">
|
||||
<h2 style="margin-bottom: 12px;">InsightRadar Email Verification</h2>
|
||||
<p>Your {purpose_text} verification code is:</p>
|
||||
<p style="font-size: 28px; font-weight: bold; letter-spacing: 4px; color: #0b57d0;">{code}</p>
|
||||
<p>The code is valid for {expire_minutes} minutes. Do not share it with others.</p>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def _invalidate_unused_codes(db: Session, email: str, purpose: VerificationPurpose) -> None:
|
||||
db.query(EmailVerificationCode).filter(
|
||||
EmailVerificationCode.email == email,
|
||||
EmailVerificationCode.purpose == purpose,
|
||||
EmailVerificationCode.is_used.is_(False),
|
||||
).update({EmailVerificationCode.is_used: True}, synchronize_session=False)
|
||||
db.commit()
|
||||
|
||||
|
||||
def _create_code_record(
|
||||
db: Session,
|
||||
*,
|
||||
email: str,
|
||||
purpose: VerificationPurpose,
|
||||
expire_minutes: int,
|
||||
) -> Tuple[EmailVerificationCode, str]:
|
||||
code = generate_verification_code()
|
||||
now = utcnow()
|
||||
code_record = EmailVerificationCode(
|
||||
email=email,
|
||||
purpose=purpose,
|
||||
code_hash=hash_verification_code(code),
|
||||
expires_at=now + timedelta(minutes=expire_minutes),
|
||||
)
|
||||
db.add(code_record)
|
||||
db.commit()
|
||||
return code_record, code
|
||||
|
||||
|
||||
def _build_auth_response(user: AppUser) -> AuthTokenResponse:
|
||||
token, expires_in = create_access_token(user_id=user.id, email=user.email)
|
||||
return AuthTokenResponse(
|
||||
access_token=token,
|
||||
expires_in=expires_in,
|
||||
user=UserProfileResponse.model_validate(user),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/register/send-code", response_model=MessageResponse)
|
||||
async def send_register_code(payload: RegisterCodeSendRequest, db: Session = Depends(get_db)):
|
||||
email = _normalize_email(payload.email)
|
||||
|
||||
existing_user = db.query(AppUser).filter(AppUser.email == email).first()
|
||||
if existing_user:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email is already registered")
|
||||
|
||||
_invalidate_unused_codes(db, email, VerificationPurpose.REGISTER)
|
||||
code_record, code = _create_code_record(
|
||||
db,
|
||||
email=email,
|
||||
purpose=VerificationPurpose.REGISTER,
|
||||
expire_minutes=REGISTER_CODE_EXPIRE_MINUTES,
|
||||
)
|
||||
|
||||
email_sent = await send_html_email(
|
||||
to_email=email,
|
||||
subject="InsightRadar Registration Code",
|
||||
html_content=_build_verification_email(code, "registration", REGISTER_CODE_EXPIRE_MINUTES),
|
||||
)
|
||||
if not email_sent:
|
||||
code_record.is_used = True
|
||||
db.add(code_record)
|
||||
db.commit()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to send verification code",
|
||||
)
|
||||
|
||||
return MessageResponse(message="Verification code sent")
|
||||
|
||||
|
||||
@router.post("/login/send-code", response_model=MessageResponse)
|
||||
async def send_login_code(payload: LoginCodeSendRequest, db: Session = Depends(get_db)):
|
||||
email = _normalize_email(payload.email)
|
||||
user = db.query(AppUser).filter(AppUser.email == email).first()
|
||||
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Email is not registered")
|
||||
|
||||
_invalidate_unused_codes(db, email, VerificationPurpose.LOGIN)
|
||||
code_record, code = _create_code_record(
|
||||
db,
|
||||
email=email,
|
||||
purpose=VerificationPurpose.LOGIN,
|
||||
expire_minutes=LOGIN_CODE_EXPIRE_MINUTES,
|
||||
)
|
||||
|
||||
email_sent = await send_html_email(
|
||||
to_email=email,
|
||||
subject="InsightRadar Login Code",
|
||||
html_content=_build_verification_email(code, "login", LOGIN_CODE_EXPIRE_MINUTES),
|
||||
)
|
||||
if not email_sent:
|
||||
code_record.is_used = True
|
||||
db.add(code_record)
|
||||
db.commit()
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="Failed to send verification code",
|
||||
)
|
||||
|
||||
return MessageResponse(message="Verification code sent")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/register",
|
||||
response_model=AuthTokenResponse,
|
||||
status_code=status.HTTP_201_CREATED,
|
||||
)
|
||||
async def register(payload: RegisterRequest, db: Session = Depends(get_db)):
|
||||
email = _normalize_email(payload.email)
|
||||
existing_user = db.query(AppUser).filter(AppUser.email == email).first()
|
||||
if existing_user:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Email is already registered")
|
||||
|
||||
now = utcnow()
|
||||
code_record = db.query(EmailVerificationCode).filter(
|
||||
EmailVerificationCode.email == email,
|
||||
EmailVerificationCode.purpose == VerificationPurpose.REGISTER,
|
||||
EmailVerificationCode.is_used.is_(False),
|
||||
EmailVerificationCode.expires_at >= now,
|
||||
).order_by(EmailVerificationCode.created_at.desc()).first()
|
||||
|
||||
if not code_record:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Verification code does not exist or expired")
|
||||
|
||||
if not verify_verification_code(payload.verification_code, code_record.code_hash):
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Invalid verification code")
|
||||
|
||||
nickname = payload.nickname or email.split("@")[0]
|
||||
user = AppUser(
|
||||
email=email,
|
||||
password_hash=hash_password(payload.password),
|
||||
nickname=nickname,
|
||||
metadata_={"email_verified_at": now.isoformat()},
|
||||
)
|
||||
|
||||
code_record.is_used = True
|
||||
db.add(user)
|
||||
db.add(code_record)
|
||||
db.commit()
|
||||
db.refresh(user)
|
||||
|
||||
return _build_auth_response(user)
|
||||
|
||||
|
||||
@router.post("/login", response_model=AuthTokenResponse)
|
||||
async def login(payload: LoginRequest, db: Session = Depends(get_db)):
|
||||
email = _normalize_email(payload.email)
|
||||
user = db.query(AppUser).filter(AppUser.email == email).first()
|
||||
|
||||
if not user or not user.password_hash:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or password")
|
||||
|
||||
if not verify_password(payload.password, user.password_hash):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or password")
|
||||
|
||||
return _build_auth_response(user)
|
||||
|
||||
|
||||
@router.post("/login/code", response_model=AuthTokenResponse)
|
||||
async def login_with_code(payload: LoginWithCodeRequest, db: Session = Depends(get_db)):
|
||||
email = _normalize_email(payload.email)
|
||||
user = db.query(AppUser).filter(AppUser.email == email).first()
|
||||
|
||||
if not user:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or verification code")
|
||||
|
||||
now = utcnow()
|
||||
code_record = db.query(EmailVerificationCode).filter(
|
||||
EmailVerificationCode.email == email,
|
||||
EmailVerificationCode.purpose == VerificationPurpose.LOGIN,
|
||||
EmailVerificationCode.is_used.is_(False),
|
||||
EmailVerificationCode.expires_at >= now,
|
||||
).order_by(EmailVerificationCode.created_at.desc()).first()
|
||||
|
||||
if not code_record:
|
||||
raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Verification code does not exist or expired")
|
||||
|
||||
if not verify_verification_code(payload.verification_code, code_record.code_hash):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid email or verification code")
|
||||
|
||||
code_record.is_used = True
|
||||
db.add(code_record)
|
||||
db.commit()
|
||||
|
||||
return _build_auth_response(user)
|
||||
@@ -0,0 +1,69 @@
|
||||
# app/api/endpoints/events.py
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from datetime import timedelta
|
||||
from typing import List
|
||||
|
||||
from app.api.dependencies import get_db
|
||||
from app.models.models import UnifiedEvent, TrendingEvent, InfoSource, RankingLog, utcnow
|
||||
# 导入你上传的 Schema
|
||||
from app.schemas.event_schema import UnifiedEventResponse, PlatformTrendResponse
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/unified", response_model=List[UnifiedEventResponse])
|
||||
def list_unified_events(
|
||||
min_hot: int = Query(5, description="热度过滤阈值"),
|
||||
hours: int = Query(24, description="查询过去 X 小时的数据"),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""
|
||||
获取聚合大事件列表,完全适配前端 template.html 所需的数据结构
|
||||
"""
|
||||
# 计算时间水位线
|
||||
time_limit = utcnow() - timedelta(hours=hours)
|
||||
|
||||
# 1. 查询大事件(按热度降序,且满足时间范围)
|
||||
events = db.query(UnifiedEvent).filter(
|
||||
UnifiedEvent.hot_score >= min_hot,
|
||||
UnifiedEvent.created_at >= time_limit
|
||||
).order_by(UnifiedEvent.hot_score.desc()).all()
|
||||
|
||||
results = []
|
||||
for ev in events:
|
||||
# 2. 联表查询:获取该大事件下关联的所有平台及其具体热搜信息
|
||||
trends = db.query(TrendingEvent, InfoSource.source_name).join(
|
||||
InfoSource, TrendingEvent.source_id == InfoSource.id
|
||||
).filter(TrendingEvent.unified_event_id == ev.id).all()
|
||||
|
||||
platform_list = []
|
||||
for trend, s_name in trends:
|
||||
# 3. 获取排名历史轨迹 (用于前端渲染)
|
||||
# 这里的排序顺序 asc 保证了数组从旧到新
|
||||
logs = db.query(RankingLog.ranking_position).filter(
|
||||
RankingLog.event_id == trend.id,
|
||||
RankingLog.observed_at >= time_limit
|
||||
).order_by(RankingLog.observed_at.asc()).all()
|
||||
|
||||
# 组装符合 PlatformTrendResponse 结构的字典
|
||||
platform_list.append(PlatformTrendResponse(
|
||||
source_id=trend.source_id,
|
||||
platform_name=s_name,
|
||||
headline=trend.current_headline,
|
||||
url=trend.event_url,
|
||||
current_ranking=trend.current_ranking,
|
||||
ranking_history=[log[0] for log in logs]
|
||||
))
|
||||
|
||||
# 4. 组装符合 UnifiedEventResponse 结构的字典
|
||||
results.append(UnifiedEventResponse(
|
||||
event_id=ev.id,
|
||||
unified_title=ev.unified_title if ev.unified_title else "暂无标题",
|
||||
summary=ev.ai_comprehensive_summary,
|
||||
hot_score=ev.hot_score,
|
||||
created_at=ev.created_at,
|
||||
platforms=platform_list
|
||||
))
|
||||
|
||||
return results
|
||||
@@ -4,7 +4,7 @@ from sqlalchemy.orm import Session
|
||||
from typing import List
|
||||
|
||||
from app.database import get_db
|
||||
from app.schemas.schemas import (
|
||||
from app.schemas.source_schema import (
|
||||
InfoSourceCreate, InfoSourceUpdate, InfoSourceResponse, PaginatedResponse
|
||||
)
|
||||
from app.crud import crud_source
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
# app/api/router.py
|
||||
from fastapi import APIRouter
|
||||
from app.api.endpoints import sources
|
||||
from app.api.endpoints import auth, sources, events
|
||||
|
||||
api_router = APIRouter()
|
||||
|
||||
# 信息源管理
|
||||
api_router.include_router(sources.router, prefix="/sources", tags=["信息源管理"])
|
||||
api_router.include_router(sources.router, prefix="/sources", tags=["信息源管理"])
|
||||
|
||||
# 注册大事件相关的路由
|
||||
api_router.include_router(events.router, prefix="/events", tags=["Unified Events"])
|
||||
api_router.include_router(auth.router, prefix="/auth", tags=["Auth"])
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import time
|
||||
from typing import Tuple
|
||||
|
||||
|
||||
PASSWORD_HASH_ITERATIONS = int(os.getenv("PASSWORD_HASH_ITERATIONS", "120000"))
|
||||
AUTH_SECRET_KEY = os.getenv("AUTH_SECRET_KEY", "change-this-secret-in-env")
|
||||
AUTH_TOKEN_EXPIRE_MINUTES = int(os.getenv("AUTH_TOKEN_EXPIRE_MINUTES", "10080"))
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
salt = secrets.token_hex(16)
|
||||
digest = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
password.encode("utf-8"),
|
||||
salt.encode("utf-8"),
|
||||
PASSWORD_HASH_ITERATIONS,
|
||||
)
|
||||
return (
|
||||
f"pbkdf2_sha256${PASSWORD_HASH_ITERATIONS}${salt}$"
|
||||
f"{base64.urlsafe_b64encode(digest).decode('utf-8')}"
|
||||
)
|
||||
|
||||
|
||||
def verify_password(plain_password: str, password_hash: str) -> bool:
|
||||
try:
|
||||
algorithm, iterations, salt, expected = password_hash.split("$", 3)
|
||||
if algorithm != "pbkdf2_sha256":
|
||||
return False
|
||||
|
||||
digest = hashlib.pbkdf2_hmac(
|
||||
"sha256",
|
||||
plain_password.encode("utf-8"),
|
||||
salt.encode("utf-8"),
|
||||
int(iterations),
|
||||
)
|
||||
calculated = base64.urlsafe_b64encode(digest).decode("utf-8")
|
||||
return hmac.compare_digest(calculated, expected)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def generate_verification_code(length: int = 6) -> str:
|
||||
return "".join(secrets.choice("0123456789") for _ in range(length))
|
||||
|
||||
|
||||
def hash_verification_code(code: str) -> str:
|
||||
return hashlib.sha256(code.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def verify_verification_code(code: str, expected_hash: str) -> bool:
|
||||
return hmac.compare_digest(hash_verification_code(code), expected_hash)
|
||||
|
||||
|
||||
def _urlsafe_b64encode(raw: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(raw).decode("utf-8").rstrip("=")
|
||||
|
||||
|
||||
def create_access_token(user_id: int, email: str) -> Tuple[str, int]:
|
||||
expires_in = AUTH_TOKEN_EXPIRE_MINUTES * 60
|
||||
payload = {
|
||||
"sub": str(user_id),
|
||||
"email": email,
|
||||
"exp": int(time.time()) + expires_in,
|
||||
}
|
||||
payload_bytes = json.dumps(payload, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
|
||||
encoded_payload = _urlsafe_b64encode(payload_bytes)
|
||||
signature = hmac.new(
|
||||
AUTH_SECRET_KEY.encode("utf-8"),
|
||||
encoded_payload.encode("utf-8"),
|
||||
hashlib.sha256,
|
||||
).digest()
|
||||
token = f"{encoded_payload}.{_urlsafe_b64encode(signature)}"
|
||||
return token, expires_in
|
||||
@@ -3,7 +3,7 @@ from sqlalchemy.orm import Session
|
||||
from typing import List, Optional
|
||||
|
||||
from app.models.models import InfoSource
|
||||
from app.schemas.schemas import InfoSourceCreate, InfoSourceUpdate
|
||||
from app.schemas.source_schema import InfoSourceCreate, InfoSourceUpdate
|
||||
|
||||
|
||||
def get(db: Session, source_id: int) -> Optional[InfoSource]:
|
||||
|
||||
+15
-1
@@ -6,6 +6,7 @@ from dotenv import load_dotenv
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from app.services.fetcher_service import fetch_and_save_trending_data
|
||||
from app.services.summary_service import generate_unified_summaries
|
||||
from app.database import engine
|
||||
from app.models.models import Base
|
||||
|
||||
@@ -14,6 +15,7 @@ from app.api.router import api_router
|
||||
|
||||
load_dotenv()
|
||||
CRAWL_INTERVAL = int(os.getenv("CRAWL_INTERVAL_MINUTES", 10))
|
||||
SUMMARY_INTERVAL = int(os.getenv("SUMMARY_INTERVAL_MINUTES", 30))
|
||||
|
||||
scheduler = AsyncIOScheduler()
|
||||
|
||||
@@ -36,15 +38,27 @@ async def lifespan(app: FastAPI):
|
||||
id='trending_fetch_job',
|
||||
replace_existing=True
|
||||
)
|
||||
|
||||
# 平台摘要
|
||||
scheduler.add_job(
|
||||
generate_unified_summaries,
|
||||
'interval',
|
||||
minutes=SUMMARY_INTERVAL,
|
||||
id='ai_summary_job',
|
||||
replace_existing=True
|
||||
)
|
||||
scheduler.start()
|
||||
print(f"定时抓取任务已启动,每 {CRAWL_INTERVAL} 分钟执行一次")
|
||||
print(f"AI 摘要生成任务已启动,每 {SUMMARY_INTERVAL} 分钟执行一次")
|
||||
|
||||
# 为了测试方便,启动时立即执行一次
|
||||
await fetch_and_save_trending_data()
|
||||
|
||||
await generate_unified_summaries()
|
||||
|
||||
yield # 此时 FastAPI 开始接受请求
|
||||
|
||||
# 3. 优雅关闭
|
||||
# 优雅关闭
|
||||
scheduler.shutdown()
|
||||
print("定时任务已安全关闭")
|
||||
|
||||
|
||||
+194
-111
@@ -29,35 +29,40 @@ BigIntType = BigInteger().with_variant(Integer, "sqlite")
|
||||
|
||||
class SourceType(str, enum.Enum):
|
||||
"""信息源的抓取方式"""
|
||||
HOT_TREND = "HOT_TREND" # 热搜榜单类
|
||||
RSS_FEED = "RSS_FEED" # 传统RSS订阅
|
||||
API = "API" # 接口抓取类
|
||||
HOT_TREND = "HOT_TREND" # 热搜榜单类 (如微博热搜)
|
||||
RSS_FEED = "RSS_FEED" # 传统RSS订阅 (如36氪、纽约时报)
|
||||
API = "API" # 接口直接接入类
|
||||
|
||||
|
||||
class TargetType(str, enum.Enum):
|
||||
"""
|
||||
多态目标类型 (Polymorphic Target)
|
||||
用于标记一条评论或一个标签到底是挂载在哪个实体下的。
|
||||
用于标记一条评论、标签或推送记录,到底是挂载在哪个实体下的。
|
||||
"""
|
||||
EVENT = "EVENT" # 挂载在单个热搜事件下
|
||||
TREND = "TREND" # 挂载在宏观趋势下
|
||||
ARTICLE = "ARTICLE" # 挂载在具体新闻文章下
|
||||
EVENT = "EVENT" # 挂载在AI聚合后的大事件下
|
||||
TREND = "TREND" # 挂载在单个平台的热搜条目下
|
||||
ARTICLE = "ARTICLE" # 挂载在具体的长篇新闻文章下
|
||||
|
||||
|
||||
class TaskStatus(str, enum.Enum):
|
||||
"""后台任务状态"""
|
||||
SUCCESS = "SUCCESS"
|
||||
ERROR = "ERROR"
|
||||
"""后台爬虫/推送任务的执行状态"""
|
||||
SUCCESS = "SUCCESS" # 执行成功
|
||||
ERROR = "ERROR" # 发生报错
|
||||
|
||||
|
||||
class GenderType(str, enum.Enum):
|
||||
"""用户性别枚举"""
|
||||
"""用户性别枚举,常用于给AI提供Prompt背景信息以生成个性化摘要"""
|
||||
MALE = "MALE"
|
||||
FEMALE = "FEMALE"
|
||||
OTHER = "OTHER"
|
||||
UNKNOWN = "UNKNOWN"
|
||||
|
||||
|
||||
class VerificationPurpose(str, enum.Enum):
|
||||
REGISTER = "REGISTER"
|
||||
LOGIN = "LOGIN"
|
||||
|
||||
|
||||
def utcnow():
|
||||
"""
|
||||
获取带UTC时区的当前时间 (最佳实践)
|
||||
@@ -77,10 +82,14 @@ class InfoSource(Base):
|
||||
__tablename__ = "info_sources"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
|
||||
source_name: Mapped[str] = mapped_column(String(100), comment="信息源名称")
|
||||
source_type: Mapped[SourceType] = mapped_column(Enum(SourceType))
|
||||
home_url: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
# 信息源的展示名称,例如 "微博热搜", "今日头条"
|
||||
source_name: Mapped[str] = mapped_column(String(100), comment="信息源中文名称")
|
||||
# 抓取类型,决定爬虫调用哪个解析逻辑
|
||||
source_type: Mapped[SourceType] = mapped_column(Enum(SourceType), comment="抓取方式枚举")
|
||||
# 极其重要:原意存官网链接,但实际开发中常借用来存放 API的专属标识(如 'weibo', 'toutiao')
|
||||
home_url: Mapped[Optional[str]] = mapped_column(String(255), comment="官网链接或API的平台标识ID")
|
||||
# 爬虫开关:如果某平台封禁了我们,可以直接置为False,爬虫将自动跳过该平台
|
||||
is_enabled: Mapped[bool] = mapped_column(Boolean, default=True, comment="是否开启定时抓取")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
@@ -91,19 +100,22 @@ class InfoSource(Base):
|
||||
# ==========================================
|
||||
class UnifiedEvent(Base):
|
||||
"""
|
||||
AI 统一事件表
|
||||
核心业务逻辑:比如微博热搜叫“苹果发布会”,知乎热搜叫“iPhone 16 测评”,
|
||||
它们在子表(TrendingEvent)是两条记录,但通过 AI 语义向量对比后,
|
||||
会将它们统一挂载到这个表的一个 UnifiedEvent ID 下,实现跨平台事件聚合。
|
||||
AI 统一事件表 (核心大脑)
|
||||
逻辑:微博的“苹果发布会”和知乎的“iPhone 16 测评”,通过语义相似度碰撞后,统一归入此表的一行记录中。
|
||||
"""
|
||||
__tablename__ = "unified_events"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
unified_title: Mapped[str] = mapped_column(String(255), comment="AI统一标题")
|
||||
ai_comprehensive_summary: Mapped[Optional[str]] = mapped_column(Text, comment="AI全局深度总结")
|
||||
# 经过AI润色去重后的中立、客观的标准大标题
|
||||
unified_title: Mapped[str] = mapped_column(String(255), comment="AI生成的客观统一大标题")
|
||||
# AI阅读子新闻后生成的千字长文摘要,直接用于早报推送
|
||||
ai_comprehensive_summary: Mapped[Optional[str]] = mapped_column(Text, comment="AI综合全网子新闻生成的深度总结")
|
||||
|
||||
center_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="中心向量") # 用于高维空间相似度计算
|
||||
hot_score: Mapped[int] = mapped_column(Integer, default=0, comment="聚合热度得分")
|
||||
# [高阶字段] 将文本转化成高维浮点数向量,爬虫抓到新新闻时,跟这个向量算余弦相似度来判断是不是同一个事件
|
||||
center_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="该事件簇的中心语义向量")
|
||||
# 事件热度值:挂载的平台越多、相关评论越多,分数越高,用于首页的热榜排序
|
||||
hot_score: Mapped[int] = mapped_column(Integer, default=0, comment="聚合热度得分(分数越高排名越靠前)")
|
||||
last_summarized_trends_count: Mapped[int] = mapped_column(Integer, default=0, comment="用于判断是否需要重新调用LLM")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
@@ -115,51 +127,71 @@ class UnifiedEvent(Base):
|
||||
class TrendingEvent(Base):
|
||||
"""
|
||||
各平台热搜数据明细表
|
||||
存放从爬虫直接拉取下来的最原始的热搜数据。
|
||||
"""
|
||||
__tablename__ = "trending_events"
|
||||
__table_args__ = (
|
||||
# 联合唯一索引:同一个来源(比如微博)的同一条外部ID(MD5)只能存在一条记录,防重插核心保障
|
||||
# 联合唯一索引:同一个来源的同一个哈希只能存一条,完美实现 UPSERT (去重更新)
|
||||
UniqueConstraint("source_id", "external_id", name="idx_unique_external_trend"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
||||
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
|
||||
# 关联:这条热搜是从哪个平台(InfoSource)抓来的
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="所属信息源ID")
|
||||
# 关联:它被AI归类到了哪个大事件(UnifiedEvent)之下 (可为空,表示未归类的新鲜事)
|
||||
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"),
|
||||
comment="所属的聚合大事件ID")
|
||||
|
||||
external_id: Mapped[str] = mapped_column(String(32), comment="32位MD5哈希指纹防重")
|
||||
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 极其核心:将第三方易变的标题/URL,强制压平为32位不变的 MD5 字符串,用作唯一防重指纹
|
||||
external_id: Mapped[str] = mapped_column(String(32), comment="通过平台ID+原始ID生成的32位MD5防重指纹")
|
||||
# 这条特定热搜标题的独立语义向量,用于和 unified_events 做碰撞
|
||||
title_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="标题的语义向量")
|
||||
|
||||
icon_url: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
current_headline: Mapped[str] = mapped_column(String(255))
|
||||
event_url: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
app_link: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
current_ranking: Mapped[Optional[int]] = mapped_column(Integer)
|
||||
brief_snippet: Mapped[Optional[str]] = mapped_column(Text)
|
||||
# 爬虫抓下来的热搜配图、带'爆'或'热'字的角标链接
|
||||
icon_url: Mapped[Optional[str]] = mapped_column(String(500), comment="热榜附带的小图标或配图链接")
|
||||
# 最新标题 (注意:小编随时可能改标题,所以绝不能放入唯一索引)
|
||||
current_headline: Mapped[str] = mapped_column(String(255), comment="当前最新的热搜标题")
|
||||
# 该热点在PC/H5端的访问链接
|
||||
event_url: Mapped[Optional[str]] = mapped_column(String(500), comment="浏览器访问链接")
|
||||
# 该热点专门用于手机App唤醒的 DeepLink (如 sinaweibo://...)
|
||||
app_link: Mapped[Optional[str]] = mapped_column(String(500), comment="移动端App唤醒专属链接")
|
||||
# 本次抓取时,它在平台上的名次 (如 1, 2, 3)
|
||||
current_ranking: Mapped[Optional[int]] = mapped_column(Integer, comment="当前最新排名(可能随时上下浮动)")
|
||||
# 有些平台在热搜底下会配一句话简介
|
||||
brief_snippet: Mapped[Optional[str]] = mapped_column(Text, comment="平台自带的一句话简介或导语")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
class NewsArticle(Base):
|
||||
"""新闻文章明细表 (与 TrendingEvent 类似,但侧重长文本阅读)"""
|
||||
"""
|
||||
新闻文章明细表 (长篇资讯)
|
||||
与 TrendingEvent 类似,但它主要用来存放 36氪、纽约时报等长篇正文,用于提供深度的阅读素材。
|
||||
"""
|
||||
__tablename__ = "news_articles"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_id", "external_id", name="idx_unique_external_article"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
||||
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"))
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="所属信息源ID")
|
||||
unified_event_id: Mapped[Optional[int]] = mapped_column(ForeignKey("unified_events.id"),
|
||||
comment="深度文章也可归入大事件分析")
|
||||
|
||||
external_id: Mapped[str] = mapped_column(String(32))
|
||||
title_embedding: Mapped[Optional[str]] = mapped_column(Text)
|
||||
external_id: Mapped[str] = mapped_column(String(32), comment="RSS原文<guid>生成的MD5防重指纹")
|
||||
title_embedding: Mapped[Optional[str]] = mapped_column(Text, comment="新闻标题/摘要的语义向量")
|
||||
|
||||
cover_image_url: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
article_title: Mapped[str] = mapped_column(String(255))
|
||||
article_url: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
author_name: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
original_summary: Mapped[Optional[str]] = mapped_column(Text)
|
||||
publish_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
# 新闻文章的封面大图,很适合前端做瀑布流展示
|
||||
cover_image_url: Mapped[Optional[str]] = mapped_column(String(500), comment="新闻封面大图链接")
|
||||
article_title: Mapped[str] = mapped_column(String(255), comment="新闻原文标题")
|
||||
article_url: Mapped[Optional[str]] = mapped_column(String(500), comment="新闻原文链接")
|
||||
# 作者或发布机构 (如 "澎湃新闻", "虎嗅作者X")
|
||||
author_name: Mapped[Optional[str]] = mapped_column(String(100), comment="作者或发布机构名称")
|
||||
# RSS原文中附带的长摘要,甚至是完整的 HTML 格式正文
|
||||
original_summary: Mapped[Optional[str]] = mapped_column(Text, comment="原文自带的长摘要或正文片段")
|
||||
# 新闻在平台上的真实发布时间 (可能比我们爬取的时间要早几天)
|
||||
publish_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), comment="新闻实际发布的时间")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
@@ -171,33 +203,40 @@ class NewsArticle(Base):
|
||||
class HeadlineRevision(Base):
|
||||
"""
|
||||
标题修订历史表
|
||||
用于记录平台方暗戳戳修改热搜词条的行为(例如公关介入改标题)。
|
||||
当系统通过哈希发现某条新闻是老熟人,但标题发生了改变时,会自动往这里插一条记录。
|
||||
常用于公关监测(看看谁半夜偷偷改了标题)。
|
||||
"""
|
||||
__tablename__ = "headline_revisions"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
|
||||
previous_headline: Mapped[str] = mapped_column(String(255))
|
||||
revised_headline: Mapped[str] = mapped_column(String(255))
|
||||
# 属于哪一条被修改的热搜
|
||||
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"), comment="关联的热搜ID")
|
||||
previous_headline: Mapped[str] = mapped_column(String(255), comment="修改前的旧标题")
|
||||
revised_headline: Mapped[str] = mapped_column(String(255), comment="修改后的新标题")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
|
||||
comment="系统发现被修改的时间")
|
||||
|
||||
|
||||
class RankingLog(Base):
|
||||
"""
|
||||
热搜排名时间序列化日志
|
||||
每一次抓取都会生成一条记录,可以用于前端绘制热搜“排名起伏折线图”。
|
||||
每次爬虫运行(例如每10分钟),都会往这里塞一堆数据,记录某热搜当时的具体名次。
|
||||
前端可以通过这张表画出非常漂亮的“名次起伏折线图(K线图)”。
|
||||
"""
|
||||
__tablename__ = "ranking_logs"
|
||||
__table_args__ = (
|
||||
# 针对时间序列查询优化的复合索引,加速类似 "查询某事件在过去24小时内的排名变化" 的操作
|
||||
# 复合索引,加速 "查询某事件在某段时间内的走势"
|
||||
Index("idx_event_time", "event_id", "observed_at"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"))
|
||||
ranking_position: Mapped[int] = mapped_column(Integer)
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
event_id: Mapped[int] = mapped_column(ForeignKey("trending_events.id"), comment="关联的热搜ID")
|
||||
# 当时它在第几名
|
||||
ranking_position: Mapped[int] = mapped_column(Integer, comment="当时抓取时的排名名次")
|
||||
# 爬虫看到它的那一瞬间的时间
|
||||
observed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
|
||||
comment="观察到该名次的准确时间")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
@@ -205,44 +244,49 @@ class RankingLog(Base):
|
||||
# ==========================================
|
||||
# 模块五:多态话题与多态评论
|
||||
# ==========================================
|
||||
# 【设计模式】:多态设计
|
||||
# 通过 target_type (存表名/类型) + target_id (存主键ID) 的组合,
|
||||
# 让这两个表既能挂载在"单一热搜"下,也能挂载在"新闻文章"下,甚至挂在"统一大事件"下,避免了建立无数个外键的冗余。
|
||||
|
||||
class ExtractedTopic(Base):
|
||||
"""AI 提取的核心话题标签表"""
|
||||
"""
|
||||
AI 提取的核心话题标签表
|
||||
设计模式(多态):一条标签("AI")既能打在大事件上,也能打在单篇文章上。
|
||||
"""
|
||||
__tablename__ = "extracted_topics"
|
||||
__table_args__ = (
|
||||
Index("idx_topic_keyword", "topic_keyword"),
|
||||
# 多态查询索引,加速 target_type + target_id 的组合查询
|
||||
Index("idx_polymorphic_topics", "target_type", "target_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
||||
target_id: Mapped[int] = mapped_column(BigIntType)
|
||||
topic_keyword: Mapped[str] = mapped_column(String(100))
|
||||
relevance_score: Mapped[Optional[float]] = mapped_column(Float)
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="挂载目标的类型(大事件/热点/文章)")
|
||||
target_id: Mapped[int] = mapped_column(BigIntType, comment="对应的具体主键ID")
|
||||
# 提取出的标签词,例如 "自动驾驶", "马斯克"
|
||||
topic_keyword: Mapped[str] = mapped_column(String(100), comment="提取出的核心关键词汇")
|
||||
# AI 认为这个词和这篇文章的相关程度(0~100),方便以后做精准度过滤
|
||||
relevance_score: Mapped[Optional[float]] = mapped_column(Float, comment="AI计算的相关度得分")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class DiscussionComment(Base):
|
||||
"""全平台统一评论表"""
|
||||
"""
|
||||
全平台统一评论库
|
||||
不论是微博网友的短评,还是新闻网站的长评,全部扔进这张多态表。
|
||||
"""
|
||||
__tablename__ = "discussion_comments"
|
||||
__table_args__ = (
|
||||
Index("idx_polymorphic_comments", "target_type", "target_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
||||
target_id: Mapped[int] = mapped_column(BigIntType)
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="被评论内容的类型")
|
||||
target_id: Mapped[int] = mapped_column(BigIntType, comment="被评论内容的主键ID")
|
||||
|
||||
commenter_name: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
comment_content: Mapped[str] = mapped_column(Text)
|
||||
likes_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
external_comment_id: Mapped[Optional[str]] = mapped_column(String(32))
|
||||
comment_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True))
|
||||
commenter_name: Mapped[Optional[str]] = mapped_column(String(100), comment="发评人昵称")
|
||||
comment_content: Mapped[str] = mapped_column(Text, comment="评论正文内容")
|
||||
# 这条评论本身获得的点赞数,可用于筛选出“神评论”一并推送给用户
|
||||
likes_count: Mapped[int] = mapped_column(Integer, default=0, comment="评论被点赞的数量")
|
||||
# 防复抓:用第三方平台原生评论ID做的MD5哈希
|
||||
external_comment_id: Mapped[Optional[str]] = mapped_column(String(32), comment="第三方评论ID的MD5指纹")
|
||||
comment_time: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), comment="评论实际发布时间")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
@@ -251,29 +295,53 @@ class DiscussionComment(Base):
|
||||
# 模块六:用户画像与多渠道高可用推送系统
|
||||
# ==========================================
|
||||
class AppUser(Base):
|
||||
"""系统核心用户表"""
|
||||
"""
|
||||
系统核心用户表
|
||||
不仅存放密码,还包含用户的基础画像,供大模型做个性化阅读推荐。
|
||||
"""
|
||||
__tablename__ = "app_users"
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
email: Mapped[str] = mapped_column(String(150), unique=True, index=True)
|
||||
password_hash: Mapped[Optional[str]] = mapped_column(String(255))
|
||||
email: Mapped[str] = mapped_column(String(150), unique=True, index=True, comment="主账号邮箱")
|
||||
password_hash: Mapped[Optional[str]] = mapped_column(String(255), comment="密码哈希(第三方登录可为空)")
|
||||
|
||||
nickname: Mapped[Optional[str]] = mapped_column(String(100))
|
||||
avatar_url: Mapped[Optional[str]] = mapped_column(String(500))
|
||||
gender: Mapped[GenderType] = mapped_column(Enum(GenderType), default=GenderType.UNKNOWN)
|
||||
nickname: Mapped[Optional[str]] = mapped_column(String(100), comment="用户展示昵称")
|
||||
avatar_url: Mapped[Optional[str]] = mapped_column(String(500), comment="用户头像地址")
|
||||
gender: Mapped[GenderType] = mapped_column(Enum(GenderType), default=GenderType.UNKNOWN,
|
||||
comment="用户性别(用于AI调整行文语气)")
|
||||
|
||||
# 预留的 JSON 字段,可以存放未来灵活变化的用户配置,避免频繁修改表结构
|
||||
metadata_: Mapped[Optional[Any]] = mapped_column("metadata", JSON, comment="自定义扩展偏好")
|
||||
# 极其强大:一个万能收纳箱!前端未来想加任何诸如“夜间模式”、“字体变大”的开关,
|
||||
# 全部丢进这个 JSON 字段即可,从此免去手动修改后端表结构的麻烦。
|
||||
metadata_: Mapped[Optional[Any]] = mapped_column("metadata", JSON,
|
||||
comment="JSON扩展字段: 存放灵活多变的前端用户偏好设置")
|
||||
|
||||
timezone: Mapped[str] = mapped_column(String(50), default="Asia/Shanghai")
|
||||
# 时区对于定时推送系统极其重要!保证纽约的用户和北京的用户都能在早晨8点收到新闻。
|
||||
timezone: Mapped[str] = mapped_column(String(50), default="Asia/Shanghai", comment="用户所在地时区")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
class EmailVerificationCode(Base):
|
||||
__tablename__ = "email_verification_codes"
|
||||
__table_args__ = (
|
||||
Index("idx_email_code_lookup", "email", "purpose", "is_used", "expires_at"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
email: Mapped[str] = mapped_column(String(150), index=True, nullable=False)
|
||||
purpose: Mapped[VerificationPurpose] = mapped_column(Enum(VerificationPurpose), nullable=False)
|
||||
code_hash: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
is_used: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
class UserPushEndpoint(Base):
|
||||
"""
|
||||
多渠道推送端点配置表
|
||||
一个用户可能绑定了邮箱(EMAIL)和微信(WECHAT),支持配置降级重试(priority_level)。
|
||||
多渠道推送端点配置表 (高可用解耦设计)
|
||||
一个用户可以配置好几个推送渠道(邮箱、微信、钉钉),
|
||||
万一主渠道今天报错了,系统会自动按优先级(priority)降级寻找备用渠道重发。
|
||||
"""
|
||||
__tablename__ = "user_push_endpoints"
|
||||
__table_args__ = (
|
||||
@@ -281,63 +349,76 @@ class UserPushEndpoint(Base):
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
||||
channel_type: Mapped[str] = mapped_column(String(50), comment="如 EMAIL, WECHAT")
|
||||
channel_account: Mapped[str] = mapped_column(String(255))
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
priority_level: Mapped[int] = mapped_column(Integer, default=1, comment="1最高,降级重试")
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
|
||||
# 填入大写的纯字符串,如 EMAIL, WECHAT_BOT, TELEGRAM
|
||||
channel_type: Mapped[str] = mapped_column(String(50), comment="推送渠道类型标识")
|
||||
# 具体的发送目标地址
|
||||
channel_account: Mapped[str] = mapped_column(String(255), comment="具体的接收账号(邮箱号/微信号/Webhook)")
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, comment="用户是否临时关闭了该渠道")
|
||||
# 高可用容灾:比如 1 代表必须先发微信,如果报错了,再去找 priority=2 的邮箱补发
|
||||
priority_level: Mapped[int] = mapped_column(Integer, default=1, comment="推送优先级(1最高,用于错误降级重试)")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, onupdate=utcnow)
|
||||
|
||||
|
||||
class UserTopicPreference(Base):
|
||||
"""用户订阅的兴趣标签库"""
|
||||
"""
|
||||
用户订阅的兴趣标签库
|
||||
当这里的标签和 ExtractedTopic 表里的标签匹配上时,就会触发相关新闻的推送。
|
||||
"""
|
||||
__tablename__ = "user_topic_preferences"
|
||||
__table_args__ = (
|
||||
# 联合防抖限制:防止用户在界面卡顿时连点两次,订阅了两个同样的词
|
||||
UniqueConstraint("user_id", "interested_keyword", name="idx_unique_preference"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
||||
interested_keyword: Mapped[str] = mapped_column(String(100))
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
|
||||
interested_keyword: Mapped[str] = mapped_column(String(100), comment="用户填写的感兴趣标签(如'马斯克')")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class UserDeliverySchedule(Base):
|
||||
"""用户勿扰/定时推送时间表"""
|
||||
"""
|
||||
用户专属的定时推送时间表
|
||||
如果用户设定了早上 08:30,后台的定时任务就会在每天 08:30 精准地把匹配到的聚合新闻发出去。
|
||||
"""
|
||||
__tablename__ = "user_delivery_schedules"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("user_id", "delivery_time", name="idx_unique_schedule"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
||||
delivery_time: Mapped[time] = mapped_column(Time, comment="如 08:30:00")
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="所属用户ID")
|
||||
delivery_time: Mapped[time] = mapped_column(Time, comment="每天期望收到推送的具体时间")
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True, comment="是否启用此时段")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
|
||||
class DeliveryHistory(Base):
|
||||
"""
|
||||
推送历史防刷表
|
||||
核心用途:一旦给某个用户推送过某条新闻/事件,就记录在这里。
|
||||
下次再触发推荐时,检查这个表,防止给同一个用户反复发送相同的内容。
|
||||
推送历史防刷表 (推送系统的绝对底线)
|
||||
核心业务逻辑:一旦大事件或者文章通过某个渠道成功发给用户,就记一笔帐。
|
||||
明天如果这条旧新闻还在热搜上,系统查一下这个表,发现发过了,直接抛弃,绝不轰炸用户。
|
||||
"""
|
||||
__tablename__ = "delivery_history"
|
||||
__table_args__ = (
|
||||
# 终极去重约束:一个用户,针对同一篇新闻,永远只允许存在一条记录
|
||||
UniqueConstraint("user_id", "target_type", "target_id", name="idx_prevent_duplicate_push"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"))
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType))
|
||||
target_id: Mapped[int] = mapped_column(BigIntType)
|
||||
status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_users.id"), comment="接收推送的用户")
|
||||
target_type: Mapped[TargetType] = mapped_column(Enum(TargetType), comment="推送出去的具体内容类型")
|
||||
target_id: Mapped[int] = mapped_column(BigIntType, comment="推送内容的主键ID")
|
||||
# 记录这次推送是彻底成功了,还是由于渠道网络问题失败了
|
||||
status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), comment="最终推送结果状态")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow,
|
||||
comment="记录或实际推送的准确时间")
|
||||
|
||||
|
||||
# ==========================================
|
||||
@@ -345,15 +426,17 @@ class DeliveryHistory(Base):
|
||||
# ==========================================
|
||||
class DataSyncTask(Base):
|
||||
"""
|
||||
数据同步健康度监控表
|
||||
这就是爬虫脚本每次运行都要写入记录的地方,用于后台 Dashboard 监控爬虫健康状态和错误堆栈。
|
||||
数据同步健康度监控表 (运维巡检专用)
|
||||
爬虫每跑完一个平台的轮询,就在这里打卡上报。
|
||||
方便后台画出爬虫成功率的饼图,一旦 error_trace 堆积,能迅速报警排查。
|
||||
"""
|
||||
__tablename__ = "data_sync_tasks"
|
||||
|
||||
id: Mapped[int] = mapped_column(BigIntType, primary_key=True, autoincrement=True)
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"))
|
||||
items_fetched: Mapped[int] = mapped_column(Integer, default=0)
|
||||
task_status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus))
|
||||
error_trace: Mapped[Optional[str]] = mapped_column(Text)
|
||||
source_id: Mapped[int] = mapped_column(ForeignKey("info_sources.id"), comment="本次运行爬取的哪个源")
|
||||
items_fetched: Mapped[int] = mapped_column(Integer, default=0, comment="本次爬虫成功插入或更新的新闻条数")
|
||||
task_status: Mapped[TaskStatus] = mapped_column(Enum(TaskStatus), comment="该平台的宏观抓取状态")
|
||||
# 如果代码意外崩溃、或是遭遇403/502,把 Python的 traceback 堆栈原封不动存进这里
|
||||
error_trace: Mapped[Optional[str]] = mapped_column(Text, comment="若失败则保存完整报错堆栈")
|
||||
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, comment="任务执行的发生时间")
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
SUMMARY_SYSTEM_PROMPT = "你是一个输出严格 JSON 格式的后台引擎。"
|
||||
|
||||
SUMMARY_USER_PROMPT_TEMPLATE = """
|
||||
你是一个专业的新闻聚合编辑。请根据以下同一个大事件在不同平台的热搜标题,
|
||||
为该事件生成一个客观、吸睛的【统一大标题】,以及一段【多平台视角的综合摘要】。
|
||||
|
||||
要求:
|
||||
1. 摘要结构类似:"该事件在多平台发酵。微博侧重讨论...,知乎硬核解析...,科技媒体关注..."。
|
||||
2. 提炼出各平台的讨论侧重点,不要简单罗列标题。
|
||||
3. 必须以严格的 JSON 格式返回,只包含 "unified_title" 和 "ai_comprehensive_summary" 两个字段,不要有多余的说明。
|
||||
|
||||
各平台热搜标题数据:
|
||||
{platform_data_text}
|
||||
"""
|
||||
@@ -0,0 +1,56 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.models.models import GenderType
|
||||
|
||||
|
||||
EMAIL_PATTERN = r"^[^@\s]+@[^@\s]+\.[^@\s]+$"
|
||||
|
||||
|
||||
class RegisterCodeSendRequest(BaseModel):
|
||||
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
|
||||
|
||||
|
||||
class LoginCodeSendRequest(BaseModel):
|
||||
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
|
||||
|
||||
|
||||
class RegisterRequest(BaseModel):
|
||||
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
|
||||
password: str = Field(..., min_length=8, max_length=128)
|
||||
verification_code: str = Field(..., pattern=r"^\d{6}$")
|
||||
nickname: Optional[str] = Field(default=None, max_length=100)
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
|
||||
password: str = Field(..., min_length=8, max_length=128)
|
||||
|
||||
|
||||
class LoginWithCodeRequest(BaseModel):
|
||||
email: str = Field(..., max_length=150, pattern=EMAIL_PATTERN)
|
||||
verification_code: str = Field(..., pattern=r"^\d{6}$")
|
||||
|
||||
|
||||
class UserProfileResponse(BaseModel):
|
||||
id: int
|
||||
email: str
|
||||
nickname: Optional[str]
|
||||
avatar_url: Optional[str]
|
||||
gender: GenderType
|
||||
created_at: datetime
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
class AuthTokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
expires_in: int
|
||||
user: UserProfileResponse
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
message: str
|
||||
@@ -0,0 +1,23 @@
|
||||
# app/schemas/event_schema.py
|
||||
from pydantic import BaseModel
|
||||
from typing import List, Optional
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
class PlatformTrendResponse(BaseModel):
|
||||
source_id: int
|
||||
platform_name: str # 平台名称,如 "微博热搜"
|
||||
headline: str # 平台对应的具体热搜标题
|
||||
url: Optional[str] # 跳转链接
|
||||
current_ranking: Optional[int] # 当前排名
|
||||
ranking_history: List[int] # 排名历史轨迹,如 [50, 45, 20, 5, 1],供 ApexCharts 渲染
|
||||
|
||||
|
||||
class UnifiedEventResponse(BaseModel):
|
||||
event_id: int
|
||||
unified_title: str # AI 生成的统一大标题
|
||||
summary: Optional[str] # AI 生成的摘要
|
||||
hot_score: int # 总热度值
|
||||
created_at: datetime # 事件发现时间
|
||||
platforms: List[PlatformTrendResponse] # 挂载的各个平台子热搜
|
||||
# tags: List[str] = [] # 如果后续打通了 ExtractedTopic,可以在这里返回标签
|
||||
@@ -1,4 +1,4 @@
|
||||
# app/schemas/schemas.py
|
||||
# app/schemas/source_schema.py
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from typing import Optional
|
||||
from datetime import datetime
|
||||
@@ -1,62 +1,222 @@
|
||||
# app/services/fetcher_service.py
|
||||
import os
|
||||
import hashlib
|
||||
import httpx
|
||||
from dotenv import load_dotenv
|
||||
from datetime import timedelta
|
||||
|
||||
import httpx
|
||||
import json
|
||||
import numpy as np
|
||||
from dotenv import load_dotenv
|
||||
from sklearn.metrics.pairwise import cosine_similarity
|
||||
from sentence_transformers import SentenceTransformer
|
||||
from app.database import SessionLocal
|
||||
from app.models.models import (
|
||||
InfoSource, TrendingEvent, NewsArticle, DataSyncTask, TaskStatus,
|
||||
HeadlineRevision, RankingLog, SourceType, utcnow
|
||||
HeadlineRevision, RankingLog, SourceType, utcnow, UnifiedEvent
|
||||
)
|
||||
|
||||
# ==========================================
|
||||
# 环境变量与全局配置
|
||||
# ==========================================
|
||||
# 加载环境变量
|
||||
load_dotenv()
|
||||
|
||||
# 从环境变量获取 API 基础地址,如果没有配置则提供默认回退地址
|
||||
hf_token = os.getenv("HF_TOKEN")
|
||||
SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", 0.72))
|
||||
API_BASE_URL = os.getenv("API_BASE_URL", "https://newsnow.busiyi.world/api/s")
|
||||
EMBEDDING_MODEL_PATH = os.getenv("EMBEDDING_MODEL_PATH", "")
|
||||
|
||||
print("正在加载 BAAI/bge-m3 向量模型...")
|
||||
# 全局单例
|
||||
embedder_model = SentenceTransformer(EMBEDDING_MODEL_PATH, local_files_only=True, device="cuda")
|
||||
print("模型加载完成。")
|
||||
|
||||
|
||||
def generate_md5(text: str) -> str:
|
||||
"""
|
||||
生成32位MD5哈希值
|
||||
维护说明:
|
||||
各个平台(微博、知乎、微信等)返回的原始 ID 格式千奇百怪(有长数字、有UUID、有URL)。
|
||||
为了方便数据库建立统一的高性能唯一索引(UniqueConstraint),
|
||||
我们统一将其转为长度固定的 32 位 MD5 字符串作为 external_id。
|
||||
"""
|
||||
"""生成32位MD5哈希值作为全局唯一指纹"""
|
||||
return hashlib.md5(text.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def generate_embedding_json(text: str) -> str:
|
||||
"""辅助函数:调用大模型生成向量,并序列化为 JSON 字符串"""
|
||||
raw_vec = embedder_model.encode([text], normalize_embeddings=True)[0]
|
||||
truncated_vec = [round(float(x), 5) for x in raw_vec]
|
||||
return json.dumps(truncated_vec, separators=(',', ':'))
|
||||
|
||||
|
||||
def match_or_create_unified_event(db, title: str, embedding_json: str) -> int:
|
||||
"""
|
||||
辅助函数:大事件聚类中枢。
|
||||
拿着新计算的向量去数据库里碰,碰到了就返回老 ID,碰不到就建新的。
|
||||
"""
|
||||
# 提取刚算出来的向量
|
||||
new_vec = np.array(json.loads(embedding_json))
|
||||
|
||||
# 只取最近 3 天的活跃大事件进行比对
|
||||
three_days_ago = utcnow() - timedelta(days=3)
|
||||
recent_events = db.query(UnifiedEvent).filter(
|
||||
UnifiedEvent.created_at >= three_days_ago
|
||||
).order_by(UnifiedEvent.created_at.desc()).limit(200).all()
|
||||
|
||||
if recent_events:
|
||||
valid_events = [ev for ev in recent_events if ev.center_embedding]
|
||||
if valid_events:
|
||||
event_vectors = [json.loads(ev.center_embedding) for ev in valid_events]
|
||||
|
||||
# 批量矩阵计算相似度
|
||||
sim_scores = cosine_similarity([new_vec], event_vectors)[0]
|
||||
max_idx = np.argmax(sim_scores)
|
||||
|
||||
if sim_scores[max_idx] >= SIMILARITY_THRESHOLD:
|
||||
matched_event = valid_events[max_idx]
|
||||
|
||||
matched_event.hot_score += 1
|
||||
return matched_event.id
|
||||
|
||||
# 没匹配到,创建一个新的统一大事件
|
||||
new_unified = UnifiedEvent(
|
||||
unified_title=title,
|
||||
center_embedding=embedding_json,
|
||||
hot_score=1 # 初始热度
|
||||
)
|
||||
db.add(new_unified)
|
||||
db.flush() # 获取自增的主键 ID
|
||||
return new_unified.id
|
||||
|
||||
|
||||
def process_hot_trend_item(db, source, item, index: int, external_id: str):
|
||||
"""
|
||||
处理【热搜/短新闻】的业务逻辑,现已加入 AI 聚类功能
|
||||
"""
|
||||
title = item.get("title")
|
||||
item_url = item.get("url", "")
|
||||
|
||||
existing_event = db.query(TrendingEvent).filter(
|
||||
TrendingEvent.source_id == source.id,
|
||||
TrendingEvent.external_id == external_id
|
||||
).first()
|
||||
|
||||
event_to_log = None
|
||||
|
||||
# 核心逻辑:查重后再决定是否调用模型
|
||||
if existing_event:
|
||||
# 场景 A1:老熟人
|
||||
if existing_event.current_headline != title:
|
||||
# 标题被暗改,此时需要重新算一次 Embedding
|
||||
new_embedding_json = generate_embedding_json(title)
|
||||
|
||||
revision = HeadlineRevision(
|
||||
event_id=existing_event.id,
|
||||
previous_headline=existing_event.current_headline,
|
||||
revised_headline=title
|
||||
)
|
||||
db.add(revision)
|
||||
existing_event.current_headline = title
|
||||
existing_event.title_embedding = new_embedding_json # 更新为新标题的语义向量
|
||||
# 注:这里不改变它所属的 unified_event_id,因为大体还是同一件事
|
||||
|
||||
existing_event.current_ranking = index
|
||||
existing_event.event_url = item_url
|
||||
event_to_log = existing_event
|
||||
|
||||
else:
|
||||
# 场景 A2:这是一条彻底的全新热搜
|
||||
# 1. 计算向量
|
||||
new_embedding_json = generate_embedding_json(title)
|
||||
|
||||
# 2. 扔进聚类中枢找归宿
|
||||
matched_event_id = match_or_create_unified_event(db, title, new_embedding_json)
|
||||
|
||||
# 3. 落库
|
||||
new_event = TrendingEvent(
|
||||
source_id=source.id,
|
||||
external_id=external_id,
|
||||
current_headline=title,
|
||||
event_url=item_url,
|
||||
current_ranking=index,
|
||||
title_embedding=new_embedding_json, # 存入向量
|
||||
unified_event_id=matched_event_id # 挂载到大事件下
|
||||
)
|
||||
db.add(new_event)
|
||||
db.flush()
|
||||
event_to_log = new_event
|
||||
|
||||
# 强制记录排名轨迹
|
||||
rank_log = RankingLog(
|
||||
event_id=event_to_log.id,
|
||||
ranking_position=index
|
||||
)
|
||||
db.add(rank_log)
|
||||
|
||||
|
||||
def process_rss_feed_item(db, source, item, external_id: str):
|
||||
"""
|
||||
处理【长文章/传统订阅】分支的核心业务逻辑 (写入 NewsArticle 表)
|
||||
"""
|
||||
title = item.get("title")
|
||||
item_url = item.get("url", "")
|
||||
|
||||
existing_article = db.query(NewsArticle).filter(
|
||||
NewsArticle.source_id == source.id,
|
||||
NewsArticle.external_id == external_id
|
||||
).first()
|
||||
|
||||
if existing_article:
|
||||
# 文章若存在,仅更新基础字段
|
||||
existing_article.article_title = title
|
||||
existing_article.article_url = item_url
|
||||
else:
|
||||
# 全新文章入库
|
||||
new_article = NewsArticle(
|
||||
source_id=source.id,
|
||||
external_id=external_id,
|
||||
article_title=title,
|
||||
article_url=item_url,
|
||||
)
|
||||
db.add(new_article)
|
||||
|
||||
|
||||
def process_source_data(db, source, items: list) -> int:
|
||||
"""
|
||||
数据清洗与路由分发层:
|
||||
遍历 API 返回的 items,生成唯一指纹,并路由到不同的处理模块。
|
||||
返回成功处理的条目数量。
|
||||
"""
|
||||
saved_count = 0
|
||||
platform_id = source.home_url
|
||||
|
||||
for index, item in enumerate(items, 1):
|
||||
title = item.get("title")
|
||||
if not title:
|
||||
continue
|
||||
|
||||
item_url = item.get("url", "")
|
||||
|
||||
# ID 兜底策略:接口ID -> URL -> Title
|
||||
raw_id = item.get("id") or item_url or title
|
||||
external_id = generate_md5(f"{platform_id}_{raw_id}")
|
||||
|
||||
# 核心路由分流
|
||||
if source.source_type in (SourceType.HOT_TREND, SourceType.API):
|
||||
process_hot_trend_item(db, source, item, index, external_id)
|
||||
elif source.source_type == SourceType.RSS_FEED:
|
||||
process_rss_feed_item(db, source, item, external_id)
|
||||
|
||||
saved_count += 1
|
||||
|
||||
return saved_count
|
||||
|
||||
|
||||
async def fetch_and_save_trending_data():
|
||||
"""
|
||||
核心定时任务:从数据库读取信息源 -> 抓取API -> 解析 -> 根据业务类型分流存入对应的数据库表
|
||||
|
||||
执行流程:
|
||||
1. 查询所有配置为“已启用”的信息源 (is_enabled == True)。
|
||||
2. 伪装 HTTP 请求头,规避目标服务器的反爬机制。
|
||||
3. 遍历解析数据,生成 MD5 唯一指纹进行全局去重。
|
||||
4. 核心路由分流:
|
||||
- 若源为 HOT_TREND/API,按热搜逻辑处理(记录名次轨迹、标题变更)。
|
||||
- 若源为 RSS_FEED,按长文章逻辑处理(忽略名次,直接落库)。
|
||||
5. 严格的事务管理:成功则统一提交,报错则回滚业务数据并独立提交错误日志。
|
||||
调度层:负责网络请求、数据库事务管理和异常监控隔离。
|
||||
"""
|
||||
print(f"[{utcnow()}] 开始执行定时抓取任务...")
|
||||
|
||||
# 使用上下文管理器确保数据库连接池正确获取和归还连接
|
||||
with SessionLocal() as db:
|
||||
# 1. 动态获取抓取源。
|
||||
# 优势:在后台修改数据库的信息源开关,下一次定时任务立刻生效,无需重启服务。
|
||||
# 获取启用的信息源
|
||||
sources = db.query(InfoSource).filter(InfoSource.is_enabled == True).all()
|
||||
if not sources:
|
||||
print("没有找到启用的信息源,任务结束。")
|
||||
return
|
||||
|
||||
# 2. 伪装成真实的浏览器 HTTP 请求头
|
||||
# 维护注意:如果抓取接口返回 403 Forbidden,通常是这里的反爬策略失效了,需要更新 User-Agent 或 Cookie
|
||||
# 伪装请求头,规避反爬
|
||||
custom_headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
@@ -64,157 +224,40 @@ async def fetch_and_save_trending_data():
|
||||
"Origin": "https://newsnow.busiyi.world"
|
||||
}
|
||||
|
||||
# 复用异步 HTTP 客户端,比每次循环新建 Client 性能更高
|
||||
async with httpx.AsyncClient(timeout=15.0, headers=custom_headers) as client:
|
||||
for source in sources:
|
||||
# platform_id 对应第三方接口的入参标识,如 "weibo", "zhihu" 等
|
||||
platform_id = source.home_url
|
||||
if not platform_id:
|
||||
continue
|
||||
|
||||
# ==========================================
|
||||
# 【技术债预警 / TODO】
|
||||
# 目前无论 source_type 是什么,都统一请求了这个 JSON API。
|
||||
# 未来如果加入了真正的外部 RSS 订阅源(返回的是 XML 格式),
|
||||
# 这里需要增加判断逻辑:如果是 RSS_FEED,应当使用 feedparser 库去解析 XML,而不是用 httpx 获取 JSON。
|
||||
# ==========================================
|
||||
url = f"{API_BASE_URL}?id={platform_id}&latest"
|
||||
|
||||
# 初始化本次特定信息源抓取任务的系统监控日志
|
||||
# 初始化监控日志
|
||||
task_log = DataSyncTask(source_id=source.id, items_fetched=0)
|
||||
|
||||
try:
|
||||
# 发起请求并校验 HTTP 状态码 (非 2xx 会抛出异常进入 except 块)
|
||||
# 发起网络请求
|
||||
response = await client.get(url)
|
||||
response.raise_for_status()
|
||||
data_json = response.json()
|
||||
|
||||
items = data_json.get("items", [])
|
||||
saved_count = 0
|
||||
|
||||
for index, item in enumerate(items, 1):
|
||||
title = item.get("title")
|
||||
if not title:
|
||||
continue
|
||||
# 调用数据处理层
|
||||
saved_count = process_source_data(db, source, items)
|
||||
|
||||
item_url = item.get("url", "")
|
||||
|
||||
# 3. ID 兜底与去重策略
|
||||
# 优先用接口自带的 ID -> 没有则用 URL 代替 -> 最差情况用 title 兜底
|
||||
raw_id = item.get("id") or item_url or title
|
||||
# 组合“平台标识+原始ID”算出全局唯一的 MD5 外部标识
|
||||
external_id = generate_md5(f"{platform_id}_{raw_id}")
|
||||
|
||||
# ==========================================
|
||||
# 4. 核心数据分流路由
|
||||
# 根据信息源的业务类型,将数据推入不同的物理表
|
||||
# ==========================================
|
||||
|
||||
if source.source_type in (SourceType.HOT_TREND, SourceType.API):
|
||||
# --------------------------------------------------
|
||||
# 分支 A:热搜/短新闻逻辑 -> 写入 TrendingEvent 表
|
||||
# --------------------------------------------------
|
||||
existing_event = db.query(TrendingEvent).filter(
|
||||
TrendingEvent.source_id == source.id,
|
||||
TrendingEvent.external_id == external_id
|
||||
).first()
|
||||
|
||||
event_to_log = None # 临时指针,用于后续绑定排名轨迹
|
||||
|
||||
if existing_event:
|
||||
# 场景 A1:该热搜已经在数据库中
|
||||
|
||||
# 监控并记录“标题暗改”(常见于热搜公关介入)
|
||||
if existing_event.current_headline != title:
|
||||
revision = HeadlineRevision(
|
||||
event_id=existing_event.id,
|
||||
previous_headline=existing_event.current_headline,
|
||||
revised_headline=title
|
||||
)
|
||||
db.add(revision)
|
||||
existing_event.current_headline = title # 覆盖为主表最新标题
|
||||
|
||||
# 更新当前的实时排名和 URL
|
||||
existing_event.current_ranking = index
|
||||
existing_event.event_url = item_url
|
||||
event_to_log = existing_event
|
||||
else:
|
||||
# 场景 A2:发现全新热搜
|
||||
new_event = TrendingEvent(
|
||||
source_id=source.id,
|
||||
external_id=external_id,
|
||||
current_headline=title,
|
||||
event_url=item_url,
|
||||
current_ranking=index,
|
||||
)
|
||||
db.add(new_event)
|
||||
# db.flush() 是关键:它将数据推给数据库生成了自增的主键 ID,但尚未最终 commit。
|
||||
# 拿到合法的 event_to_log.id
|
||||
db.flush()
|
||||
event_to_log = new_event
|
||||
|
||||
# 排名轨迹强制记录
|
||||
# 只要抓到了热搜(无论新旧),必须打点记录当前名次,用于前端绘制排名趋势图
|
||||
rank_log = RankingLog(
|
||||
event_id=event_to_log.id,
|
||||
ranking_position=index
|
||||
)
|
||||
db.add(rank_log)
|
||||
|
||||
elif source.source_type == SourceType.RSS_FEED:
|
||||
# --------------------------------------------------
|
||||
# 分支 B:长文章/传统订阅逻辑 -> 写入 NewsArticle 表
|
||||
# --------------------------------------------------
|
||||
existing_article = db.query(NewsArticle).filter(
|
||||
NewsArticle.source_id == source.id,
|
||||
NewsArticle.external_id == external_id
|
||||
).first()
|
||||
|
||||
if existing_article:
|
||||
# 文章如果已存在,通常只需要更新基础字段(文章一般不涉及排名起伏)
|
||||
existing_article.article_title = title
|
||||
existing_article.article_url = item_url
|
||||
# 预留位置:如果以后接口返回了摘要,可以在这里 update existing_article.original_summary
|
||||
else:
|
||||
# 全新文章入库
|
||||
new_article = NewsArticle(
|
||||
source_id=source.id,
|
||||
external_id=external_id,
|
||||
article_title=title,
|
||||
article_url=item_url,
|
||||
# original_summary=item.get("desc", ""),
|
||||
# author_name=item.get("author", "")
|
||||
)
|
||||
db.add(new_article)
|
||||
|
||||
|
||||
|
||||
saved_count += 1
|
||||
|
||||
# --------------------------------------------------
|
||||
# 5. 业务事务成功提交
|
||||
# --------------------------------------------------
|
||||
# 只有当前平台(source)的所有 item 都顺畅走完,才标记成功
|
||||
# 业务事务成功提交
|
||||
task_log.items_fetched = saved_count
|
||||
task_log.task_status = TaskStatus.SUCCESS
|
||||
db.add(task_log)
|
||||
|
||||
# 统一将当前信息源爬取的所有业务数据持久化到硬盘
|
||||
db.commit()
|
||||
print(f"[{source.source_name}] ({source.source_type}) 成功抓取并更新了 {saved_count} 条数据")
|
||||
|
||||
except Exception as e:
|
||||
# --------------------------------------------------
|
||||
# 6. 异常拦截与错误隔离机制
|
||||
# --------------------------------------------------
|
||||
# 回滚本次抓取的全部脏数据,
|
||||
# 异常拦截与错误隔离
|
||||
db.rollback()
|
||||
|
||||
# 错误日志记下来
|
||||
task_log.task_status = TaskStatus.ERROR
|
||||
task_log.error_trace = str(e)
|
||||
db.add(task_log)
|
||||
|
||||
|
||||
db.commit()
|
||||
print(f"[{source.source_name}] 抓取失败: {e}")
|
||||
print(f"[{source.source_name}] 抓取失败: {e}")
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# app/services/summary_service.py
|
||||
import os
|
||||
import json
|
||||
from datetime import timedelta
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from app.database import SessionLocal
|
||||
from app.models.models import UnifiedEvent, TrendingEvent, InfoSource, utcnow
|
||||
from app.prompts.summary_prompts import (
|
||||
SUMMARY_SYSTEM_PROMPT,
|
||||
SUMMARY_USER_PROMPT_TEMPLATE,
|
||||
)
|
||||
|
||||
HOT_SCORE_THRESHOLD = int(os.getenv("HOT_SCORE_THRESHOLD", 3))
|
||||
AI_API_KEY = os.getenv("AI_API_KEY", '')
|
||||
|
||||
# 1. 初始化异步客户端 (全局复用)
|
||||
deepseek_client = AsyncOpenAI(
|
||||
api_key=AI_API_KEY,
|
||||
base_url="https://api.deepseek.com"
|
||||
)
|
||||
|
||||
|
||||
async def call_llm_for_summary(platform_data_text: str) -> dict:
|
||||
"""调用 DeepSeek 生成统一标题和多平台视角摘要"""
|
||||
prompt = SUMMARY_USER_PROMPT_TEMPLATE.format(
|
||||
platform_data_text=platform_data_text
|
||||
)
|
||||
|
||||
# await
|
||||
response = await deepseek_client.chat.completions.create(
|
||||
model="deepseek-chat",
|
||||
messages=[
|
||||
{"role": "system", "content": SUMMARY_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
response_format={"type": "json_object"},
|
||||
temperature=1
|
||||
)
|
||||
|
||||
result_text = response.choices[0].message.content
|
||||
return json.loads(result_text)
|
||||
|
||||
|
||||
async def generate_unified_summaries():
|
||||
"""定时任务:扫描高热度事件并生成/更新摘要"""
|
||||
print(f"[{utcnow()}] 开始执行 DeepSeek 摘要生成任务...")
|
||||
|
||||
with SessionLocal() as db:
|
||||
recent_threshold = utcnow() - timedelta(days=3)
|
||||
|
||||
# 必须满足:热度达标 AND (当前热度 > 上次生成摘要时的热度) AND 近期活跃
|
||||
events = db.query(UnifiedEvent).filter(
|
||||
UnifiedEvent.hot_score >= HOT_SCORE_THRESHOLD,
|
||||
UnifiedEvent.hot_score > UnifiedEvent.last_summarized_trends_count,
|
||||
UnifiedEvent.created_at >= recent_threshold
|
||||
).all()
|
||||
|
||||
if not events:
|
||||
print("当前没有需要更新摘要的大事件,任务结束。")
|
||||
return
|
||||
|
||||
for event in events:
|
||||
# 联合查询获取该事件在各平台的子新闻
|
||||
trends = db.query(TrendingEvent, InfoSource.source_name) \
|
||||
.join(InfoSource, TrendingEvent.source_id == InfoSource.id) \
|
||||
.filter(TrendingEvent.unified_event_id == event.id) \
|
||||
.all()
|
||||
|
||||
if not trends:
|
||||
continue
|
||||
|
||||
# 按平台归类标题并去重
|
||||
platform_dict = {}
|
||||
for trend_record, source_name in trends:
|
||||
if source_name not in platform_dict:
|
||||
platform_dict[source_name] = set()
|
||||
platform_dict[source_name].add(trend_record.current_headline)
|
||||
|
||||
# 组装给大模型的 Prompt 数据
|
||||
prompt_lines = [f"【{platform}】: {', '.join(headlines)}" for platform, headlines in platform_dict.items()]
|
||||
platform_data_text = "\n".join(prompt_lines)
|
||||
|
||||
try:
|
||||
# 调用封装好的异步函数
|
||||
llm_result = await call_llm_for_summary(platform_data_text)
|
||||
|
||||
if "unified_title" in llm_result:
|
||||
event.unified_title = llm_result["unified_title"]
|
||||
if "ai_comprehensive_summary" in llm_result:
|
||||
event.ai_comprehensive_summary = llm_result["ai_comprehensive_summary"]
|
||||
|
||||
# 成功后更新水位线
|
||||
# 将最后一次总结时的热搜数量,更新为当前最新的 hot_score
|
||||
event.last_summarized_trends_count = event.hot_score
|
||||
|
||||
print(f"成功更新大事件 ID {event.id} 的深度摘要 (当前热度: {event.hot_score})。")
|
||||
|
||||
except Exception as e:
|
||||
print(f"大事件 ID {event.id} 摘要生成失败: {e}")
|
||||
continue
|
||||
|
||||
# 提交事务
|
||||
db.commit()
|
||||
@@ -0,0 +1,49 @@
|
||||
# app/utils/email_utils.py
|
||||
import os
|
||||
from email.message import EmailMessage
|
||||
import aiosmtplib
|
||||
from dotenv import load_dotenv
|
||||
import asyncio
|
||||
|
||||
load_dotenv()
|
||||
|
||||
SMTP_HOST = os.getenv("SMTP_HOST", "smtp.qiye.aliyun.com")
|
||||
SMTP_PORT = int(os.getenv("SMTP_PORT", 465))
|
||||
SMTP_USER = os.getenv("SMTP_USER", "noreply@yourdomain.com")
|
||||
SMTP_PASS = os.getenv("SMTP_PASS", "your_password")
|
||||
|
||||
|
||||
async def send_html_email(
|
||||
to_email: str,
|
||||
subject: str,
|
||||
html_content: str,
|
||||
sender_name: str = "AI 新闻早报",
|
||||
sender_email: str = None
|
||||
) -> bool:
|
||||
"""底层纯异步发送邮件工具"""
|
||||
# 如果未指定发送者邮箱,默认使用环境配置中的认证邮箱
|
||||
if sender_email is None:
|
||||
sender_email = SMTP_USER
|
||||
|
||||
message = EmailMessage()
|
||||
# 动态拼接 From 字段
|
||||
message["From"] = f"{sender_name} <{sender_email}>"
|
||||
message["To"] = to_email
|
||||
message["Subject"] = subject
|
||||
|
||||
# 设定内容为 HTML
|
||||
message.set_content(html_content, subtype="html")
|
||||
|
||||
try:
|
||||
await aiosmtplib.send(
|
||||
message,
|
||||
hostname=SMTP_HOST,
|
||||
port=SMTP_PORT,
|
||||
username=SMTP_USER,
|
||||
password=SMTP_PASS,
|
||||
use_tls=True,
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"邮件发送至 {to_email} 失败: {str(e)}")
|
||||
return False
|
||||
Reference in New Issue
Block a user