✨ 存储历史IO Rank数据至本地

nonebot_plugin_tetris_stats/config/migrations/0d50142b780f_add_field.py

@@ -0,0 +1,43 @@
+"""add field
+
+迁移 ID: 0d50142b780f
+父迁移: 09d4bb60160d
+创建时间: 2024-04-24 14:55:08.064098
+
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = '0d50142b780f'
+down_revision: str | Sequence[str] | None = '09d4bb60160d'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade(name: str = '') -> None:
+    if name:
+        return
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('nonebot_plugin_tetris_stats_iorank', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('file_hash', sa.String(length=128), nullable=True))
+        batch_op.create_index(
+            batch_op.f('ix_nonebot_plugin_tetris_stats_iorank_file_hash'), ['file_hash'], unique=False
+        )
+
+    # ### end Alembic commands ###
+
+
+def downgrade(name: str = '') -> None:
+    if name:
+        return
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('nonebot_plugin_tetris_stats_iorank', schema=None) as batch_op:
+        batch_op.drop_index(batch_op.f('ix_nonebot_plugin_tetris_stats_iorank_file_hash'))
+        batch_op.drop_column('file_hash')
+
+    # ### end Alembic commands ###

nonebot_plugin_tetris_stats/game_data_processor/io_data_processor/model.py

@@ -27,3 +27,4 @@ class IORank(MappedAsDataclass, Model):
         DateTime,
         index=True,
     )
+    file_hash: Mapped[str | None] = mapped_column(String(128), index=True)

nonebot_plugin_tetris_stats/game_data_processor/io_data_processor/processor.py

@@ -1,16 +1,21 @@
 from collections import defaultdict
 from collections.abc import Callable
 from datetime import datetime, timedelta, timezone
+from hashlib import sha512
 from math import floor
 from re import match
 from statistics import mean
 from typing import Literal
 
+from aiofiles import open
 from nonebot import get_driver
 from nonebot.compat import type_validate_json
+from nonebot.utils import run_sync
 from nonebot_plugin_apscheduler import scheduler  # type: ignore[import-untyped]
+from nonebot_plugin_localstore import get_data_file
 from nonebot_plugin_orm import get_session
 from sqlalchemy import select
+from zstandard import ZstdCompressor
 
 from ...db import create_or_update_bind
 from ...utils.exception import MessageFormatError, RequestError, WhatTheFuckError
@@ -160,7 +165,7 @@ class Processor(ProcessorMeta):
 async def get_io_rank_data() -> None:
     league_all: LeagueAll = type_validate_json(
         LeagueAll,  # type: ignore[arg-type]
-        await Cache.get(splice_url([BASE_URL, 'users/lists/league/all'])),
+        (data := await Cache.get(splice_url([BASE_URL, 'users/lists/league/all']))),
     )
     if isinstance(league_all, LeagueAllFailed):
         raise RequestError(f'排行榜数据请求错误:\n{league_all.error}')
@@ -188,6 +193,13 @@ async def get_io_rank_data() -> None:
         user = sort(users, field)
         return User(ID=user.id, name=user.username).dict(), field(user)
 
+    data_hash: str | None = await run_sync((await run_sync(sha512)(data)).hexdigest)()
+    try:
+        async with open(get_data_file('nonebot_plugin_tetris_stats', f'{data_hash}.json.zst'), mode='rb') as file:
+            await file.write(await run_sync(ZstdCompressor(level=12, threads=-1).compress)(data))
+    except:  # noqa: E722 FIXME: 确定错误类型
+        data_hash = None
+
     users = [i for i in league_all.data.users if isinstance(i, LeagueAllUser)]
     rank_to_users: defaultdict[Rank, list[LeagueAllUser]] = defaultdict(list)
     for i in users:
@@ -212,6 +224,7 @@ async def get_io_rank_data() -> None:
                 high_apm=(build_extremes_data(rank_users, apm, _max)),
                 high_vs=(build_extremes_data(rank_users, vs, _max)),
                 update_time=league_all.cache.cached_at,
+                file_hash=data_hash,
             )
         )
     async with get_session() as session: