🔖 1.2.7

nonebot_plugin_tetris_stats/db/__init__.py

@@ -1,3 +1,4 @@
+from asyncio import Lock
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
@@ -72,9 +73,11 @@ async def create_or_update_bind(
 
 T = TypeVar('T', 'TETRIOHistoricalData', 'TOPHistoricalData', 'TOSHistoricalData')
 
+lock = Lock()
+
 
 async def anti_duplicate_add(cls: type[T], model: T) -> None:
-    async with get_session() as session:
+    async with lock, get_session() as session:
         result = (
             await session.scalars(
                 select(cls)
@@ -88,8 +91,8 @@ async def anti_duplicate_add(cls: type[T], model: T) -> None:
                 if i.data == model.data:
                     logger.debug('Anti duplicate successfully')
                     return
-            session.add(model)
-            await session.commit()
+        session.add(model)
+        await session.commit()
 
 
 @asynccontextmanager
@@ -121,12 +124,10 @@ async def trigger(
     command_type: CommandType | Literal['rank'],
     command_args: list[str],
 ) -> AsyncGenerator:
-    logger.debug('running')
     trigger_time = datetime.now(UTC)
     try:
         yield
     except FinishedException:
-        logger.debug('yield')
         async with get_session() as session:
             session.add(
                 TriggerHistoricalData(

nonebot_plugin_tetris_stats/db/models.py

@@ -23,9 +23,8 @@ class PydanticType(TypeDecorator):
         *args: Any,
         **kwargs: Any,
     ):
-        for i in get_model:
-            models.update(i())
-        self.models = models
+        self.get_model = get_model
+        self._models = models
         super().__init__(*args, **kwargs)
 
     if PYDANTIC_V2:
@@ -33,7 +32,7 @@ class PydanticType(TypeDecorator):
         @override
         def process_bind_param(self, value: Any | None, dialect: Dialect) -> str:
             # 将 Pydantic 模型实例转换为 JSON
-            if isinstance(value, tuple(self.pydantic_models)):
+            if isinstance(value, tuple(self.models)):
                 return value.model_dump_json(by_alias=True)  # type: ignore[union-attr]
             raise TypeError
     else:
@@ -41,7 +40,7 @@ class PydanticType(TypeDecorator):
         @override
         def process_bind_param(self, value: Any | None, dialect: Dialect) -> str:
             # 将 Pydantic 模型实例转换为 JSON
-            if isinstance(value, tuple(self.pydantic_models)):
+            if isinstance(value, tuple(self.models)):
                 return value.json(by_alias=True)  # type: ignore[union-attr]
             raise TypeError
 
@@ -49,13 +48,21 @@ class PydanticType(TypeDecorator):
     def process_result_value(self, value: Any | None, dialect: Dialect) -> BaseModel:
         # 将 JSON 转换回 Pydantic 模型实例
         if isinstance(value, str | bytes):
-            for i in self.pydantic_models:
+            for i in self.models:
                 try:
                     return type_validate_json(i, value)
                 except ValidationError:  # noqa: PERF203
                     ...
         raise ValueError
 
+    @property
+    def models(self) -> tuple[type[BaseModel], ...]:
+        models: set[type[BaseModel]] = set()
+        for i in self.get_model:
+            models.update(i())
+        models.update(self._models)
+        return tuple(models)
+
 
 class Bind(MappedAsDataclass, Model):
     id: Mapped[int] = mapped_column(init=False, primary_key=True)

nonebot_plugin_tetris_stats/game_data_processor/io_data_processor/api/player.py

@@ -43,7 +43,7 @@ class Player:
         if self.user_id is not None:
             return self.user_id
         if self.user_name is not None:
-            return self.user_name
+            return self.user_name.lower()
         msg = 'Invalid user'
         raise ValueError(msg)
 

nonebot_plugin_tetris_stats/game_data_processor/io_data_processor/api/schemas/user_records.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 from pydantic import BaseModel, Field
 
+from .....utils.typing import Number
 from .base import FailedModel
 from .base import SuccessModel as BaseSuccessModel
 
@@ -45,7 +46,7 @@ class Finesse(BaseModel):
 
 
 class EndContext(BaseModel):
-    seed: int
+    seed: Number
     lines: int
     level_lines: int
     level_lines_needed: int

nonebot_plugin_tetris_stats/game_data_processor/io_data_processor/query.py

@@ -1,19 +1,27 @@
 import contextlib
 from asyncio import gather
-from datetime import datetime, timedelta, timezone
+from collections import defaultdict
+from datetime import date, datetime, timedelta, timezone
 from hashlib import md5
 from math import ceil, floor
+from typing import ClassVar
 from urllib.parse import urlunparse
 from zoneinfo import ZoneInfo
 
+from aiofiles import open
+from nonebot import get_driver
 from nonebot.adapters import Bot, Event
+from nonebot.compat import type_validate_json
 from nonebot.matcher import Matcher
 from nonebot_plugin_alconna import At
 from nonebot_plugin_alconna.uniseg import UniMessage
+from nonebot_plugin_apscheduler import scheduler  # type: ignore[import-untyped]
+from nonebot_plugin_localstore import get_data_file  # type: ignore[import-untyped]
 from nonebot_plugin_orm import get_session
 from nonebot_plugin_session import EventSession  # type: ignore[import-untyped]
 from nonebot_plugin_session_orm import get_session_persist_id  # type: ignore[import-untyped]
 from sqlalchemy import select
+from zstandard import ZstdDecompressor
 
 from ...db import query_bind_info, trigger
 from ...utils.host import HostPage, get_self_netloc
@@ -23,17 +31,21 @@ from ...utils.render.schemas.base import Avatar
 from ...utils.render.schemas.tetrio_info import Data, Radar, Ranking, TetraLeague, TetraLeagueHistory
 from ...utils.render.schemas.tetrio_info import User as TemplateUser
 from ...utils.screenshot import screenshot
-from ...utils.typing import Me
+from ...utils.typing import Me, Number
 from ..constant import CANT_VERIFY_MESSAGE
 from . import alc
 from .api import Player, User, UserInfoSuccess
 from .api.models import TETRIOHistoricalData
+from .api.schemas.tetra_league import TetraLeagueSuccess
 from .api.schemas.user_info import NeverPlayedLeague, NeverRatedLeague, RatedLeague
 from .api.schemas.user_records import SoloModeRecord, SoloRecord
 from .constant import GAME_TYPE, TR_MAX, TR_MIN
+from .model import IORank
 
 UTC = timezone.utc
 
+driver = get_driver()
+
 
 @alc.assign('query')
 async def _(bot: Bot, event: Event, matcher: Matcher, target: At | Me, event_session: EventSession):
@@ -57,9 +69,9 @@ async def _(bot: Bot, event: Event, matcher: Matcher, target: At | Me, event_ses
         user, user_info, user_records = await gather(player.user, player.get_info(), player.get_records())
         sprint = user_records.data.records.sprint
         blitz = user_records.data.records.blitz
-        with contextlib.suppress(TypeError):
-            message += UniMessage.image(raw=await make_query_image(user, user_info, sprint.record, blitz.record))
-            await message.finish()
+        # with contextlib.suppress(TypeError):
+        message += UniMessage.image(raw=await make_query_image(user, user_info, sprint.record, blitz.record))
+        await message.finish()
         message += make_query_text(user_info, sprint, blitz)
         await message.finish()
 
@@ -156,7 +168,8 @@ async def query_historical_data(user: User, user_info: UserInfoSuccess) -> list[
             if extra is not None:
                 historical_data = list(historical_data)
                 historical_data.append(extra)
-    if not historical_data:
+    full_export_data = FullExport.get_data(user.unique_identifier)
+    if not historical_data and not full_export_data:
         return [
             Data(record_at=today - forward, tr=user_info.data.user.league.rating),
             Data(record_at=today.replace(microsecond=1000), tr=user_info.data.user.league.rating),
@@ -168,13 +181,13 @@ async def query_historical_data(user: User, user_info: UserInfoSuccess) -> list[
         )
         for i in historical_data
         if isinstance(i.data, UserInfoSuccess) and isinstance(i.data.data.user.league, RatedLeague)
-    ]
+    ] + full_export_data
 
     # 按照时间排序
     histories = sorted(histories, key=lambda x: x.record_at)
     for index, value in enumerate(histories):
-        # 在历史记录里找有没有今天0点后的数据
-        if value.record_at > today:
+        # 在历史记录里找有没有今天0点后的数据, 并且至少要有两个数据点
+        if value.record_at > today and len(histories) >= 2:  # noqa: PLR2004
             histories = histories[:index] + [
                 get_specified_point(histories[index - 1], histories[index], today.replace(microsecond=1000))
             ]
@@ -294,3 +307,82 @@ def make_query_text(user_info: UserInfoSuccess, sprint: SoloModeRecord, blitz: S
         message += f'\nBlitz: {blitz.record.endcontext.score}'
         message += f' ( #{blitz.rank} )' if blitz.rank is not None else ''
     return UniMessage(message)
+
+
+class FullExport:
+    cache: ClassVar[defaultdict[str, set[tuple[datetime, Number]]]] = defaultdict(set)
+    latest_update: ClassVar[date | None] = None
+
+    @classmethod
+    async def init(cls) -> None:
+        async with get_session() as session:
+            full_exports = (await session.scalars(select(IORank).where(IORank.update_time >= cls.start_time()))).all()
+        await gather(
+            *[
+                cls._load(update_time, file_hash)
+                for file_hash, update_time in {
+                    i.file_hash: i.update_time for i in full_exports if i.file_hash is not None
+                }.items()
+            ]
+        )
+
+    @classmethod
+    async def update(cls) -> None:
+        if cls.latest_update == datetime.now(tz=ZoneInfo('Asia/Shanghai')).date():
+            return
+        start_time = cls.start_time()
+        for i in cls.cache:
+            cls.cache[i] = {j for j in cls.cache[i] if j[0] >= start_time}
+        latest_time = max(cls.cache)
+        async with get_session() as session:
+            full_exports = (await session.scalars(select(IORank).where(IORank.update_time > latest_time))).all()
+        await gather(
+            *[
+                cls._load(update_time, file_hash)
+                for file_hash, update_time in {
+                    i.file_hash: i.update_time for i in full_exports if i.file_hash is not None
+                }.items()
+            ]
+        )
+        cls.latest_update = datetime.now(tz=ZoneInfo('Asia/Shanghai')).date()
+
+    @classmethod
+    def get_data(cls, unique_identifier: str) -> list[Data]:
+        return [Data(record_at=i[0], tr=i[1]) for i in cls.cache[unique_identifier]]
+
+    @classmethod
+    def start_time(cls) -> datetime:
+        return (
+            datetime.now(ZoneInfo('Asia/Shanghai')).replace(hour=0, minute=0, second=0, microsecond=0)
+            - timedelta(days=9)
+        ).astimezone(UTC)
+
+    @classmethod
+    async def _load(cls, update_time: datetime, file_hash: str) -> None:
+        try:
+            users = type_validate_json(TetraLeagueSuccess, await cls.decompress(file_hash)).data.users
+        except FileNotFoundError:
+            await cls.clear_invalid(file_hash)
+            return
+        update_time = update_time.astimezone(ZoneInfo('Asia/Shanghai'))
+        for i in users:
+            cls.cache[i.id].add((update_time, i.league.rating))
+
+    @classmethod
+    async def decompress(cls, file_hash: str) -> bytes:
+        async with open(get_data_file('nonebot_plugin_tetris_stats', f'{file_hash}.json.zst'), mode='rb') as file:
+            return ZstdDecompressor().decompress(await file.read())
+
+    @classmethod
+    async def clear_invalid(cls, file_hash: str) -> None:
+        async with get_session() as session:
+            full_exports = (await session.scalars(select(IORank).where(IORank.file_hash == file_hash))).all()
+            for i in full_exports:
+                i.file_hash = None
+            await session.commit()
+
+
+@driver.on_startup
+async def _():
+    await FullExport.init()
+    scheduler.add_job(FullExport.update, 'interval', hours=1)

poetry.lock

@@ -860,6 +860,20 @@ files = [
     {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"},
 ]
 
+[[package]]
+name = "memory-profiler"
+version = "0.61.0"
+description = "A module for monitoring memory usage of a python program"
+optional = false
+python-versions = ">=3.5"
+files = [
+    {file = "memory_profiler-0.61.0-py3-none-any.whl", hash = "sha256:400348e61031e3942ad4d4109d18753b2fb08c2f6fb8290671c5513a34182d84"},
+    {file = "memory_profiler-0.61.0.tar.gz", hash = "sha256:4e5b73d7864a1d1292fb76a03e82a3e78ef934d06828a698d9dada76da2067b0"},
+]
+
+[package.dependencies]
+psutil = "*"
+
 [[package]]
 name = "msgpack"
 version = "1.0.8"
@@ -1401,7 +1415,6 @@ optional = false
 python-versions = ">=3.9"
 files = [
     {file = "pandas-2.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:90c6fca2acf139569e74e8781709dccb6fe25940488755716d1d354d6bc58bce"},
-    {file = "pandas-2.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c7adfc142dac335d8c1e0dcbd37eb8617eac386596eb9e1a1b77791cf2498238"},
     {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4abfe0be0d7221be4f12552995e58723c7422c80a659da13ca382697de830c08"},
     {file = "pandas-2.2.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8635c16bf3d99040fdf3ca3db669a7250ddf49c55dc4aa8fe0ae0fa8d6dcc1f0"},
     {file = "pandas-2.2.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:40ae1dffb3967a52203105a077415a86044a2bea011b5f321c6aa64b379a3f51"},
@@ -1422,7 +1435,6 @@ files = [
     {file = "pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32"},
     {file = "pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23"},
     {file = "pandas-2.2.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0ca6377b8fca51815f382bd0b697a0814c8bda55115678cbc94c30aacbb6eff2"},
-    {file = "pandas-2.2.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9057e6aa78a584bc93a13f0a9bf7e753a5e9770a30b4d758b8d5f2a62a9433cd"},
     {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:001910ad31abc7bf06f49dcc903755d2f7f3a9186c0c040b827e522e9cef0863"},
     {file = "pandas-2.2.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:66b479b0bd07204e37583c191535505410daa8df638fd8e75ae1b383851fe921"},
     {file = "pandas-2.2.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a77e9d1c386196879aa5eb712e77461aaee433e54c68cf253053a73b7e49c33a"},
@@ -1587,6 +1599,34 @@ files = [
 greenlet = "3.0.3"
 pyee = "11.1.0"
 
+[[package]]
+name = "psutil"
+version = "5.9.8"
+description = "Cross-platform lib for process and system monitoring in Python."
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*"
+files = [
+    {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"},
+    {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"},
+    {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7"},
+    {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36"},
+    {file = "psutil-5.9.8-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d"},
+    {file = "psutil-5.9.8-cp27-none-win32.whl", hash = "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e"},
+    {file = "psutil-5.9.8-cp27-none-win_amd64.whl", hash = "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631"},
+    {file = "psutil-5.9.8-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81"},
+    {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421"},
+    {file = "psutil-5.9.8-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4"},
+    {file = "psutil-5.9.8-cp36-cp36m-win32.whl", hash = "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee"},
+    {file = "psutil-5.9.8-cp36-cp36m-win_amd64.whl", hash = "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2"},
+    {file = "psutil-5.9.8-cp37-abi3-win32.whl", hash = "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0"},
+    {file = "psutil-5.9.8-cp37-abi3-win_amd64.whl", hash = "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf"},
+    {file = "psutil-5.9.8-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"},
+    {file = "psutil-5.9.8.tar.gz", hash = "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c"},
+]
+
+[package.extras]
+test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]
+
 [[package]]
 name = "pycparser"
 version = "2.22"
@@ -2790,4 +2830,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "c4b405273e804e8259016c4a8fee0ed26650226cfaec0da7ac485e29f704c68c"
+content-hash = "530776363d325178abc516c410ee72f7efe99885a8510b4ebc989705284a67fc"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = 'nonebot-plugin-tetris-stats'
-version = '1.2.0'
+version = '1.2.7'
 description = '一款基于 NoneBot2 的用于查询 Tetris 相关游戏数据的插件'
 authors = ['scdhh <wallfjjd@gmail.com>']
 readme = 'README.md'
@@ -46,6 +46,7 @@ nonebot-adapter-qq = "^1.4.4"
 nonebot-adapter-satori = "^0.11.4"
 
 [tool.poetry.group.debug.dependencies]
+memory-profiler = "^0.61.0"
 objprint = '^0.2.2'
 viztracer = "^0.16.2"