使用随机/特殊 UA (#526)

*  添加依赖 fake-useragent

*  使用随机/特殊 UA
This commit is contained in:
呵呵です
2024-12-19 05:24:48 +08:00
committed by GitHub
parent cfcda6f597
commit f56f993c69
4 changed files with 38 additions and 10 deletions

View File

@@ -1,10 +1,12 @@
from typing import Literal, overload
from uuid import UUID
from nonebot import __version__ as __nonebot_version__
from nonebot.compat import type_validate_json
from yarl import URL
from ....utils.exception import RequestError
from ....version import __version__
from ..constant import BASE_URL
from .cache import Cache
from .schemas.base import FailedModel
@@ -22,7 +24,12 @@ async def by(
await get(
BASE_URL / f'users/by/{by_type}',
parameter,
{'X-Session-ID': str(x_session_id)} if x_session_id is not None else None,
{
'X-Session-ID': str(x_session_id),
'User-Agent': f'nonebot-plugin-tetris-stats/{__version__} (Windows NT 10.0; Win64; x64) NoneBot2/{__nonebot_version__}',
}
if x_session_id is not None
else None,
),
)
if isinstance(model, FailedModel):

View File

@@ -2,6 +2,7 @@ from collections.abc import Sequence
from http import HTTPStatus
from typing import Any
from fake_useragent import UserAgent
from httpx import AsyncClient, HTTPError
from msgspec import DecodeError, Struct, json
from nonebot import get_driver
@@ -113,6 +114,8 @@ class Request:
def __init__(self, proxy: str | None) -> None:
self.proxy = proxy
self.anti_cloudflares: dict[str, AntiCloudflare] = {}
self.client = AsyncClient(timeout=config.tetris.request_timeout, proxy=self.proxy)
self.ua = UserAgent()
async def request(
self,
@@ -129,16 +132,20 @@ class Request:
else:
cookies = None
headers = None
headers = headers if extra_headers is None else extra_headers if headers is None else headers | extra_headers
if headers is None:
headers = {}
if extra_headers:
headers.update(extra_headers)
headers.setdefault('User-Agent', self.ua.random)
try:
async with AsyncClient(cookies=cookies, timeout=config.tetris.request_timeout, proxy=self.proxy) as session:
response = await session.get(str(url), headers=headers)
if response.status_code != HTTPStatus.OK:
msg = f'请求错误 code: {response.status_code} {HTTPStatus(response.status_code).phrase}\n{response.text}'
raise RequestError(msg, status_code=response.status_code)
if is_json:
decoder.decode(response.content)
return response.content
response = await self.client.get(str(url), cookies=cookies, headers=headers)
if response.status_code != HTTPStatus.OK:
msg = (
f'请求错误 code: {response.status_code} {HTTPStatus(response.status_code).phrase}\n{response.text}'
)
raise RequestError(msg, status_code=response.status_code)
if is_json:
decoder.decode(response.content)
except HTTPError as e:
msg = f'请求错误 \n{e!r}'
raise RequestError(msg) from e
@@ -146,6 +153,8 @@ class Request:
if enable_anti_cloudflare and url.host is not None:
return await self.anti_cloudflares.setdefault(url.host, AntiCloudflare(url.host))(str(url), self.proxy)
raise
else:
return response.content
async def failover_request(
self,

View File

@@ -10,6 +10,7 @@ dependencies = [
"aiofiles>=24.1.0",
"arclet-alconna<2",
"async-lru>=2.0.4",
"fake-useragent>=2.0.3",
"httpx>=0.27.2",
"jinja2>=3.1.4",
"lxml>=5.3.0",

11
uv.lock generated
View File

@@ -756,6 +756,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b7/cd/ff116c050330c4b8c8ebc657f40501460c3e38905fb15993ef21f7d24208/expiringdictx-1.1.0-py3-none-any.whl", hash = "sha256:f5d38ae23b46a8f97da27ce0dd74e669430d43b2efe98232a4a81f10b43cde25", size = 7596 },
]
[[package]]
name = "fake-useragent"
version = "2.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/d6/a4/f8d204c20e7879c2c1fd1719095673f447a3111282bfe09c0a74a5ed5000/fake_useragent-2.0.3.tar.gz", hash = "sha256:af86a26ef8229efece8fed529b4aeb5b73747d889b60f01cd477b6f301df46e6", size = 194741 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/11/4f/a639b1dbdc557241e702eefb931ba24ba235c84f8fffdca3e272f096c6af/fake_useragent-2.0.3-py3-none-any.whl", hash = "sha256:8bae50abb72c309a5b3ae2f01a0b82426613fd5c4e2a04dca9332399ec44daa1", size = 201110 },
]
[[package]]
name = "fastapi"
version = "0.115.3"
@@ -1825,6 +1834,7 @@ dependencies = [
{ name = "aiofiles" },
{ name = "arclet-alconna" },
{ name = "async-lru" },
{ name = "fake-useragent" },
{ name = "httpx" },
{ name = "jinja2" },
{ name = "lxml" },
@@ -1887,6 +1897,7 @@ requires-dist = [
{ name = "aiofiles", specifier = ">=24.1.0" },
{ name = "arclet-alconna", specifier = "<2" },
{ name = "async-lru", specifier = ">=2.0.4" },
{ name = "fake-useragent", specifier = ">=2.0.3" },
{ name = "httpx", specifier = ">=0.27.2" },
{ name = "jinja2", specifier = ">=3.1.4" },
{ name = "lxml", specifier = ">=5.3.0" },