hygc/hts-code-query-system-backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: 提高sql和es查询速度

Browse Source
This commit is contained in:
皓月归尘 2025-03-14 18:14:36 +08:00
parent a54693a7bf
commit d921a2e7ba
12 changed files with 697 additions and 453 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

977
api/code.py
View File

File diff suppressed because it is too large Load Diff

2
api/config.py
View File

@ -125,7 +125,7 @@ async def get_config_list(request: Request,
type: Optional[str] = Query(default=None, description="系统内置"),
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'name': name,
'key': key,
'type': type,

2
api/department.py
View File

@ -171,7 +171,7 @@ async def get_department_list(
current_user: dict = Depends(LoginController.get_current_user)
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'name': name,
'principal': principal,
'phone': phone,

19
api/file.py
View File

@ -7,7 +7,6 @@
# @Comment : 本程序
import os
from datetime import datetime
from fastapi import APIRouter, UploadFile, File, Path, Depends, Request, Query
from fastapi.responses import FileResponse, JSONResponse
@ -40,7 +39,7 @@ async def upload_file(
raise ModelValidatorException(message="文件类型不支持")
# 2. 生成唯一的文件名
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
timestamp = datetime.now().strftime("%Y%m%d%H%M%S%f")
unique_filename = f"{current_user.get('id')}_{timestamp}.{file_extension}"
# 3. 保存文件到服务器
@ -61,7 +60,7 @@ async def upload_file(
relative_path=relative_path,
uploader_id=current_user.get("id"),
)
result = await file_record.first().values(
result = await FileModel.get_or_none(id=file_record.id).values(
id="id",
name="name",
size="size",
@ -86,7 +85,7 @@ async def download_file(
id: str = Path(..., description="文件ID"),
):
# 1. 查询文件记录
file_record = await FileModel.get_or_none(id=id)
file_record = await FileModel.get_or_none(id=id, del_flag=1)
if not file_record:
raise ServiceException(message="文件不存在!")
@ -110,10 +109,10 @@ async def get_file_info(
current_user: dict = Depends(LoginController.get_current_user),
):
# 1. 查询文件记录
file_record = await FileModel.get_or_none(id=id)
file_record = await FileModel.get_or_none(id=id, del_flag=1)
if not file_record:
raise ServiceException(message="文件不存在!")
result = await file_record.first().values(
result = await FileModel.get_or_none(id=id, del_flag=1).values(
id="id",
name="name",
size="size",
@ -139,11 +138,11 @@ async def delete_file(
id: str = Path(..., description="文件ID"),
current_user: dict = Depends(LoginController.get_current_user), ):
# 1. 查询文件记录
file_record = await FileModel.get_or_none(id=id)
file_record = await FileModel.get_or_none(id=id, del_flag=1)
if not file_record:
raise ServiceException(message="文件不存在!")
if await Upload.check_file_exists(file_record.absolute_path):
await Upload.delete_file(file_record.absolute_path)
if Upload.check_file_exists(file_record.absolute_path):
Upload.delete_file(file_record.absolute_path)
await file_record.delete()
return Response.success()
@ -164,7 +163,7 @@ async def get_file_list(
current_user: dict = Depends(LoginController.get_current_user), ):
# 1. 查询文件记录
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'name': name,
'file_type': file_type,
'uploader__id': uploader_id,

4
api/i18n.py
View File

@ -123,7 +123,7 @@ async def get_locale_list(request: Request,
code: Optional[str] = Query(default=None, description="国际化类型代码"),
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'name': name,
'code': code
}.items() if v
@ -242,7 +242,7 @@ async def get_i18n_list(request: Request,
translation: Optional[str] = Query(default=None, description="国际化内容翻译内容"),
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'key': key,
'locale_id': locale_id,
'translation': translation

4
api/log.py
View File

@ -44,7 +44,7 @@ async def get_login_log(request: Request,
user_id = current_user.get("id")
online_user_list = await LoginController.get_online_user(request, sub_departments)
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'username': username,
'nickname': nickname,
}.items() if v
@ -182,7 +182,7 @@ async def get_operation_log(request: Request,
sub_departments = current_user.get("sub_departments")
user_id = current_user.get("id")
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'operation_name': name,
'operation_type': type,
'operator__username': username,

2
api/permission.py
View File

@ -217,7 +217,7 @@ async def get_permission_list(
current_user: dict = Depends(LoginController.get_current_user),
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
"id": id,
"name": name,
"parent_id": parentId,

2
api/role.py
View File

@ -187,7 +187,7 @@ async def get_role_list(
current_user: dict = Depends(LoginController.get_current_user)
):
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
"name": name,
"code": code,
"description": description,

2
api/user.py
View File

@ -184,7 +184,7 @@ async def get_user_list(
):
sub_departments = current_user.get("sub_departments")
filterArgs = {
f'{k}__contains': v for k, v in {
f'{k}__icontains': v for k, v in {
'username': username,
'nickname': nickname,
'phone': phone,

130
config/database.py
View File

@ -71,68 +71,68 @@ async def configure_tortoise_logging(enable_logging: bool = True, log_level: int
if aiomysql_logger.hasHandlers():
aiomysql_logger.handlers.clear()
if enable_logging:
# 设置日志格式
fmt = logging.Formatter(
fmt="%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
# 创建控制台处理器（输出到控制台）
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(log_level)
console_handler.setFormatter(fmt)
# 创建文件处理器（输出到文件）
file_handler = RotatingFileHandler(
filename=log_path_sql,
maxBytes=50 * 1024 * 1024, # 日志文件大小达到 50MB 时轮换
backupCount=5, # 保留 5 个旧日志文件
encoding="utf-8",
)
file_handler.setLevel(log_level)
file_handler.setFormatter(fmt)
# 配置 tortoise 顶级日志记录器
tortoise_logger.setLevel(log_level)
tortoise_logger.addHandler(console_handler) # 添加控制台处理器
tortoise_logger.addHandler(file_handler) # 添加文件处理器
# 配置 aiomysql 日志记录器
aiomysql_logger.setLevel(log_level)
aiomysql_logger.addHandler(console_handler) # 添加控制台处理器
aiomysql_logger.addHandler(file_handler) # 添加文件处理器
# 配置 SQL 查询日志记录器
sql_logger = logging.getLogger("tortoise.db_client")
sql_logger.setLevel(log_level)
class SQLResultLogger(logging.Handler):
async def emit(self, record):
# 只处理 SQL 查询相关的日志
if "SELECT" in record.getMessage() or "INSERT" in record.getMessage() or "UPDATE" in record.getMessage() or "DELETE" in record.getMessage():
# 输出 SQL 查询语句
console_handler.emit(record)
file_handler.emit(record)
# 异步获取并记录查询结果
await self.log_query_result(record)
async def log_query_result(self, record):
"""
执行查询并返回结果。
"""
try:
from tortoise import Tortoise
connection = Tortoise.get_connection("default")
result = await connection.execute_query_dict(record.getMessage())
return result
except Exception as e:
return f"获取查询结果失败: {str(e)}"
# 添加自定义 SQL 查询日志处理器
sql_result_handler = SQLResultLogger()
sql_result_handler.setLevel(log_level)
sql_logger.addHandler(sql_result_handler)
else:
# 如果禁用日志，设置日志级别为 WARNING 以抑制大部分输出
tortoise_logger.setLevel(logging.WARNING)
# if enable_logging:
# # 设置日志格式
# fmt = logging.Formatter(
# fmt="%(asctime)s - %(name)s:%(lineno)d - %(levelname)s - %(message)s",
# datefmt="%Y-%m-%d %H:%M:%S",
# )
#
# # 创建控制台处理器（输出到控制台）
# console_handler = logging.StreamHandler(sys.stdout)
# console_handler.setLevel(log_level)
# console_handler.setFormatter(fmt)
#
# # 创建文件处理器（输出到文件）
# file_handler = RotatingFileHandler(
# filename=log_path_sql,
# maxBytes=50 * 1024 * 1024, # 日志文件大小达到 50MB 时轮换
# backupCount=5, # 保留 5 个旧日志文件
# encoding="utf-8",
# )
# file_handler.setLevel(log_level)
# file_handler.setFormatter(fmt)
#
# # 配置 tortoise 顶级日志记录器
# tortoise_logger.setLevel(log_level)
# tortoise_logger.addHandler(console_handler) # 添加控制台处理器
# tortoise_logger.addHandler(file_handler) # 添加文件处理器
#
# # 配置 aiomysql 日志记录器
# aiomysql_logger.setLevel(log_level)
# aiomysql_logger.addHandler(console_handler) # 添加控制台处理器
# aiomysql_logger.addHandler(file_handler) # 添加文件处理器
# # 配置 SQL 查询日志记录器
# sql_logger = logging.getLogger("tortoise.db_client")
# sql_logger.setLevel(log_level)
#
# class SQLResultLogger(logging.Handler):
# async def emit(self, record):
# # 只处理 SQL 查询相关的日志
# if "SELECT" in record.getMessage() or "INSERT" in record.getMessage() or "UPDATE" in record.getMessage() or "DELETE" in record.getMessage():
# # 输出 SQL 查询语句
# console_handler.emit(record)
# file_handler.emit(record)
#
# # 异步获取并记录查询结果
# await self.log_query_result(record)
#
# async def log_query_result(self, record):
# """
# 执行查询并返回结果。
# """
# try:
# from tortoise import Tortoise
# connection = Tortoise.get_connection("default")
# result = await connection.execute_query_dict(record.getMessage())
# return result
# except Exception as e:
# return f"获取查询结果失败: {str(e)}"
#
# # 添加自定义 SQL 查询日志处理器
# sql_result_handler = SQLResultLogger()
# sql_result_handler.setLevel(log_level)
# sql_logger.addHandler(sql_result_handler)
# else:
# # 如果禁用日志，设置日志级别为 WARNING 以抑制大部分输出
# tortoise_logger.setLevel(logging.WARNING)

1
config/env.py
View File

@ -313,6 +313,7 @@ class UploadSettings:
'doc',
'docx',
'xls',
'csv',
'xlsx',
'ppt',
'pptx',

5
models/code.py
View File

@ -143,6 +143,11 @@ class QueryCode(BaseModel):
"""
查询编码模型
"""
id=fields.UUIDField(
pk=True,
description="主键",
source_field="id"
)
session = fields.ForeignKeyField(
"models.QueryCodeLog",
related_name="query_code",