# _*_ coding : UTF-8 _*_ # @Time : 2025/02/13 19:20 # @UpdateTime : 2025/02/13 19:20 # @Author : sonder # @File : code.py # @Software : PyCharm # @Comment : 本程序 import json import os import re import time import uuid from datetime import datetime from typing import Optional import pandas as pd from elasticsearch.helpers import async_bulk from fastapi import APIRouter, Depends, Path, Request, Query from fastapi.responses import JSONResponse, FileResponse from tortoise.transactions import in_transaction from annotation.auth import Auth, hasAuth from annotation.log import Log from config.constant import BusinessType from config.env import ElasticSearchConfig from controller.login import LoginController from exceptions.exception import ServiceException, PermissionException from models import File, Code, QueryCode, QueryCodeLog, CodeFeedback, CodeImport, HtsItem from schemas.code import GetCodeInfoResponse, GetCodeListResponse, GetQueryCodeParams, QueryCodeResponse, AddCodeParams, \ GetQueryCodeLogResponse, GetQueryCodeLogDetailResponse, \ GetCodeLogAllResponse, AddCodeFeedbackParams, GetCodeFeedbackResponse, GetCodeFeedbackListResponse, \ UpdateCodeFeedbackStatusParams, GetCodeImportListResponse, UpdateCodeImportStatusParams from schemas.common import BaseResponse, DeleteListParams from utils.log import logger from utils.response import Response codeAPI = APIRouter( prefix="/code" ) @codeAPI.get("/template/{type}", summary="获取上传编码模板") @Log(title="获取上传编码模板", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:uploadTemplate"]) async def get_upload_template(request: Request, type: str = Path(description="文件类型"), current_user=Depends(LoginController.get_current_user)): if type not in ["xlsx", "xls", "csv"]: raise ServiceException(message="文件类型错误!") template_path = os.path.join(os.path.abspath(os.getcwd()), 'assets', 'templates', f'上传模版.{type}') media_type = { "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xls": "application/vnd.ms-excel", "csv": "text/csv" }.get(type) if not os.path.exists(template_path): raise ServiceException(message="文件不存在!") return FileResponse( path=template_path, filename=f"上传模版.{type}", media_type=media_type ) @codeAPI.get("/queryTemplate/{type}", summary="获取查询编码模板") @Log(title="获取查询编码模板", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:queryTemplate"]) async def get_query_template(request: Request, type: str = Path(description="文件类型"), current_user=Depends(LoginController.get_current_user)): if type not in ["xlsx", "xls", "csv"]: raise ServiceException(message="文件类型错误!") template_path = os.path.join(os.path.abspath(os.getcwd()), 'assets', 'templates', f'查询模版.{type}') if not os.path.exists(template_path): raise ServiceException(message="文件不存在!") media_type = { "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xls": "application/vnd.ms-excel", "csv": "text/csv" }.get(type) return FileResponse( path=template_path, filename=f"查询模版.{type}", media_type=media_type ) @codeAPI.post("/add", response_class=JSONResponse, response_model=BaseResponse, summary="添加编码") @Log(title="添加编码", business_type=BusinessType.INSERT) @Auth(permission_list=["code:btn:add"]) async def add_code(request: Request, params: AddCodeParams, current_user=Depends(LoginController.get_current_user)): params.code = params.code.replace(".", "").replace("/", "").replace("_", "").replace("-", "").replace("?", "").replace( ":", "").replace("：", "").replace("？", "").strip() if await CodeImport.get_or_none(code=params.code, del_flag=1): return Response.failure(msg="编码已存在") else: user_id = current_user.get("id") code_import = await CodeImport.create( code=params.code, description=params.description, status=3, user_id=user_id ) if code_import: return Response.success(msg="添加成功") else: return Response.failure(msg="添加失败") SPECIAL_CHARS_PATTERN = r"[.,\/_\-?:：？！!@#$%^&*()+=<>|{}[\]\\]" @codeAPI.post("/addCode", response_class=JSONResponse, response_model=BaseResponse, summary="导入编码") @Log(title="导入编码", business_type=BusinessType.INSERT) @Auth(permission_list=["code:btn:import"]) async def add_code_by_file( request: Request, params: DeleteListParams, # 这里的 params 传入的是 {"ids": [...]} current_user=Depends(LoginController.get_current_user) ): user_id = current_user.get("id") # 查询所有文件 files = await File.filter(id__in=set(params.ids), del_flag=1).values(id="id", uploader_id="uploader__id", file_type="file_type", absolute_path="absolute_path") if not files: return Response.failure(msg="文件不存在") # 确保用户对所有文件都有权限 unauthorized_files = [] for file in files: if str(file["uploader_id"]) != user_id: unauthorized_files.append(file['id']) if unauthorized_files: return Response.failure(msg=f"权限不足，文件ID: {unauthorized_files}") total_imported = 0 # 统计导入的总数量 try: for file in files: logger.info(f"正在处理文件: {file['id']}, 类型: {file['file_type']},{file['absolute_path']}") # **确保 df 在每次循环都重新创建** if file["file_type"] in ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "application/vnd.ms-excel"]: df = pd.read_excel(file["absolute_path"], dtype=str) elif file["file_type"] == "text/csv": df = pd.read_csv(file["absolute_path"], dtype=str) else: logger.error(f"文件 {file['id']} 类型错误!") continue # 跳过错误文件 df.columns = df.columns.str.strip().str.lower() # 统一列名 df = df.dropna(how="all") # 删除空行 df = df.drop_duplicates() # 删除重复行 # 处理 code 列，确保格式一致 if "code" in df.columns: df["code"] = df["code"].astype(str).str.zfill(8) df["code"] = df["code"].apply(lambda x: re.sub(SPECIAL_CHARS_PATTERN, "", x).strip()) else: logger.error(f"文件 {file['id']} 缺少 'code' 列") continue # 跳过错误文件 logger.info(f"文件 {file['id']} 解析出 {df.shape[0]} 条数据") # **确保 all_records 在每个文件都是新的** all_records = [] for _, row in df.iterrows(): all_records.append(CodeImport( code=row["code"], description=row.get("description", ""), status=3, user_id=user_id )) # **每个文件独立事务处理** async with in_transaction(): await CodeImport.bulk_create(all_records) logger.info(f"文件 {file['id']} 经过清理后成功导入 {len(all_records)} 条数据") total_imported += len(all_records) # 累加总数 # **清除 df 变量，确保下一个文件不会复用** del df except Exception as e: logger.error(f"文件导入失败: {str(e)}") return Response.failure(msg="文件读取失败") logger.info(f"✅ 所有文件导入完成，共导入 {total_imported} 条数据") return Response.success(msg="添加成功") @codeAPI.delete("/delete/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码") @codeAPI.post("/delete/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码") @Log(title="删除编码", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:delete"]) async def delete_code_by_id(request: Request, id: str = Path(description="编码ID"), current_user=Depends(LoginController.get_current_user)): if code := await Code.get_or_none(id=id, del_flag=1): code.del_flag = 0 await code.save() await request.app.state.es.delete(index=ElasticSearchConfig.ES_INDEX, id=code.id) return Response.success(msg="删除成功") else: return Response.failure(msg="编码不存在") @codeAPI.delete("/deleteList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码") @codeAPI.post("/deleteList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码") @Log(title="批量删除编码", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:delete"]) async def delete_code_by_ids(request: Request, params: DeleteListParams, current_user=Depends(LoginController.get_current_user)): # 异步批量查询 Code codes = await Code.filter(id__in=set(params.ids), del_flag=1) if not codes: return Response.error(msg="未找到相关数据") # 修改删除标志 for code in codes: code.del_flag = 0 # 异步批量更新 Code await Code.bulk_update(codes, fields=["del_flag"]) # 构造 Elasticsearch 删除操作 actions = [ { "_op_type": "delete", # 指定为删除操作 "_index": ElasticSearchConfig.ES_INDEX, "_id": code.id } for code in codes ] # 异步批量删除 Elasticsearch 数据 es_client = request.app.state.es if actions: success, failed = await async_bulk(es_client, actions) logger.info(f"成功删除 {success} 条数据，失败 {failed} 条") return Response.success(msg="删除成功") @codeAPI.put("/update/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="更新编码") @codeAPI.post("/update/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="更新编码") @Log(title="更新编码", business_type=BusinessType.UPDATE) @Auth(permission_list=["code:btn:update"]) async def update_code(request: Request, params: AddCodeParams, id: str = Path(description="编码ID"), current_user=Depends(LoginController.get_current_user)): if code := await Code.get_or_none(id=id, del_flag=1): code.code = params.code code.description = params.description await code.save() await request.app.state.es.update(index=ElasticSearchConfig.ES_INDEX, id=code.id, body={"doc": {"id": code.id, "code": code.code, "description": params.description}}) return Response.success(msg="更新成功") return Response.failure(msg="编码不存在") @codeAPI.get("/info/{id}", response_class=JSONResponse, response_model=GetCodeInfoResponse, summary="获取编码信息") @Log(title="获取编码信息", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:info"]) async def get_code_info(request: Request, id: str = Path(description="编码ID"), current_user=Depends(LoginController.get_current_user)): if code := await Code.get_or_none(id=id, del_flag=1).values( id="id", code="code", description="description", create_time="create_time", create_by="create_by", update_time="update_time", update_by="update_by", user_id="user__id", username="user__username", nickname="user__nickname", department_id="user__department__id", department_name="user__department__name", ): return Response.success(data=code) return Response.failure(msg="编码不存在") @codeAPI.get("/list", response_class=JSONResponse, response_model=GetCodeListResponse, summary="获取编码列表") @Log(title="获取编码列表", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:list"]) async def get_code_list(request: Request, page: int = Query(default=1, description="页码"), pageSize: int = Query(default=10, description="每页数量"), code: Optional[str] = Query(default=None, description="编码"), description: Optional[str] = Query(default=None, description="商品描述"), department_id: Optional[str] = Query(default=None, description="部门ID"), username: Optional[str] = Query(default=None, description="用户账号"), nickname: Optional[str] = Query(default=None, description="用户昵称"), startTime: Optional[str] = Query(default=None, description="开始时间"), endTime: Optional[str] = Query(default=None, description="结束时间"), current_user=Depends(LoginController.get_current_user)): filter_args = { f'{k}__icontains': v for k, v in { 'user__username': username, 'user__nickname': nickname, 'code': code, }.items() if v } if description: # 使用全文索引优化模糊查询 filter_args['description__full_text_search'] = description if startTime and endTime: startTime = float(startTime) / 1000 endTime = float(endTime) / 1000 startTime = datetime.fromtimestamp(startTime) endTime = datetime.fromtimestamp(endTime) filter_args['create_time__range'] = [startTime, endTime] if department_id: filter_args['user__department__id'] = department_id total = await Code.filter(**filter_args, del_flag=1).count() data = await Code.filter(**filter_args, del_flag=1).offset((page - 1) * pageSize).limit(pageSize).values( "id", "code", "description", "create_time", "create_by", "update_time", "update_by", user_id="user__id", username="user__username", nickname="user__nickname", department_id="user__department__id", department_name="user__department__name" ) return Response.success(data={ "page": page, "pageSize": pageSize, "total": total, "result": data }) @codeAPI.post("/query", response_class=JSONResponse, response_model=QueryCodeResponse, summary="查询编码") @Log(title="查询编码", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:query"]) async def get_code_list( request: Request, params: GetQueryCodeParams, current_user: dict = Depends(LoginController.get_current_user), ): start_time = time.time() user_id = current_user.get("id") # 预创建查询日志 log = await QueryCodeLog.create( operator_id=user_id, query_count=0, result_count=0, cost_time=0, request_params=params.query_text, response_result={}, status=0, del_flag=0, ) descriptions = list(desc.strip() for desc in params.query_text.split("\n") if desc.strip()) if not descriptions: return Response.failure(msg="查询失败！") query_count = len(descriptions) async def execute_es_queries(description_list: list | set) -> dict: """执行 Elasticsearch 批量查询""" es_queries = [] for desc in description_list: es_queries.append({}) es_queries.append({ "query": { "match": { "description": { "query": desc, "fuzziness": "AUTO" } } }, "size": 5, "_source": ["id", "code", "description"], "sort": [{"_score": {"order": "desc"}}], "timeout": "600s" }) return await request.app.state.es.msearch(index=ElasticSearchConfig.ES_INDEX, body=es_queries) async def process_es_results(es_results: dict, description_list: list | set) -> list: """处理 Elasticsearch 查询结果""" data_list = [] for i, desc in enumerate(description_list): hits = es_results["responses"][i]["hits"]["hits"] max_score = max(hit["_score"] for hit in hits) if hits else 1 matches = [ { "id": hit["_source"].get("id"), "code": hit["_source"].get("code"), "description": hit["_source"].get("description"), "match_rate": round((hit["_score"] / max_score) * 100, 2) if max_score else 0, } for hit in hits ] data_list.append({ "id": uuid.uuid4().__str__(), "query_text": desc, "result_text": json.dumps(matches, ensure_ascii=False), "status": 1 if matches else 0, }) return data_list async def update_query_log(log: QueryCodeLog, data_list: list, query_count: int, cost_time: float): """更新查询日志""" await QueryCodeLog.filter(id=log.id).update( request_params="\n".join(descriptions), query_count=query_count, result_count=len(data_list), cost_time=cost_time, status=1 if data_list else 0, response_result=json.dumps(data_list, ensure_ascii=False), del_flag=1, ) try: # 批量查询 Elasticsearch BATCH_SIZE = 300 # 每批查询的数量 description_batches = [descriptions[i:i + BATCH_SIZE] for i in range(0, len(descriptions), BATCH_SIZE)] all_results = [] for batch in description_batches: es_results = await execute_es_queries(batch) batch_results = await process_es_results(es_results, batch) all_results.extend(batch_results) # 批量插入查询结果 query_tasks = [ QueryCode( id=item["id"], query_text=desc, result_text=item["result_text"], session_id=log.id, status=item["status"], ) for desc, item in zip(descriptions, all_results) ] await QueryCode.bulk_create(query_tasks) # 更新查询日志 cost_time = round((time.time() - start_time) * 100, 2) await update_query_log(log, all_results, query_count, cost_time) return Response.success(data={ "id": log.id, "result_count": len(all_results), "query": "\n".join(descriptions), "response_result": json.dumps(all_results, ensure_ascii=False), "query_count": query_count, "cost_time": cost_time, "status": 1 if all_results else 0, "operation_time": log.operation_time, }) except Exception as e: logger.error(f"查询失败：{e}") await log.delete() raise ServiceException(message="查询失败！") @codeAPI.get("/query/{id}", response_class=JSONResponse, response_model=QueryCodeResponse, summary="查询编码") @Log(title="查询编码", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:importQuery"]) async def get_code_list( request: Request, id: str = Path(description="文件ID"), current_user: dict = Depends(LoginController.get_current_user), ): start_time = time.time() user_id = current_user.get("id") # 获取文件信息和上传者 ID file = await File.get_or_none(id=id, del_flag=1).values(uploader_id="uploader__id", file_type="file_type", absolute_path="absolute_path") if not file: return Response.failure(msg="文件不存在") if str(file["uploader_id"]) != user_id: raise PermissionException(message="权限不足") # 预创建查询日志 log = await QueryCodeLog.create( operator_id=user_id, query_count=0, result_count=0, cost_time=0, request_params="", response_result={}, status=0, del_flag=0, ) try: media_types = { "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": "excel", "application/vnd.ms-excel": "excel", "text/csv": "csv" } file_type = media_types.get(file["file_type"]) if not file_type: raise ServiceException(message="文件类型错误!") # 读取 Excel 或 CSV df = pd.read_excel(file["absolute_path"], dtype={"code": str}) if file_type == "excel" else pd.read_csv( file["absolute_path"], dtype={"code": str}) descriptions = list({row["text"].strip() for _, row in df.iterrows() if row["text"].strip()}) if not descriptions: raise ServiceException(message="文件内容为空!") query_count = len(descriptions) async def execute_es_queries(description_list: list | set) -> dict: """执行 Elasticsearch 批量查询""" es_queries = [] for desc in description_list: es_queries.append({}) es_queries.append({ "query": { "match": { "description": { "query": desc, "fuzziness": "AUTO" } } }, "size": 5, "_source": ["id", "code", "description"], "sort": [{"_score": {"order": "desc"}}], "timeout": "600s" }) return await request.app.state.es.msearch(index=ElasticSearchConfig.ES_INDEX, body=es_queries) async def process_es_results(es_results: dict, description_list: list | set) -> list: """处理 Elasticsearch 查询结果""" data_list = [] for i, desc in enumerate(description_list): hits = es_results["responses"][i]["hits"]["hits"] max_score = max(hit["_score"] for hit in hits) if hits else 1 matches = [ { "id": hit["_source"].get("id"), "code": hit["_source"].get("code"), "description": hit["_source"].get("description"), "match_rate": round((hit["_score"] / max_score) * 100, 2) if max_score else 0, } for hit in hits ] data_list.append({ "id": uuid.uuid4().__str__(), "query_text": desc, "result_text": json.dumps(matches, ensure_ascii=False), "status": 1 if matches else 0, }) return data_list async def update_query_log(log: QueryCodeLog, data_list: list, query_count: int, cost_time: float): """更新查询日志""" await QueryCodeLog.filter(id=log.id).update( request_params="\n".join(descriptions), query_count=query_count, result_count=len(data_list), cost_time=cost_time, status=1 if data_list else 0, response_result=json.dumps(data_list, ensure_ascii=False), del_flag=1, ) # 批量查询 Elasticsearch BATCH_SIZE = 300 # 每批查询的数量 description_batches = [descriptions[i:i + BATCH_SIZE] for i in range(0, len(descriptions), BATCH_SIZE)] all_results = [] for batch in description_batches: es_results = await execute_es_queries(batch) batch_results = await process_es_results(es_results, batch) all_results.extend(batch_results) # 批量插入查询结果 query_tasks = [ QueryCode( id=item["id"], query_text=desc, result_text=item["result_text"], session_id=log.id, status=item["status"], ) for desc, item in zip(descriptions, all_results) ] await QueryCode.bulk_create(query_tasks) # 更新查询日志 cost_time = round((time.time() - start_time) * 100, 2) await update_query_log(log, all_results, query_count, cost_time) return Response.success(data={ "id": log.id, "result_count": len(all_results), "query": "\n".join(descriptions), "response_result": json.dumps(all_results, ensure_ascii=False), "query_count": query_count, "cost_time": cost_time, "status": 1 if all_results else 0, "operation_time": log.operation_time, }) except Exception as e: logger.error(f"查询失败：{e}") await log.delete() raise ServiceException(message="查询失败！") @codeAPI.get("/logList", response_class=JSONResponse, response_model=GetQueryCodeLogResponse, summary="查询编码日志列表") @Log(title="查询编码日志列表", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:logList"]) async def get_code_log_list(request: Request, page: int = Query(default=1, description="当前页码"), pageSize: int = Query(default=10, description="每页数量"), department_id: Optional[str] = Query(default=None, description="部门ID"), username: Optional[str] = Query(default=None, description="用户账号"), nickname: Optional[str] = Query(default=None, description="用户昵称"), startTime: Optional[str] = Query(default=None, description="开始时间"), endTime: Optional[str] = Query(default=None, description="结束时间"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") filterArgs = { f'{k}__icontains': v for k, v in { 'operator__username': username, 'operator__nickname': nickname, }.items() if v } if startTime and endTime: startTime = float(startTime) / 1000 endTime = float(endTime) / 1000 startTime = datetime.fromtimestamp(startTime) endTime = datetime.fromtimestamp(endTime) filterArgs['operation_time__range'] = [startTime, endTime] if await hasAuth(request, "code:btn:logAdmin"): if department_id: filterArgs['operator__department__id'] = department_id else: filterArgs['operator__department__id__in'] = sub_departments else: if department_id: filterArgs['operator__department__id'] = department_id else: filterArgs['operator__department__id'] = current_user.get("department_id") count = await QueryCodeLog.filter(**filterArgs, operator__del_flag=1, del_flag=1).count() data = await QueryCodeLog.filter(**filterArgs, operator__del_flag=1, del_flag=1).order_by("-operation_time").offset( (page - 1) * pageSize).limit(pageSize).values( id="id", query_count="query_count", result_count="result_count", cost_time="cost_time", operation_time="operation_time", status="status", request_params="request_params", response_result="response_result", create_time="create_time", update_time="update_time", operator_id="operator__id", operator_name="operator__username", operator_nickname="operator__nickname", department_id="operator__department__id", department_name="operator__department__name", ) return Response.success(data={ "page": page, "pageSize": pageSize, "result": data, "total": count }) @codeAPI.get("/logList/all", response_class=JSONResponse, response_model=GetCodeLogAllResponse, summary="查询所有编码日志列表") @Log(title="查询所有编码日志列表", business_type=BusinessType.EXPORT) @Auth(permission_list=["code:btn:export"]) async def get_code_log_list(request: Request, department_id: Optional[str] = Query(default=None, description="部门ID"), username: Optional[str] = Query(default=None, description="用户账号"), nickname: Optional[str] = Query(default=None, description="用户昵称"), startTime: Optional[str] = Query(default=None, description="开始时间"), endTime: Optional[str] = Query(default=None, description="结束时间"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") filterArgs = { f'{k}__icontains': v for k, v in { 'operator__username': username, 'operator__nickname': nickname, }.items() if v } if startTime and endTime: startTime = float(startTime) / 1000 endTime = float(endTime) / 1000 startTime = datetime.fromtimestamp(startTime) endTime = datetime.fromtimestamp(endTime) filterArgs['operation_time__range'] = [startTime, endTime] if await hasAuth(request, "code:btn:logAdmin"): if department_id: filterArgs['operator__department__id'] = department_id else: filterArgs['operator__department__id__in'] = sub_departments else: if department_id: filterArgs['operator__department__id'] = department_id else: filterArgs['operator__department__id'] = current_user.get("department_id") count = await QueryCodeLog.filter(**filterArgs, operator__del_flag=1, del_flag=1).count() data = await QueryCodeLog.filter(**filterArgs, operator__del_flag=1, del_flag=1).order_by("-operation_time").values( id="id", query_count="query_count", result_count="result_count", cost_time="cost_time", operation_time="operation_time", status="status", request_params="request_params", response_result="response_result", create_time="create_time", update_time="update_time", operator_id="operator__id", operator_name="operator__username", operator_nickname="operator__nickname", department_id="operator__department__id", department_name="operator__department__name", ) return Response.success(data={ "result": data, "total": count }) @codeAPI.get("/logInfo/{id}", response_class=JSONResponse, response_model=GetQueryCodeLogDetailResponse, summary="查询编码日志详情") @Log(title="查询编码日志详情", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:logInfo"]) async def get_code_log_detail(request: Request, id: str = Path(..., description="日志ID"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") if log := await QueryCodeLog.get_or_none(id=id, operator__department__id__in=sub_departments, del_flag=1): data = await log.get(id=id).values( id="id", query_count="query_count", result_count="result_count", cost_time="cost_time", operation_time="operation_time", status="status", request_params="request_params", response_result="response_result", create_time="create_time", update_time="update_time", operator_id="operator__id", operator_name="operator__username", operator_nickname="operator__nickname", department_id="operator__department__id", department_name="operator__department__name", ) return Response.success(data=data) return Response.failure(msg="日志不存在！") @codeAPI.delete("/deleteFeedback/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码反馈") @codeAPI.post("/deleteFeedback/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码反馈") @Log(title="删除编码反馈", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:deleteFeedback"]) async def delete_feedback(request: Request, id: str = Path(..., description="编码反馈ID"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") if feedback := await CodeFeedback.get_or_none(id=id, user__department__id__in=sub_departments, del_flag=1): feedback.del_flag = 0 await feedback.save() return Response.success(msg="删除成功！") return Response.failure(msg="删除失败！") @codeAPI.post("/addFeedback", response_class=JSONResponse, response_model=BaseResponse, summary="添加编码反馈") @Log(title="添加编码反馈", business_type=BusinessType.INSERT) @Auth(permission_list=["code:btn:addFeedback"]) async def add_feedback(request: Request, params: AddCodeFeedbackParams, current_user: dict = Depends(LoginController.get_current_user), ): user_id = current_user.get("id") if code := await Code.get_or_none(id=params.code_id, del_flag=1): feedback = await CodeFeedback.create( code_id=code.id, feedback_code=params.feedback_code, feedback_description=params.feedback_description, user_id=str(user_id), ) else: feedback = await CodeFeedback.create( code=None, feedback_code=params.feedback_code, feedback_description=params.feedback_description, user_id=str(user_id), ) if feedback: return Response.success(msg="添加成功！") return Response.failure(msg="添加失败！") @codeAPI.delete("/deleteFeedbackList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码反馈") @codeAPI.post("/deleteFeedbackList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码反馈") @Log(title="批量删除编码反馈", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:deleteFeedback"]) async def delete_feedback_list(request: Request, params: DeleteListParams, current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") # 批量查询 CodeFeedback feedbacks = await CodeFeedback.filter( id__in=set(params.ids), user__department__id__in=sub_departments, del_flag=1 ) if not feedbacks: return Response.error(msg="未找到相关数据") # 修改删除标志 for feedback in feedbacks: feedback.del_flag = 0 # 批量更新数据 async with in_transaction(): await CodeFeedback.bulk_update(feedbacks, fields=["del_flag"]) return Response.success(msg="删除成功！") @codeAPI.put("/updateFeedback/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="修改编码反馈") @codeAPI.post("/updateFeedback/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="修改编码反馈") @Log(title="修改编码反馈", business_type=BusinessType.UPDATE) @Auth(permission_list=["code:btn:updateFeedback"]) async def update_feedback(request: Request, params: AddCodeFeedbackParams, id: str = Path(..., description="编码反馈ID"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") if feedback := await CodeFeedback.get_or_none(id=id, user__department__id__in=sub_departments, del_flag=1): code = await Code.get_or_none(id=params.code_id, del_flag=1) if code: feedback.code = code else: feedback.code = None feedback.feedback_description = params.feedback_description feedback.feedback_code = params.feedback_code await feedback.save() return Response.success(msg="修改成功！") return Response.failure(msg="编码反馈不存在！") @codeAPI.get("/feedbackInfo/{id}", response_class=JSONResponse, response_model=GetCodeFeedbackResponse, summary="编码反馈详情") @Log(title="编码反馈详情", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:feedbackInfo"]) async def feedback_info(request: Request, id: str = Path(..., description="编码反馈ID"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") if feedback := await CodeFeedback.get_or_none(id=id, user__department__id__in=sub_departments, del_flag=1): data = await feedback.first().values( id="id", code_id="code__id", code="code__code", description="code__description", feedback_code="feedback_code", feedback_description="feedback_description", create_time="create_time", update_time="update_time", user_id="user__id", username="user__username", nickname="user__nickname", department_id="user__department__id", department_name="user__department__name", ) return Response.success(data=data) return Response.failure(msg="编码反馈不存在！") @codeAPI.get("/feedbackList", response_class=JSONResponse, response_model=GetCodeFeedbackListResponse, summary="编码反馈列表") @Log(title="编码反馈列表", business_type=BusinessType.SELECT) @Auth(permission_list=["code:btn:feedbackList"]) async def feedback_list(request: Request, page: int = Query(default=1, description="当前页码"), pageSize: int = Query(default=10, description="每页数量"), code: Optional[str] = Query(default=None, description="编码"), feedback_code: Optional[str] = Query(default=None, description="反馈编码"), feedback_description: Optional[str] = Query(default=None, description="反馈文本"), username: Optional[str] = Query(default=None, description="用户名"), status: Optional[str] = Query(default=None, description="状态"), nickname: Optional[str] = Query(default=None, description="用户昵称"), department_id: Optional[str] = Query(default=None, description="部门ID"), startTime: Optional[str] = Query(default=None, description="开始时间"), endTime: Optional[str] = Query(default=None, description="结束时间"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") filterArgs = { f'{k}__icontains': v for k, v in { 'user__username': username, 'user__nickname': nickname, 'code__code': code, 'feedback_code': feedback_code, 'feedback_description': feedback_description, }.items() if v } if startTime and endTime: startTime = float(startTime) / 1000 endTime = float(endTime) / 1000 startTime = datetime.fromtimestamp(startTime) endTime = datetime.fromtimestamp(endTime) filterArgs['create_time__range'] = [startTime, endTime] if await hasAuth(request, "code:btn:feedbackAdmin"): if department_id: filterArgs['user__department__id'] = department_id else: filterArgs['user__department__id__in'] = sub_departments else: if department_id: filterArgs['user__department__id'] = department_id else: filterArgs['user__department__id'] = current_user.get("department_id") if status is not None: filterArgs['status'] = int(status) total = await CodeFeedback.filter(**filterArgs, del_flag=1).count() data = await CodeFeedback.filter(**filterArgs, del_flag=1).order_by('-create_time').offset( (page - 1) * pageSize).limit(pageSize).values( id="id", code_id="code__id", code="code__code", description="code__description", feedback_code="feedback_code", feedback_description="feedback_description", create_time="create_time", update_time="update_time", user_id="user__id", username="user__username", nickname="user__nickname", department_id="user__department__id", department_name="user__department__name", status="status", ) return Response.success(data={ "page": page, "pageSize": pageSize, "result": data, "total": total, }) @codeAPI.put("/feedbackAudit", response_class=JSONResponse, response_model=BaseResponse, summary="编码反馈审核") @codeAPI.post("/feedbackAudit", response_class=JSONResponse, response_model=BaseResponse, summary="编码反馈审核") @Log(title="编码反馈审核", business_type=BusinessType.UPDATE) @Auth(permission_list=["code:btn:feedbackAudit"]) async def feedback_audit(request: Request, params: UpdateCodeFeedbackStatusParams, current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") # 获取所有符合条件的反馈数据 feedback_list = await CodeFeedback.filter( id__in=set(params.ids), user__department__id__in=sub_departments, del_flag=1 ).prefetch_related("user") # 预加载 user 关联数据，减少查询 if not feedback_list: return Response.failure(msg="编码反馈不存在！") code_updates = [] # 需要更新的 Code 对象 code_creates = [] # 需要创建的 Code 对象 es_bulk_operations = [] # 批量更新 Elasticsearch async with in_transaction(): for feedback in feedback_list: feedback.status = params.status if params.status == 1: # 查找对应的 Code code = await Code.get_or_none(id=feedback.code_id, del_flag=1) if code: # 更新 Code code.code = re.sub(SPECIAL_CHARS_PATTERN, "", str(feedback.feedback_code)).strip() code.description = re.sub(SPECIAL_CHARS_PATTERN, "", str(feedback.feedback_description)).strip() code_updates.append(code) else: # 创建新 Code code = Code( user_id=feedback.user_id, code=re.sub(SPECIAL_CHARS_PATTERN, "", str(feedback.feedback_code)).strip(), description=re.sub(SPECIAL_CHARS_PATTERN, "", str(feedback.feedback_description)).strip() ) code_creates.append(code) # Elasticsearch 更新或创建操作 es_bulk_operations.append({ "update" if code.id else "create": { "_index": ElasticSearchConfig.ES_INDEX, "_id": code.id or None, "doc" if code.id else "doc_as_upsert": { "id": code.id, "code": code.code, "description": code.description } } }) # 记录更新 feedback await feedback.save() # 批量更新 Code if code_updates: await Code.bulk_update(code_updates, fields=["code", "description"]) # 批量创建 Code if code_creates: await Code.bulk_create(code_creates) # 批量更新 Elasticsearch if es_bulk_operations: await request.app.state.es.bulk(index=ElasticSearchConfig.ES_INDEX, body=es_bulk_operations) return Response.success(msg="审核成功！") @codeAPI.delete("/deleteCodeImport/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码导入") @codeAPI.post("/deleteCodeImport/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="删除编码导入") @Log(title="删除编码导入", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:deleteCodeImport"]) async def delete_code_import(request: Request, id: str = Path(description="编码导入ID"), current_user: dict = Depends(LoginController.get_current_user)): sub_departments = current_user.get("sub_departments") if code_import := await CodeImport.get_or_none(id=id, user__department__id__in=sub_departments, del_flag=1): code_import.del_flag = 0 await code_import.save() return Response.success() return Response.failure(msg="编码导入不存在！") @codeAPI.delete("/deleteCodeImportList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码导入") @codeAPI.post("/deleteCodeImportList", response_class=JSONResponse, response_model=BaseResponse, summary="批量删除编码导入") @Log(title="批量删除编码导入", business_type=BusinessType.DELETE) @Auth(permission_list=["code:btn:deleteCodeImport"]) async def delete_code_import_list(request: Request, params: DeleteListParams, current_user: dict = Depends(LoginController.get_current_user)): sub_departments = current_user.get("sub_departments") # 获取符合条件的所有 CodeImport 记录 code_imports = await CodeImport.filter( id__in=set(params.ids), user__department__id__in=sub_departments, del_flag=1 ) if not code_imports: return Response.failure(msg="编码导入不存在！") # 批量修改 del_flag for code_import in code_imports: code_import.del_flag = 0 # 事务内批量更新 async with in_transaction(): await CodeImport.bulk_update(code_imports, fields=["del_flag"]) return Response.success() @codeAPI.put("/updateCodeImport/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="修改编码导入") @codeAPI.post("/updateCodeImport/{id}", response_class=JSONResponse, response_model=BaseResponse, summary="修改编码导入") @Log(title="修改编码导入", business_type=BusinessType.UPDATE) @Auth(permission_list=["code:btn:updateCodeImport"]) async def update_code_import(request: Request, params: AddCodeParams, id: str = Path(description="编码导入ID"), current_user: dict = Depends(LoginController.get_current_user)): sub_departments = current_user.get("sub_departments") if code_import := await CodeImport.get_or_none(id=id, user__department__id__in=sub_departments, del_flag=1): code_import.code = params.code code_import.description = params.description code_import.status = 3 await code_import.save() return Response.success() return Response.failure(msg="编码导入不存在！") @codeAPI.get("/codeImportList", response_class=JSONResponse, response_model=GetCodeImportListResponse, summary="查询编码导入列表") @Auth(permission_list=["code:btn:codeImportList"]) async def get_code_import_list( request: Request, page: int = Query(default=1, description="页码"), pageSize: int = Query(default=10, description="每页数量"), code: Optional[str] = Query(default=None, description="编码"), description: Optional[str] = Query(default=None, description="商品描述"), username: Optional[str] = Query(default=None, description="用户名"), status: Optional[str] = Query(default=None, description="状态"), nickname: Optional[str] = Query(default=None, description="用户昵称"), department_id: Optional[str] = Query(default=None, description="部门ID"), startTime: Optional[str] = Query(default=None, description="开始时间"), endTime: Optional[str] = Query(default=None, description="结束时间"), current_user: dict = Depends(LoginController.get_current_user), ): sub_departments = current_user.get("sub_departments") filter_args = { f'{k}__icontains': v for k, v in { 'user__username': username, 'user__nickname': nickname, 'code': code, }.items() if v } if description: # 使用全文索引优化模糊查询 filter_args['description__full_text_search'] = description if startTime and endTime: startTime = float(startTime) / 1000 endTime = float(endTime) / 1000 startTime = datetime.fromtimestamp(startTime) endTime = datetime.fromtimestamp(endTime) filter_args['create_time__range'] = [startTime, endTime] if await hasAuth(request, "code:btn:codeImportAdmin"): if department_id: filter_args['user__department__id'] = department_id else: filter_args['user__department__id__in'] = sub_departments else: if department_id: filter_args['user__department__id'] = department_id else: filter_args['user__department__id'] = current_user.get("department_id") if status is not None: filter_args['status'] = int(status) # 查询总记录数 total = await CodeImport.filter(**filter_args, del_flag=1).count() # 分页查询 data = await CodeImport.filter(**filter_args, del_flag=1).order_by('-create_time').offset( (page - 1) * pageSize).limit(pageSize).values( id="id", code="code", description="description", create_time="create_time", update_time="update_time", user_id="user__id", username="user__username", nickname="user__nickname", department_id="user__department__id", department_name="user__department__name", status="status", ) return Response.success(data={ "page": page, "pageSize": pageSize, "result": data, "total": total, }) @codeAPI.put("/codeImportAudit", response_class=JSONResponse, response_model=BaseResponse, summary="编码导入审核") @codeAPI.post("/codeImportAudit", response_class=JSONResponse, response_model=BaseResponse, summary="编码导入审核") @Log(title="编码导入审核", business_type=BusinessType.INSERT) @Auth(permission_list=["code:btn:codeImportAudit"]) async def code_import_audit(request: Request, params: UpdateCodeImportStatusParams, current_user: dict = Depends(LoginController.get_current_user)): sub_departments = current_user.get("sub_departments") user_id = current_user.get("id") # 批量查询 CodeImport 记录 code_imports = await CodeImport.filter( id__in=set(params.ids), user__department__id__in=sub_departments, del_flag=1 ) if not code_imports: return Response.error() # 如果没有找到符合条件的记录，返回错误 actions = [] # Elasticsearch 批量操作 codes_to_create = [] # 需要批量创建的 Code 数据 # 处理每个 CodeImport for code_import in code_imports: code_import.status = params.status # 更新状态 if params.status == 1: # 只有审核通过时才需要创建 Code # 清理代码中的特殊字符 clean_code = re.sub(SPECIAL_CHARS_PATTERN, "", code_import.code).strip() clean_description = re.sub(SPECIAL_CHARS_PATTERN, "", code_import.description).strip() codes_to_create.append(Code( code=clean_code, description=clean_description, user_id=user_id )) if not codes_to_create: # 如果没有要创建的 Code 数据，直接返回 return Response.error("没有需要创建的编码数据") # 执行批量数据库操作 async with in_transaction(): # 批量更新 CodeImport 状态 await CodeImport.bulk_update(code_imports, fields=["status"]) # 批量创建 Code 数据 await Code.bulk_create(codes_to_create) # 不需要接收返回值 # 构建 Elasticsearch 批量操作数据 for code_info in codes_to_create: # 使用 codes_to_create 列表中的对象 actions.append({ "_index": ElasticSearchConfig.ES_INDEX, "_id": code_info.id, # 使用 Code 的 ID 作为 Elasticsearch 的文档 ID "_source": { "id": code_info.id, "code": code_info.code, "description": code_info.description } }) # 确保 Elasticsearch 索引存在 es_client = request.app.state.es if not await es_client.indices.exists(index=ElasticSearchConfig.ES_INDEX): await es_client.indices.create(index=ElasticSearchConfig.ES_INDEX, ignore=400) # 批量写入 Elasticsearch 数据 if actions: success, failed = await async_bulk(es_client, actions) logger.info(f"成功导入 {success} 条数据，失败 {failed} 条") return Response.success() @codeAPI.put("/codeImportAudit/all", response_class=JSONResponse, response_model=BaseResponse, summary="全部审核通过") @codeAPI.post("/codeImportAudit/all", response_class=JSONResponse, response_model=BaseResponse, summary="全部审核通过") @Log(title="全部审核通过", business_type=BusinessType.UPDATE) @Auth(permission_list=["code:btn:codeImportAuditAll"]) async def code_import_audit_all(request: Request, current_user: dict = Depends(LoginController.get_current_user)): sub_departments = current_user.get("sub_departments") user_id = current_user.get("id") actions = [] # Elasticsearch 批量操作 codes_to_create = [] # 批量插入的 Code 数据 code_ids_to_update = [] # 批量更新的 CodeImport 数据 BATCH_SIZE = 10000 # 每批处理的数量 offset = 0 # 分批查询游标 # 确保 Elasticsearch 索引存在 es_client = request.app.state.es if not await es_client.indices.exists(index=ElasticSearchConfig.ES_INDEX): await es_client.indices.create(index=ElasticSearchConfig.ES_INDEX, ignore=400) while True: # 分批查询 CodeImport 数据 code_imports = await CodeImport.filter(user__department__id__in=sub_departments, del_flag=1, status=3).offset( offset).limit(BATCH_SIZE).all() if not code_imports: break # 如果没有更多数据，退出循环 # 遍历处理每个 CodeImport for code_import in code_imports: code_import.status = 1 # 审核通过，更新状态 # 清理 code 字段中的特殊字符 cleaned_code = re.sub(SPECIAL_CHARS_PATTERN, "", str(code_import.code)).strip() cleaned_description = re.sub(SPECIAL_CHARS_PATTERN, "", str(code_import.description)).strip() # 批量创建 Code 数据 codes_to_create.append(Code( code=cleaned_code, description=cleaned_description, user_id=user_id, )) # 构造 Elasticsearch 批量操作数据 actions.append( { "_index": ElasticSearchConfig.ES_INDEX, "_id": code_import.id, # 这里用 code_import 的 ID 作为 Elasticsearch 的 ID "_source": { "id": code_import.id, "code": cleaned_code, "description": code_import.description } } ) # 保存需要更新的 CodeImport 数据 code_ids_to_update.append(code_import.id) # 批量更新 CodeImport 的状态 if code_ids_to_update: async with in_transaction(): await CodeImport.filter(id__in=code_ids_to_update).update(status=1) # 批量插入 Code 数据 if codes_to_create: await Code.bulk_create(codes_to_create) # 批量写入 Elasticsearch if actions: success, failed = await async_bulk(es_client, actions) logger.info(f"成功导入 {success} 条数据，失败 {failed} 条") # 清空操作数组，为下一批数据准备 actions.clear() codes_to_create.clear() code_ids_to_update.clear() offset += BATCH_SIZE # 更新查询游标，继续查询下一批数据 return Response.success() @codeAPI.get("/hts", response_class=JSONResponse, response_model=BaseResponse, summary="查询HTS") async def query_hts(request: Request, code=Query(description="编码"), current_user: dict = Depends(LoginController.get_current_user)): def normalize_numeric_code(code_str): """ 处理编码字符串： 1. 只保留数字字符 2. 截取前10位（不足补0） :param code_str: 原始编码字符串 :return: 处理后的10位纯数字编码 """ # 1. 只保留数字 digits_only = re.sub(r'[^0-9]', '', code_str) # 2. 锁定10位长度，右侧补0 normalized = digits_only.ljust(10, '0')[:10] return normalized code = normalize_numeric_code(code) oneResult = await HtsItem.get_or_none(htsno=code[:2]).values( id="id", create_time="create_time", update_time="update_time", parent_id="parent_id", htsno="htsno", indent="indent", description="description", units="units", general="general", special="special", other="other", quota_quantity="quota_quantity", additional_duties="additional_duties", footnotes="footnotes", ) secondResult = await HtsItem.get_or_none(htsno=code[:4]).values( id="id", create_time="create_time", update_time="update_time", parent_id="parent_id", htsno="htsno", indent="indent", description="description", units="units", general="general", special="special", other="other", quota_quantity="quota_quantity", additional_duties="additional_duties", footnotes="footnotes", ) threeResult = await HtsItem.get_or_none(htsno=code[:6]).values( id="id", create_time="create_time", update_time="update_time", parent_id="parent_id", htsno="htsno", indent="indent", description="description", units="units", general="general", special="special", other="other", quota_quantity="quota_quantity", additional_duties="additional_duties", footnotes="footnotes", ) fourResult = await HtsItem.get_or_none(htsno=code[:8]).values( id="id", create_time="create_time", update_time="update_time", parent_id="parent_id", htsno="htsno", indent="indent", description="description", units="units", general="general", special="special", other="other", quota_quantity="quota_quantity", additional_duties="additional_duties", footnotes="footnotes", ) fiveResult = await HtsItem.get_or_none(htsno=code).values( id="id", create_time="create_time", update_time="update_time", parent_id="parent_id", htsno="htsno", indent="indent", description="description", units="units", general="general", special="special", other="other", quota_quantity="quota_quantity", additional_duties="additional_duties", footnotes="footnotes", ) return Response.success( data={ "oneResult": oneResult, "secondResult": secondResult, "threeResult": threeResult, "fourResult": fourResult, "fiveResult": fiveResult, } )