308 lines
13 KiB
Python
308 lines
13 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""文档数据脱敏系统 - Web 平台（FastAPI, 端口 8736）
|
||
|
||
平台组负责：app.py / static / templates。
|
||
管线逻辑来自 tmkit（splitter/extractor/detector/redactor）；若 tmkit 尚未就绪，
|
||
自动降级为占位逻辑（TODO 标注），保证页面可用。
|
||
"""
|
||
import re
|
||
import shutil
|
||
import threading
|
||
import time
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import Optional
|
||
|
||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||
from fastapi.responses import FileResponse, HTMLResponse
|
||
|
||
WORKDIR = Path(__file__).resolve().parent
|
||
UPLOAD_ROOT = Path("/tmp/tm-uploads")
|
||
UPLOAD_ROOT.mkdir(parents=True, exist_ok=True)
|
||
|
||
# ---------------- tmkit 装载（可降级） ----------------
|
||
try:
|
||
import tmkit # noqa
|
||
TMKIT_READY = True
|
||
TMKIT_ERR = None
|
||
except Exception as e: # ImportError 或子模块缺失
|
||
tmkit = None
|
||
TMKIT_READY = False
|
||
TMKIT_ERR = str(e)
|
||
|
||
|
||
def process_pdf(input_path: Path, out_path: Path, task: dict):
|
||
"""核心处理：tmkit 就绪走真管线，否则占位逻辑。返回敏感项列表。"""
|
||
items = []
|
||
if TMKIT_READY:
|
||
import fitz # PyMuPDF，tmkit 的底层依赖
|
||
|
||
pdf_bytes = input_path.read_bytes()
|
||
|
||
# 阶段 1：分流（native / scanned）
|
||
task["stage"] = "页面分流"
|
||
task["progress"] = 10
|
||
page_types = tmkit.classify_pages(pdf_bytes) or []
|
||
|
||
# 阶段 2+3：逐页抽取 + PII 识别
|
||
task["stage"] = "文字抽取与敏感项识别"
|
||
task["progress"] = 40
|
||
detector = tmkit.default_detector()
|
||
doc = fitz.open(stream=pdf_bytes, filetype="pdf")
|
||
hits, items = [], []
|
||
for i, page in enumerate(doc):
|
||
page_no = i + 1 # 对外展示用 1 基页码
|
||
words = tmkit.extract_page_words(page)
|
||
found = detector.scan_page_words(words, i) # tmkit/redactor 内部用 0 基
|
||
for h in found or []:
|
||
items.append({"type": h.get("type", "未知"),
|
||
"text": h.get("text", ""),
|
||
"page": h.get("page", page_no)})
|
||
if h.get("bbox") is not None:
|
||
# 回写用 0 基页码，展示用 1 基，两者分开
|
||
hits.append({"page": i, "bbox": h["bbox"]})
|
||
|
||
# 阶段 4：脱敏回写
|
||
task["stage"] = "脱敏回写"
|
||
task["progress"] = 85
|
||
out_bytes = tmkit.redact_pdf(pdf_bytes, hits)
|
||
out_path.write_bytes(out_bytes)
|
||
else:
|
||
# ---------- TODO 占位逻辑（tmkit 未就绪） ----------
|
||
task["stage"] = "占位处理（tmkit 未就绪）"
|
||
task["progress"] = 50
|
||
time.sleep(1.2) # 模拟耗时，方便前端看到进度
|
||
raw = input_path.read_bytes()
|
||
# 用正则从原始字节里粗扫手机号样式，仅作演示
|
||
text = raw.decode("latin-1", errors="ignore")
|
||
for m in re.finditer(r"1[3-9]\d{9}", text):
|
||
items.append({"type": "手机号(占位)", "text": m.group(), "page": 1})
|
||
task["progress"] = 85
|
||
shutil.copy(input_path, out_path)
|
||
# ---------- 占位结束 ----------
|
||
if not out_path.exists():
|
||
shutil.copy(input_path, out_path)
|
||
return items
|
||
|
||
|
||
# ---------------- 任务表 ----------------
|
||
TASKS: dict = {} # task_id -> task dict
|
||
LOCK = threading.Lock()
|
||
|
||
|
||
def _run_task(task_id: str, input_path: Path, out_path: Path):
|
||
task = TASKS[task_id]
|
||
try:
|
||
items = process_pdf(input_path, out_path, task)
|
||
with LOCK:
|
||
task.update(status="done", stage="完成", progress=100,
|
||
items=items, finished_at=time.time())
|
||
except Exception as e:
|
||
with LOCK:
|
||
task.update(status="error", stage="失败", error=str(e))
|
||
|
||
|
||
app = FastAPI(title="文档数据脱敏系统")
|
||
|
||
|
||
@app.get("/api/tasks")
|
||
def list_tasks(task_id: Optional[str] = None):
|
||
with LOCK:
|
||
if task_id:
|
||
if task_id not in TASKS:
|
||
raise HTTPException(404, "task not found")
|
||
return TASKS[task_id]
|
||
return {"tmkit_ready": TMKIT_READY, "tmkit_error": TMKIT_ERR,
|
||
"tasks": list(TASKS.values())}
|
||
|
||
|
||
@app.post("/api/process")
|
||
async def process(file: UploadFile = File(...)):
|
||
if not (file.filename or "").lower().endswith(".pdf"):
|
||
raise HTTPException(400, "仅支持 PDF 文件")
|
||
task_id = uuid.uuid4().hex[:12]
|
||
tdir = UPLOAD_ROOT / task_id
|
||
tdir.mkdir(parents=True, exist_ok=True)
|
||
input_path = tdir / "input.pdf"
|
||
out_path = tdir / "redacted.pdf"
|
||
with open(input_path, "wb") as f:
|
||
shutil.copyfileobj(file.file, f)
|
||
TASKS[task_id] = {
|
||
"task_id": task_id, "filename": file.filename, "status": "processing",
|
||
"stage": "排队中", "progress": 5, "items": [],
|
||
"created_at": time.time(), "dir": str(tdir),
|
||
}
|
||
threading.Thread(target=_run_task, args=(task_id, input_path, out_path), daemon=True).start()
|
||
return {"task_id": task_id}
|
||
|
||
|
||
@app.get("/download/{task_id}")
|
||
def download(task_id: str, type: str = "redacted"):
|
||
task = TASKS.get(task_id)
|
||
if not task:
|
||
raise HTTPException(404, "task not found")
|
||
if task.get("status") != "done":
|
||
raise HTTPException(409, "任务未完成")
|
||
tdir = Path(task["dir"])
|
||
src = tdir / ("redacted.pdf" if type == "redacted" else "input.pdf")
|
||
if not src.exists():
|
||
raise HTTPException(404, "文件不存在")
|
||
if type == "redacted":
|
||
name = f"{Path(task['filename']).stem}_脱敏.pdf"
|
||
else:
|
||
name = task["filename"]
|
||
return FileResponse(src, filename=name)
|
||
|
||
|
||
@app.get("/progress", response_class=HTMLResponse)
|
||
def progress_page():
|
||
blocks = []
|
||
for key, label in (("backend", "管线组"), ("web", "平台组（Web）"), ("deploy", "部署组")):
|
||
p = WORKDIR / f"PROGRESS-{key}.md"
|
||
content = p.read_text(encoding="utf-8") if p.exists() else "> 进行中（暂无进展文件）"
|
||
blocks.append('<details open class="prog"><summary>{}</summary>'
|
||
'<pre>{}</pre></details>'.format(label, content.replace("&", "&").replace("<", "<")))
|
||
return """<!doctype html><html lang="zh"><head><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>开发进展看板 · 文档数据脱敏系统</title><style>
|
||
body{font-family:-apple-system,"PingFang SC",sans-serif;margin:0;background:#f5f7fa;color:#222}
|
||
header{background:#1f2d3d;color:#fff;padding:18px 28px;font-size:18px;font-weight:600}
|
||
main{max-width:900px;margin:24px auto;padding:0 16px}
|
||
details{background:#fff;border-radius:8px;margin-bottom:14px;box-shadow:0 1px 4px rgba(0,0,0,.08)}
|
||
summary{padding:12px 16px;font-weight:600;cursor:pointer;color:#1f2d3d}
|
||
pre{white-space:pre-wrap;margin:0;padding:4px 16px 16px;font-size:13px;line-height:1.7;color:#444}
|
||
</style></head><body>
|
||
<header>文档数据脱敏系统 · 开发进展看板</header>
|
||
<main>""" + "".join(blocks) + """</main></body></html>"""
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
def index():
|
||
return INDEX_HTML
|
||
|
||
|
||
INDEX_HTML = """<!doctype html>
|
||
<html lang="zh"><head><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>文档数据脱敏系统</title><style>
|
||
:root{--c:#2563eb;--bg:#f5f7fa;--card:#fff;--border:#e5e7eb;--danger:#dc2626}
|
||
*{box-sizing:border-box}
|
||
body{font-family:-apple-system,"PingFang SC","Microsoft YaHei",sans-serif;margin:0;background:var(--bg);color:#1f2937}
|
||
header{background:linear-gradient(135deg,#1e3a8a,#2563eb);color:#fff;padding:28px 20px;text-align:center}
|
||
header h1{margin:0;font-size:22px}header p{margin:6px 0 0;opacity:.85;font-size:13px}
|
||
main{max-width:860px;margin:0 auto;padding:20px 16px 60px}
|
||
.card{background:var(--card);border-radius:10px;padding:20px;margin-bottom:18px;box-shadow:0 1px 4px rgba(0,0,0,.06)}
|
||
#drop{border:2px dashed var(--border);border-radius:10px;padding:44px 16px;text-align:center;color:#6b7280;cursor:pointer;transition:.2s}
|
||
#drop.over{border-color:var(--c);background:#eff6ff;color:var(--c)}
|
||
#drop .icon{font-size:36px}
|
||
button{background:var(--c);color:#fff;border:none;border-radius:6px;padding:10px 22px;font-size:14px;cursor:pointer}
|
||
button:hover{opacity:.9}
|
||
button.ghost{background:#fff;color:var(--c);border:1px solid var(--c)}
|
||
.bar{height:8px;background:var(--border);border-radius:4px;overflow:hidden;margin:10px 0}
|
||
.bar>i{display:block;height:100%;background:var(--c);width:0;transition:width .3s}
|
||
table{width:100%;border-collapse:collapse;font-size:13px}
|
||
th,td{border-bottom:1px solid var(--border);padding:8px 10px;text-align:left}
|
||
th{background:#f9fafb;color:#6b7280;font-weight:600}
|
||
.err{color:var(--danger)}
|
||
.hidden{display:none}
|
||
#statusLine{font-size:13px;color:#6b7280}
|
||
a.dl{display:inline-block;text-decoration:none}
|
||
.actions a{margin-right:10px}
|
||
small.tip{color:#9ca3af}
|
||
</style></head><body>
|
||
<header><h1>📄 文档数据脱敏系统</h1><p>上传 PDF，自动识别并脱敏敏感信息（手机号 / 身份证 / 银行卡等）</p></header>
|
||
<main>
|
||
<div class="card">
|
||
<div id="drop">
|
||
<div class="icon">📁</div>
|
||
<div>将 PDF 拖拽到此处，或点击选择文件</div>
|
||
<div style="margin-top:12px"><button id="pickBtn" type="button">选择 PDF 文件</button></div>
|
||
<input type="file" id="fileInput" accept="application/pdf,.pdf" style="display:none">
|
||
<div id="fileName" style="margin-top:8px;font-size:13px"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="card hidden" id="progressCard">
|
||
<b>处理进度</b>
|
||
<div class="bar"><i id="barFill"></i></div>
|
||
<div id="statusLine">等待中…</div>
|
||
<button id="cancelBtn" class="ghost" type="button" style="margin-top:8px" onclick="location.reload()">重新开始</button>
|
||
</div>
|
||
|
||
<div class="card hidden" id="resultCard">
|
||
<b>识别到的敏感项（<span id="itemCount">0</span> 项）</b>
|
||
<div style="max-height:320px;overflow:auto;margin-top:8px">
|
||
<table><thead><tr><th>类型</th><th>原文</th><th>页码</th></tr></thead>
|
||
<tbody id="itemBody"></tbody></table>
|
||
</div>
|
||
<div class="actions">
|
||
<a class="dl" id="dlRedacted" href="#"><button type="button">⬇️ 下载脱敏后文件</button></a>
|
||
<a class="dl" id="dlOriginal" href="#"><button class="ghost" type="button">保留原件下载</button></a>
|
||
</div>
|
||
<small class="tip" id="modeTip"></small>
|
||
</div>
|
||
</main>
|
||
<script>
|
||
const $=id=>document.getElementById(id);
|
||
let file=null,taskId=null,timer=null;
|
||
const drop=$('drop'),input=$('fileInput');
|
||
drop.onclick=()=>input.click();
|
||
input.onchange=()=>{if(input.files[0])setFile(input.files[0])};
|
||
['dragover','dragenter'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.add('over')}));
|
||
['dragleave','drop'].forEach(e=>drop.addEventListener(e,ev=>{ev.preventDefault();drop.classList.remove('over')}));
|
||
drop.addEventListener('drop',ev=>{const f=ev.dataTransfer.files[0];if(f)setFile(f)});
|
||
function setFile(f){
|
||
if(!f.name.toLowerCase().endsWith('.pdf')){alert('请上传 PDF 文件');return}
|
||
file=f;$('fileName').textContent='已选择：'+f.name+'（'+(f.size/1024).toFixed(1)+' KB）';
|
||
upload();
|
||
}
|
||
async function upload(){
|
||
$('progressCard').classList.remove('hidden');
|
||
$('statusLine').textContent='上传中…';$('barFill').style.width='5%';
|
||
const fd=new FormData();fd.append('file',file);
|
||
try{
|
||
const r=await fetch('/api/process',{method:'POST',body:fd});
|
||
if(!r.ok){const e=await r.json().catch(()=>({}));throw new Error(e.detail||r.status)}
|
||
const d=await r.json();taskId=d.task_id;poll();
|
||
}catch(err){$('statusLine').innerHTML='<span class="err">上传失败：'+err.message+'</span>'}
|
||
}
|
||
function poll(){
|
||
timer=setInterval(async()=>{
|
||
try{
|
||
const r=await fetch('/api/tasks?task_id='+taskId);
|
||
if(!r.ok)return;
|
||
const t=await r.json();
|
||
$('barFill').style.width=t.progress+'%';
|
||
if(t.status==='processing'){$('statusLine').textContent='正在处理：'+t.stage+'（'+t.progress+'%）';return}
|
||
clearInterval(timer);
|
||
if(t.status==='error'){$('statusLine').innerHTML='<span class="err">处理失败：'+(t.error||'未知错误')+'</span>';return}
|
||
$('statusLine').textContent='处理完成 ✓';showResult(t);
|
||
}catch(e){}
|
||
},800);
|
||
}
|
||
function showResult(t){
|
||
$('progressCard').classList.add('hidden');
|
||
$('resultCard').classList.remove('hidden');
|
||
$('itemCount').textContent=t.items.length;
|
||
const tb=$('itemBody');tb.innerHTML='';
|
||
if(!t.items.length){tb.innerHTML='<tr><td colspan="3" style="color:#9ca3af">未识别到敏感项</td></tr>'}
|
||
t.items.forEach(it=>{
|
||
const tr=document.createElement('tr');
|
||
tr.innerHTML='<td>'+esc(it.type)+'</td><td>'+esc(it.text)+'</td><td>'+esc(String(it.page))+'</td>';
|
||
tb.appendChild(tr);
|
||
});
|
||
$('dlRedacted').href='/download/'+t.task_id+'?type=redacted';
|
||
$('dlOriginal').href='/download/'+t.task_id+'?type=original';
|
||
fetch('/api/tasks').then(r=>r.json()).then(d=>{
|
||
if(!d.tmkit_ready)$('modeTip').textContent='⚠️ 当前为占位模式（tmkit 管线未就绪），结果仅供演示';
|
||
}).catch(()=>{});
|
||
}
|
||
function esc(s){return String(s).replace(/&/g,'&').replace(/</g,'<')}
|
||
</script>
|
||
</body></html>"""
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import uvicorn
|
||
uvicorn.run(app, host="0.0.0.0", port=8736)
|