Commit c9146ddc authored by wenwen.tang's avatar wenwen.tang 😕

feat(tools): 数据回拉通道 + 合同×台账核对脚本

- tools/sync_pull.py:把线上 /opt/rentos/data 回拉到本地 data/(deploy_git.py 只管推代码)。
  --dry 先列差异 → 备份 data/_snap/pull-<ts>/ → SFTP 递归下载;默认跳过 attachments(61MB),
  sessions.json 永不回拉(线上活跃 token)。⚠️ 只增/改、不删本地独有文件。
- tools/check_contracts_vs_table.py:按文件名解析单元/租客/期限与台账逐条比对,
  复用 import_contracts 的单元解析保证「挂上的行」与「核对的行」是同一行。
  支持 --pdf 抽正文,-v 出明细,--json 落盘。
parent fcf3187c
This diff is collapsed.
# -*- coding: utf-8 -*-
"""把线上 /opt/rentos/data 的数据**回拉**到本地 data/。
deploy_git.py 只管推代码;线上才是权威数据,本地只是预览,
所以需要一个反向通道把台账 / 名册 / 批次 / 合同索引拉回来对账。
用法
RENTOS_SSH_PASSWORD=... python tools/sync_pull.py # 全量(默认跳过附件)
RENTOS_SSH_PASSWORD=... python tools/sync_pull.py --with-attach # 连备注栏图片一起拉
RENTOS_SSH_PASSWORD=... python tools/sync_pull.py --dry # 只列差异不下载
RENTOS_SSH_PASSWORD=... python tools/sync_pull.py --only tables,config
RENTOS_SSH_PASSWORD=... python tools/sync_pull.py --no-backup
安全
- 拉之前先把本地 data/ 整体备份到 data/_snap/pull-<时间戳>/(除非 --no-backup)
- sessions.json 永不回拉:那是**线上活跃 token**,拉到本地没意义还碍事
- 只覆盖 data/ 下的内容,不碰仓库代码(代码走 deploy_git.py)
"""
import argparse
import io
import os
import shutil
import stat as statmod
import sys
import time
HERE = os.path.dirname(os.path.abspath(__file__))
BASE = os.path.dirname(HERE)
sys.path.insert(0, HERE)
import deploy_docker as D # noqa: E402 复用 SSH
REMOTE_DATA = D.REMOTE_DIR + "/data"
LOCAL_DATA = os.path.join(BASE, "data")
# 顶层条目白名单(按名字选,不按路径)
TOPS = ["tables", "batches", "contracts", "drafts", "seed", "attachments",
"config.json", "unit_map.json", "scheduler_state.json", "send_log.json",
"receipts.json", "users.json"]
# 永不回拉
SKIP_NAMES = {"sessions.json", "__pycache__", ".DS_Store", "Thumbs.db"}
SKIP_DIRS = {"__pycache__"}
DEFAULT_SKIP = {"attachments"} # 61MB 图片,默认不拉,--with-attach 才带上
def human(n):
for u in ("B", "KB", "MB", "GB"):
if n < 1024:
return "%.1f%s" % (n, u)
n /= 1024.0
return "%.1fTB" % n
def log(msg=""):
print(msg, flush=True)
def rlist(sf, path):
"""列一层,返回 (名字, 是否目录, 大小, mtime)。"""
out = []
try:
for a in sf.listdir_attr(path):
if a.filename in SKIP_NAMES or a.filename in SKIP_DIRS:
continue
out.append((a.filename, statmod.S_ISDIR(a.st_mode or 0),
int(a.st_size or 0), int(a.st_mtime or 0)))
except IOError:
pass
return sorted(out, key=lambda x: x[0])
def walk(sf, rpath, rel=""):
"""递归列出文件:[(相对路径, 大小, mtime)]。"""
files = []
for name, isdir, size, mt in rlist(sf, rpath):
r = (rel + "/" + name) if rel else name
if isdir:
files += walk(sf, rpath + "/" + name, r)
else:
files.append((r, size, mt))
return files
def backup_local():
if not os.path.isdir(LOCAL_DATA):
return None
dst = os.path.join(LOCAL_DATA, "_snap",
"pull-" + time.strftime("%Y%m%d-%H%M%S"))
os.makedirs(os.path.dirname(dst), exist_ok=True)
for name in sorted(os.listdir(LOCAL_DATA)):
if name == "_snap":
continue
src = os.path.join(LOCAL_DATA, name)
tgt = os.path.join(dst, name)
if os.path.isdir(src):
shutil.copytree(src, tgt, symlinks=False)
else:
shutil.copy2(src, tgt)
return dst
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--only", default="", help="只拉这些顶层条目,逗号分隔")
ap.add_argument("--skip", default="", help="额外跳过,逗号分隔")
ap.add_argument("--with-attach", action="store_true", help="连附件图片一起拉")
ap.add_argument("--dry", action="store_true", help="只列差异")
ap.add_argument("--no-backup", action="store_true", help="跳过本地备份")
args = ap.parse_args()
tops = [t.strip() for t in args.only.split(",") if t.strip()] or list(TOPS)
skip = set(DEFAULT_SKIP) | {s.strip() for s in args.skip.split(",") if s.strip()}
if args.with_attach:
skip.discard("attachments")
cli = D.connect()
sf = cli.open_sftp()
log("远端 %s:%s" % (D.HOST, REMOTE_DATA))
log("本地 %s" % LOCAL_DATA)
# ---- 收集远端文件
plan = []
for top in sorted(tops):
if top in skip:
log(" - 跳过 %s" % top)
continue
rp = REMOTE_DATA + "/" + top
try:
st = sf.stat(rp)
except IOError:
continue
if statmod.S_ISDIR(st.st_mode):
for rel, size, mt in walk(sf, rp, top):
plan.append((rel, size, mt))
else:
plan.append((top, int(st.st_size or 0), int(st.st_mtime or 0)))
total = sum(p[1] for p in plan)
log("")
log("远端待拉:%d 个文件 / %s" % (len(plan), human(total)))
# ---- 差异
NEW, DIFF, SAME = [], [], 0
for rel, size, mt in plan:
lp = os.path.join(LOCAL_DATA, rel.replace("/", os.sep))
if not os.path.exists(lp):
NEW.append((rel, size))
elif os.path.getsize(lp) != size:
DIFF.append((rel, os.path.getsize(lp), size))
else:
SAME += 1
log(" 新增 %d / 有变化 %d / 一致 %d" % (len(NEW), len(DIFF), SAME))
for rel, size in NEW[:200]:
log(" + %-58s %s" % (rel, human(size)))
for rel, lo, rs in DIFF[:200]:
log(" ~ %-58s 本地 %s → 远端 %s" % (rel, human(lo), human(rs)))
if args.dry:
sf.close()
cli.close()
log("\n(干跑,未下载)")
return 0
# ---- 备份
if not args.no_backup:
b = backup_local()
log("\n本地已备份 → %s" % (b or "(无 data/,跳过)"))
# ---- 下载
log("\n开始下载…")
done = 0
t0 = time.time()
for i, (rel, size, mt) in enumerate(plan, 1):
lp = os.path.join(LOCAL_DATA, rel.replace("/", os.sep))
os.makedirs(os.path.dirname(lp), exist_ok=True)
try:
sf.get(REMOTE_DATA + "/" + rel, lp)
done += 1
except Exception as e: # noqa: BLE001
log(" !! %s 失败:%s" % (rel, e))
if i % 40 == 0 or i == len(plan):
log(" %d/%d %.0fs" % (i, len(plan), time.time() - t0))
sf.close()
cli.close()
log("\n完成:%d/%d,用时 %.0fs" % (done, len(plan), time.time() - t0))
return 0
if __name__ == "__main__":
sys.exit(main())
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment