881 lines
25 KiB
Python
881 lines
25 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
CM93 Importer Normalization 接入点只读调查
|
||
Version: v12.0
|
||
|
||
用途
|
||
====
|
||
V12 不修改数据库，也不修改源码。
|
||
|
||
它只做两件事：
|
||
|
||
Gate A：
|
||
读取 V11 输出目录或 ZIP，要求：
|
||
IMPLEMENTATION PRECHECK PASS = YES
|
||
或 report JSON 中：
|
||
summary.implementation_precheck_pass = true
|
||
|
||
Gate B：
|
||
只读扫描当前 importer 项目源码，自动定位：
|
||
|
||
1. CM93 Feature / Area geometry 构造入口
|
||
2. chart_area_geometry / chart_coverage 写入位置
|
||
3. geometry_issue = "geometry missing" 或同义逻辑
|
||
4. geometry_invalid_count / geometry_repaired_count 更新位置
|
||
5. raw_geotype / geometry_type 判定位置
|
||
6. bbox_min_lon / bbox_min_lat / bbox_max_lon / bbox_max_lat 计算位置
|
||
7. Shapely / GEOS / WKB / Polygon / MultiPolygon 相关代码
|
||
8. chart_feature / chart_area_geometry / chart_coverage SQL 写入语句
|
||
9. raw_feature_blob / chart_raw_record 保留逻辑
|
||
10. importer job summary / counters
|
||
|
||
输出的目标是让下一阶段可以基于当前真实源码生成精准修改版，
|
||
而不是凭文件名或架构猜测。
|
||
|
||
重要：
|
||
- 不修改项目文件
|
||
- 不修改 MySQL
|
||
- 不开始 PBF
|
||
- 不生成自动 apply patch
|
||
- 只输出调查报告和代码片段
|
||
|
||
执行
|
||
====
|
||
假设当前就在 importer 项目根目录：
|
||
|
||
python .\\cm93_importer_normalization_source_audit_v12.py ^
|
||
--v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147" ^
|
||
--project-root "."
|
||
|
||
也可以给 V11 ZIP：
|
||
|
||
python .\\cm93_importer_normalization_source_audit_v12.py ^
|
||
--v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147.zip" ^
|
||
--project-root "."
|
||
|
||
输出
|
||
====
|
||
- importer_source_audit_summary_v12.txt
|
||
- importer_source_audit_report_v12.json
|
||
- importer_candidate_files_v12.csv
|
||
- importer_code_hits_v12.csv
|
||
- importer_sql_hits_v12.csv
|
||
- importer_function_inventory_v12.csv
|
||
- importer_dependency_inventory_v12.csv
|
||
- importer_normalization_integration_points_v12.md
|
||
- importer_source_audit_v12.zip
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import ast
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
import shutil
|
||
import sys
|
||
import tempfile
|
||
import zipfile
|
||
from collections import Counter, defaultdict
|
||
from datetime import datetime
|
||
from pathlib import Path
|
||
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Configuration
|
||
# ---------------------------------------------------------------------------
|
||
|
||
TEXT_EXTENSIONS = {
|
||
".py", ".sql", ".toml", ".ini", ".cfg", ".json",
|
||
".yaml", ".yml", ".md", ".txt",
|
||
}
|
||
|
||
SKIP_DIRS = {
|
||
".git", ".hg", ".svn",
|
||
"__pycache__", ".pytest_cache", ".mypy_cache",
|
||
".venv", "venv", "env",
|
||
"node_modules", "dist", "build",
|
||
".idea", ".vscode",
|
||
}
|
||
|
||
SEARCH_GROUPS = {
|
||
"NORMALIZATION": [
|
||
r"\bmake_valid\b",
|
||
r"\bpolygonize_full\b",
|
||
r"\bunary_union\b",
|
||
r"\bnormalize\b",
|
||
r"\bMultiPolygon\b",
|
||
r"\bPolygon\b",
|
||
r"\bShapely\b",
|
||
r"\bshapely\b",
|
||
r"\bWKB\b",
|
||
r"\bwkb\b",
|
||
],
|
||
"GEOMETRY_ISSUE": [
|
||
r"geometry_issue",
|
||
r"geometry missing",
|
||
r"geometry_missing",
|
||
r"invalid geometry",
|
||
r"geometry_invalid",
|
||
r"geometry_repaired",
|
||
],
|
||
"GEOTYPE": [
|
||
r"raw_geotype",
|
||
r"geotype",
|
||
r"geometry_type",
|
||
r"\bMETA\b",
|
||
r"\bAREA\b",
|
||
],
|
||
"BBOX": [
|
||
r"bbox_min_lon",
|
||
r"bbox_min_lat",
|
||
r"bbox_max_lon",
|
||
r"bbox_max_lat",
|
||
r"\bbounds\b",
|
||
r"\bbbox\b",
|
||
],
|
||
"COUNTERS": [
|
||
r"geometry_invalid_count",
|
||
r"geometry_repaired_count",
|
||
r"invalid_count",
|
||
r"repaired_count",
|
||
r"summary_json",
|
||
r"chart_import_job",
|
||
],
|
||
"DB_WRITE": [
|
||
r"chart_area_geometry",
|
||
r"chart_coverage",
|
||
r"chart_feature",
|
||
r"INSERT\s+INTO",
|
||
r"UPDATE\s+",
|
||
r"executemany",
|
||
r"execute\(",
|
||
],
|
||
"RAW_SOURCE": [
|
||
r"raw_feature_blob",
|
||
r"chart_raw_record",
|
||
r"source_record_index",
|
||
r"raw_blob",
|
||
r"raw_metadata_json",
|
||
],
|
||
}
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Helpers
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def write_csv(
|
||
path: Path,
|
||
rows: Iterable[Dict[str, Any]],
|
||
fieldnames: Optional[List[str]] = None,
|
||
):
|
||
rows = list(rows)
|
||
|
||
if fieldnames is None:
|
||
seen = set()
|
||
fieldnames = []
|
||
for row in rows:
|
||
for k in row.keys():
|
||
if k not in seen:
|
||
seen.add(k)
|
||
fieldnames.append(k)
|
||
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
||
if not fieldnames:
|
||
return
|
||
w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore")
|
||
w.writeheader()
|
||
for row in rows:
|
||
out = {}
|
||
for k in fieldnames:
|
||
v = row.get(k)
|
||
if isinstance(v, (dict, list, tuple)):
|
||
v = json.dumps(v, ensure_ascii=False, default=str)
|
||
out[k] = "" if v is None else v
|
||
w.writerow(out)
|
||
|
||
|
||
def safe_read_text(path: Path) -> Optional[str]:
|
||
for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"):
|
||
try:
|
||
return path.read_text(encoding=enc)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
|
||
def relpath(path: Path, root: Path) -> str:
|
||
try:
|
||
return str(path.resolve().relative_to(root.resolve()))
|
||
except Exception:
|
||
return str(path)
|
||
|
||
|
||
def should_skip(path: Path) -> bool:
|
||
return any(part in SKIP_DIRS for part in path.parts)
|
||
|
||
|
||
def list_source_files(root: Path) -> List[Path]:
|
||
out = []
|
||
for p in root.rglob("*"):
|
||
if not p.is_file():
|
||
continue
|
||
if should_skip(p):
|
||
continue
|
||
if p.suffix.lower() in TEXT_EXTENSIONS:
|
||
out.append(p)
|
||
return sorted(out)
|
||
|
||
|
||
def open_input(src: Path) -> Tuple[Path, Optional[Path]]:
|
||
if src.is_dir():
|
||
return src, None
|
||
|
||
if src.is_file() and src.suffix.lower() == ".zip":
|
||
temp = Path(tempfile.mkdtemp(prefix="cm93_v12_v11_"))
|
||
with zipfile.ZipFile(src, "r") as z:
|
||
z.extractall(temp)
|
||
return temp, temp
|
||
|
||
raise ValueError("V11 输入必须是目录或 ZIP")
|
||
|
||
|
||
def find_file(root: Path, filename: str) -> Optional[Path]:
|
||
direct = root / filename
|
||
if direct.exists():
|
||
return direct
|
||
hits = list(root.rglob(filename))
|
||
return hits[0] if hits else None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# V11 gate
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def load_v11_gate(root: Path) -> Dict[str, Any]:
|
||
result = {
|
||
"source": None,
|
||
"implementation_precheck_pass": False,
|
||
"db_invalid_count": None,
|
||
"golden_revalidation_pass_count": None,
|
||
"golden_revalidation_fail_count": None,
|
||
"area_geometry_column_type": None,
|
||
"coverage_geometry_column_type": None,
|
||
"future_area_bbox_update_needed_count": None,
|
||
"raw_geotype16_meta_count": None,
|
||
"raw_geotype16_with_issue_count": None,
|
||
"total_feature_geometry_issue_count": None,
|
||
"import_job_geometry_invalid_sum": None,
|
||
"errors": [],
|
||
}
|
||
|
||
report_path = find_file(root, "implementation_precheck_report_v11.json")
|
||
summary_path = find_file(root, "implementation_precheck_summary_v11.txt")
|
||
|
||
if report_path:
|
||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||
s = report.get("summary", {})
|
||
result["source"] = "implementation_precheck_report_v11.json"
|
||
result["implementation_precheck_pass"] = bool(
|
||
s.get("implementation_precheck_pass")
|
||
)
|
||
for key in (
|
||
"db_invalid_count",
|
||
"golden_revalidation_pass_count",
|
||
"golden_revalidation_fail_count",
|
||
"area_geometry_column_type",
|
||
"coverage_geometry_column_type",
|
||
"future_area_bbox_update_needed_count",
|
||
"raw_geotype16_meta_count",
|
||
"raw_geotype16_with_issue_count",
|
||
"total_feature_geometry_issue_count",
|
||
"import_job_geometry_invalid_sum",
|
||
):
|
||
result[key] = s.get(key)
|
||
|
||
elif summary_path:
|
||
text = summary_path.read_text(encoding="utf-8", errors="replace")
|
||
result["source"] = "implementation_precheck_summary_v11.txt"
|
||
result["implementation_precheck_pass"] = bool(
|
||
re.search(
|
||
r"IMPLEMENTATION\s+PRECHECK\s+PASS:\s*YES",
|
||
text,
|
||
flags=re.I,
|
||
)
|
||
)
|
||
|
||
else:
|
||
result["errors"].append(
|
||
"找不到 implementation_precheck_report_v11.json "
|
||
"或 implementation_precheck_summary_v11.txt"
|
||
)
|
||
|
||
return result
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Source analysis
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def scan_hits(
|
||
path: Path,
|
||
text: str,
|
||
root: Path,
|
||
) -> List[Dict[str, Any]]:
|
||
rows = []
|
||
lines = text.splitlines()
|
||
|
||
compiled = {
|
||
group: [re.compile(pat, re.I) for pat in patterns]
|
||
for group, patterns in SEARCH_GROUPS.items()
|
||
}
|
||
|
||
for lineno, line in enumerate(lines, start=1):
|
||
stripped = line.strip()
|
||
|
||
for group, patterns in compiled.items():
|
||
matched = []
|
||
for pat in patterns:
|
||
if pat.search(line):
|
||
matched.append(pat.pattern)
|
||
|
||
if not matched:
|
||
continue
|
||
|
||
start = max(1, lineno - 2)
|
||
end = min(len(lines), lineno + 2)
|
||
context = "\n".join(
|
||
f"{n:05d}: {lines[n-1]}"
|
||
for n in range(start, end + 1)
|
||
)
|
||
|
||
rows.append({
|
||
"file": relpath(path, root),
|
||
"line": lineno,
|
||
"group": group,
|
||
"matched_patterns": matched,
|
||
"code": stripped,
|
||
"context": context,
|
||
})
|
||
|
||
return rows
|
||
|
||
|
||
def python_function_inventory(
|
||
path: Path,
|
||
text: str,
|
||
root: Path,
|
||
) -> List[Dict[str, Any]]:
|
||
if path.suffix.lower() != ".py":
|
||
return []
|
||
|
||
try:
|
||
tree = ast.parse(text)
|
||
except Exception:
|
||
return []
|
||
|
||
rows = []
|
||
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
|
||
rows.append({
|
||
"file": relpath(path, root),
|
||
"name": node.name,
|
||
"line": node.lineno,
|
||
"end_line": getattr(node, "end_lineno", None),
|
||
"async": int(isinstance(node, ast.AsyncFunctionDef)),
|
||
"args": [
|
||
arg.arg
|
||
for arg in (
|
||
list(node.args.posonlyargs)
|
||
+ list(node.args.args)
|
||
+ list(node.args.kwonlyargs)
|
||
)
|
||
],
|
||
})
|
||
|
||
return rows
|
||
|
||
|
||
def python_dependency_inventory(
|
||
path: Path,
|
||
text: str,
|
||
root: Path,
|
||
) -> List[Dict[str, Any]]:
|
||
if path.suffix.lower() != ".py":
|
||
return []
|
||
|
||
try:
|
||
tree = ast.parse(text)
|
||
except Exception:
|
||
return []
|
||
|
||
rows = []
|
||
|
||
for node in ast.walk(tree):
|
||
if isinstance(node, ast.Import):
|
||
for alias in node.names:
|
||
rows.append({
|
||
"file": relpath(path, root),
|
||
"line": node.lineno,
|
||
"module": alias.name,
|
||
"import_type": "import",
|
||
})
|
||
|
||
elif isinstance(node, ast.ImportFrom):
|
||
rows.append({
|
||
"file": relpath(path, root),
|
||
"line": node.lineno,
|
||
"module": node.module or "",
|
||
"import_type": "from",
|
||
"names": [x.name for x in node.names],
|
||
})
|
||
|
||
return rows
|
||
|
||
|
||
def candidate_score(
|
||
path: Path,
|
||
hits: List[Dict[str, Any]],
|
||
root: Path,
|
||
) -> Dict[str, Any]:
|
||
groups = Counter(r["group"] for r in hits)
|
||
score = 0
|
||
|
||
weights = {
|
||
"GEOMETRY_ISSUE": 8,
|
||
"GEOTYPE": 5,
|
||
"BBOX": 5,
|
||
"COUNTERS": 7,
|
||
"DB_WRITE": 7,
|
||
"RAW_SOURCE": 4,
|
||
"NORMALIZATION": 3,
|
||
}
|
||
|
||
for group, count in groups.items():
|
||
score += weights.get(group, 1) * min(count, 10)
|
||
|
||
name = path.name.lower()
|
||
if "import" in name:
|
||
score += 8
|
||
if "geometry" in name or "geom" in name:
|
||
score += 6
|
||
if "cm93" in name:
|
||
score += 6
|
||
if "db" in name or "mysql" in name:
|
||
score += 3
|
||
|
||
return {
|
||
"file": relpath(path, root),
|
||
"score": score,
|
||
"hit_count": len(hits),
|
||
**{f"hits_{g.lower()}": groups.get(g, 0) for g in SEARCH_GROUPS},
|
||
}
|
||
|
||
|
||
def choose_integration_points(
|
||
hits: List[Dict[str, Any]]
|
||
) -> Dict[str, List[Dict[str, Any]]]:
|
||
buckets = {
|
||
"area_geometry_build_or_write": [],
|
||
"coverage_geometry_build_or_write": [],
|
||
"geotype16_issue_logic": [],
|
||
"geometry_invalid_counter": [],
|
||
"bbox_update": [],
|
||
"raw_source_preservation": [],
|
||
}
|
||
|
||
for r in hits:
|
||
code = r["code"].lower()
|
||
file = r["file"].lower()
|
||
|
||
if (
|
||
"chart_area_geometry" in code
|
||
or (
|
||
r["group"] == "NORMALIZATION"
|
||
and "area" in file
|
||
)
|
||
):
|
||
buckets["area_geometry_build_or_write"].append(r)
|
||
|
||
if "chart_coverage" in code:
|
||
buckets["coverage_geometry_build_or_write"].append(r)
|
||
|
||
if (
|
||
"raw_geotype" in code
|
||
or "geotype" in code
|
||
or "geometry missing" in code
|
||
):
|
||
buckets["geotype16_issue_logic"].append(r)
|
||
|
||
if "geometry_invalid_count" in code:
|
||
buckets["geometry_invalid_counter"].append(r)
|
||
|
||
if (
|
||
"bbox_min_lon" in code
|
||
or "bbox_max_lon" in code
|
||
or ".bounds" in code
|
||
):
|
||
buckets["bbox_update"].append(r)
|
||
|
||
if (
|
||
"raw_feature_blob" in code
|
||
or "chart_raw_record" in code
|
||
):
|
||
buckets["raw_source_preservation"].append(r)
|
||
|
||
for key in buckets:
|
||
buckets[key] = buckets[key][:30]
|
||
|
||
return buckets
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Report
|
||
# ---------------------------------------------------------------------------
|
||
|
||
def build_integration_md(
|
||
gate: Dict[str, Any],
|
||
candidates: List[Dict[str, Any]],
|
||
integration: Dict[str, List[Dict[str, Any]]],
|
||
) -> str:
|
||
lines = []
|
||
|
||
lines.append("# CM93 Importer Normalization 接入点调查 v12")
|
||
lines.append("")
|
||
lines.append("## V11 门控")
|
||
lines.append("")
|
||
lines.append(
|
||
f"- IMPLEMENTATION PRECHECK："
|
||
f"{'PASS' if gate['implementation_precheck_pass'] else 'FAIL'}"
|
||
)
|
||
|
||
for key in (
|
||
"db_invalid_count",
|
||
"golden_revalidation_pass_count",
|
||
"golden_revalidation_fail_count",
|
||
"area_geometry_column_type",
|
||
"coverage_geometry_column_type",
|
||
"future_area_bbox_update_needed_count",
|
||
"raw_geotype16_meta_count",
|
||
"raw_geotype16_with_issue_count",
|
||
"import_job_geometry_invalid_sum",
|
||
):
|
||
lines.append(f"- {key}: `{gate.get(key)}`")
|
||
|
||
lines.append("")
|
||
lines.append("## 高优先级候选源码文件")
|
||
lines.append("")
|
||
|
||
for row in candidates[:15]:
|
||
lines.append(
|
||
f"- `{row['file']}` — score={row['score']}, "
|
||
f"hits={row['hit_count']}"
|
||
)
|
||
|
||
title_map = {
|
||
"area_geometry_build_or_write": "Area geometry 构造/写入",
|
||
"coverage_geometry_build_or_write": "Coverage geometry 构造/写入",
|
||
"geotype16_issue_logic": "raw_geotype=16 / geometry missing 判定",
|
||
"geometry_invalid_counter": "geometry_invalid_count 统计",
|
||
"bbox_update": "bbox 计算/写入",
|
||
"raw_source_preservation": "raw source 保留",
|
||
}
|
||
|
||
for key, title in title_map.items():
|
||
lines.append("")
|
||
lines.append(f"## {title}")
|
||
lines.append("")
|
||
|
||
rows = integration.get(key, [])
|
||
|
||
if not rows:
|
||
lines.append("- 未自动定位，需要人工继续查。")
|
||
continue
|
||
|
||
seen = set()
|
||
for row in rows:
|
||
marker = (row["file"], row["line"])
|
||
if marker in seen:
|
||
continue
|
||
seen.add(marker)
|
||
lines.append(
|
||
f"- `{row['file']}:{row['line']}` "
|
||
f"`{row['code']}`"
|
||
)
|
||
|
||
lines.append("")
|
||
lines.append("## 下一阶段修改边界")
|
||
lines.append("")
|
||
lines.append("下一阶段只应修改真实定位到的 importer 代码，不应重新设计数据库。")
|
||
lines.append("")
|
||
lines.append("冻结要求：")
|
||
lines.append("")
|
||
lines.append("1. valid geometry 原样 PASS-THROUGH。")
|
||
lines.append("2. invalid Area/Coverage 才执行 odd-even face selection。")
|
||
lines.append("3. normalization 后执行 GEOS normalize canonicalization。")
|
||
lines.append("4. 输出必须为 valid Polygon/MultiPolygon。")
|
||
lines.append("5. Area normalized 后同步重新计算 feature bbox。")
|
||
lines.append("6. retraced connector 不保存为 Area 面积，但 raw source 必须保留。")
|
||
lines.append("7. raw_geotype=16 / META 不生成 `geometry missing`。")
|
||
lines.append("8. raw_geotype=16 不增加 `geometry_invalid_count`。")
|
||
lines.append("9. 不删除 `raw_feature_blob` 或 `chart_raw_record`。")
|
||
lines.append("10. 当前仍不进入 PBF。")
|
||
|
||
return "\n".join(lines)
|
||
|
||
|
||
def parse_args():
|
||
p = argparse.ArgumentParser(
|
||
description="CM93 importer normalization source audit v12"
|
||
)
|
||
p.add_argument("--v11-input", required=True)
|
||
p.add_argument("--project-root", default=".")
|
||
p.add_argument("--output", default=None)
|
||
return p.parse_args()
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
|
||
v11_src = Path(args.v11_input).expanduser().resolve()
|
||
project_root = Path(args.project_root).expanduser().resolve()
|
||
|
||
if not v11_src.exists():
|
||
print(f"V11 输入不存在：{v11_src}")
|
||
sys.exit(2)
|
||
|
||
if not project_root.is_dir():
|
||
print(f"项目目录不存在：{project_root}")
|
||
sys.exit(2)
|
||
|
||
temp_v11 = None
|
||
|
||
try:
|
||
v11_root, temp_v11 = open_input(v11_src)
|
||
|
||
print("[1/6] V11 implementation precheck gate ...")
|
||
gate = load_v11_gate(v11_root)
|
||
|
||
print(json.dumps(gate, ensure_ascii=False, indent=2))
|
||
|
||
if not gate["implementation_precheck_pass"]:
|
||
print("")
|
||
print("STOP：V11 IMPLEMENTATION PRECHECK 未通过。")
|
||
print("V12 不扫描 importer 源码。")
|
||
sys.exit(3)
|
||
|
||
print(" V11 gate: PASS")
|
||
|
||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
|
||
outdir = Path(
|
||
args.output
|
||
or project_root / f"cm93_importer_source_audit_v12_{stamp}"
|
||
)
|
||
outdir.mkdir(parents=True, exist_ok=True)
|
||
|
||
print("[2/6] 扫描项目文本源码 ...")
|
||
files = list_source_files(project_root)
|
||
print(f" files={len(files)}")
|
||
|
||
all_hits = []
|
||
function_rows = []
|
||
dependency_rows = []
|
||
candidate_rows = []
|
||
|
||
for i, path in enumerate(files, start=1):
|
||
# 避免扫描当前输出目录自身
|
||
try:
|
||
path.resolve().relative_to(outdir.resolve())
|
||
continue
|
||
except Exception:
|
||
pass
|
||
|
||
text = safe_read_text(path)
|
||
if text is None:
|
||
continue
|
||
|
||
hits = scan_hits(path, text, project_root)
|
||
all_hits.extend(hits)
|
||
|
||
function_rows.extend(
|
||
python_function_inventory(path, text, project_root)
|
||
)
|
||
|
||
dependency_rows.extend(
|
||
python_dependency_inventory(path, text, project_root)
|
||
)
|
||
|
||
if hits:
|
||
candidate_rows.append(
|
||
candidate_score(path, hits, project_root)
|
||
)
|
||
|
||
if i % 100 == 0:
|
||
print(f" scanned={i}/{len(files)}")
|
||
|
||
candidate_rows.sort(
|
||
key=lambda r: (-int(r["score"]), r["file"])
|
||
)
|
||
|
||
print("[3/6] 分类 SQL 命中与函数 ...")
|
||
sql_hits = [
|
||
r for r in all_hits
|
||
if (
|
||
r["group"] == "DB_WRITE"
|
||
or re.search(
|
||
r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b",
|
||
r["code"],
|
||
re.I,
|
||
)
|
||
)
|
||
]
|
||
|
||
print("[4/6] 推导 normalization 接入点 ...")
|
||
integration = choose_integration_points(all_hits)
|
||
|
||
print("[5/6] 写报告 ...")
|
||
write_csv(
|
||
outdir / "importer_candidate_files_v12.csv",
|
||
candidate_rows
|
||
)
|
||
write_csv(
|
||
outdir / "importer_code_hits_v12.csv",
|
||
all_hits
|
||
)
|
||
write_csv(
|
||
outdir / "importer_sql_hits_v12.csv",
|
||
sql_hits
|
||
)
|
||
write_csv(
|
||
outdir / "importer_function_inventory_v12.csv",
|
||
function_rows
|
||
)
|
||
write_csv(
|
||
outdir / "importer_dependency_inventory_v12.csv",
|
||
dependency_rows
|
||
)
|
||
|
||
integration_md = build_integration_md(
|
||
gate,
|
||
candidate_rows,
|
||
integration,
|
||
)
|
||
|
||
(
|
||
outdir / "importer_normalization_integration_points_v12.md"
|
||
).write_text(
|
||
integration_md,
|
||
encoding="utf-8"
|
||
)
|
||
|
||
report = {
|
||
"version": "v12.0",
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"project_root": str(project_root),
|
||
"read_only": True,
|
||
"v11_gate": gate,
|
||
"source_file_count": len(files),
|
||
"candidate_file_count": len(candidate_rows),
|
||
"code_hit_count": len(all_hits),
|
||
"sql_hit_count": len(sql_hits),
|
||
"function_count": len(function_rows),
|
||
"dependency_count": len(dependency_rows),
|
||
"top_candidate_files": candidate_rows[:25],
|
||
"integration_point_counts": {
|
||
k: len(v)
|
||
for k, v in integration.items()
|
||
},
|
||
}
|
||
|
||
(
|
||
outdir / "importer_source_audit_report_v12.json"
|
||
).write_text(
|
||
json.dumps(
|
||
report,
|
||
ensure_ascii=False,
|
||
indent=2,
|
||
default=str,
|
||
),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
lines = []
|
||
lines.append("CM93 Importer Normalization 接入点只读调查 v12")
|
||
lines.append("=" * 88)
|
||
lines.append(f"Generated: {report['generated_at']}")
|
||
lines.append(f"Project: {project_root}")
|
||
lines.append("")
|
||
lines.append("V11 GATE")
|
||
lines.append(" PASS")
|
||
lines.append("")
|
||
lines.append("SOURCE SCAN")
|
||
lines.append(f" files: {len(files)}")
|
||
lines.append(f" candidate files:{len(candidate_rows)}")
|
||
lines.append(f" code hits: {len(all_hits)}")
|
||
lines.append(f" SQL hits: {len(sql_hits)}")
|
||
lines.append(f" functions: {len(function_rows)}")
|
||
lines.append("")
|
||
lines.append("TOP CANDIDATE FILES")
|
||
|
||
for row in candidate_rows[:15]:
|
||
lines.append(
|
||
f" {row['score']:4d} {row['file']}"
|
||
)
|
||
|
||
lines.append("")
|
||
lines.append("INTEGRATION POINT COUNTS")
|
||
|
||
for key, rows in integration.items():
|
||
lines.append(
|
||
f" {key:36s} {len(rows)}"
|
||
)
|
||
|
||
lines.append("")
|
||
lines.append(
|
||
"NO SOURCE WRITE / NO DATABASE WRITE / NO PBF"
|
||
)
|
||
|
||
(
|
||
outdir / "importer_source_audit_summary_v12.txt"
|
||
).write_text(
|
||
"\n".join(lines),
|
||
encoding="utf-8"
|
||
)
|
||
|
||
print("[6/6] 压缩 ...")
|
||
zpath = outdir.with_suffix(".zip")
|
||
|
||
with zipfile.ZipFile(
|
||
zpath,
|
||
"w",
|
||
compression=zipfile.ZIP_DEFLATED
|
||
) as z:
|
||
for p in sorted(outdir.rglob("*")):
|
||
if p.is_file():
|
||
z.write(
|
||
p,
|
||
p.relative_to(outdir.parent)
|
||
)
|
||
|
||
print("")
|
||
print("SOURCE AUDIT PASS: YES")
|
||
print(f"ZIP: {zpath.resolve()}")
|
||
print("把 V12 ZIP 上传给 ChatGPT。")
|
||
|
||
finally:
|
||
if temp_v11 and temp_v11.exists():
|
||
shutil.rmtree(temp_v11, ignore_errors=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|