#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ CM93 Importer Normalization 接入点只读调查 Version: v12.0 用途 ==== V12 不修改数据库，也不修改源码。 它只做两件事： Gate A： 读取 V11 输出目录或 ZIP，要求： IMPLEMENTATION PRECHECK PASS = YES 或 report JSON 中： summary.implementation_precheck_pass = true Gate B： 只读扫描当前 importer 项目源码，自动定位： 1. CM93 Feature / Area geometry 构造入口 2. chart_area_geometry / chart_coverage 写入位置 3. geometry_issue = "geometry missing" 或同义逻辑 4. geometry_invalid_count / geometry_repaired_count 更新位置 5. raw_geotype / geometry_type 判定位置 6. bbox_min_lon / bbox_min_lat / bbox_max_lon / bbox_max_lat 计算位置 7. Shapely / GEOS / WKB / Polygon / MultiPolygon 相关代码 8. chart_feature / chart_area_geometry / chart_coverage SQL 写入语句 9. raw_feature_blob / chart_raw_record 保留逻辑 10. importer job summary / counters 输出的目标是让下一阶段可以基于当前真实源码生成精准修改版， 而不是凭文件名或架构猜测。 重要： - 不修改项目文件 - 不修改 MySQL - 不开始 PBF - 不生成自动 apply patch - 只输出调查报告和代码片段 执行 ==== 假设当前就在 importer 项目根目录： python .\\cm93_importer_normalization_source_audit_v12.py ^ --v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147" ^ --project-root "." 也可以给 V11 ZIP： python .\\cm93_importer_normalization_source_audit_v12.py ^ --v11-input ".\\cm93_normalization_implementation_precheck_v11_dataset_1_20260901_084147.zip" ^ --project-root "." 输出 ==== - importer_source_audit_summary_v12.txt - importer_source_audit_report_v12.json - importer_candidate_files_v12.csv - importer_code_hits_v12.csv - importer_sql_hits_v12.csv - importer_function_inventory_v12.csv - importer_dependency_inventory_v12.csv - importer_normalization_integration_points_v12.md - importer_source_audit_v12.zip """ from __future__ import annotations import argparse import ast import csv import json import os import re import shutil import sys import tempfile import zipfile from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple # --------------------------------------------------------------------------- # Configuration # --------------------------------------------------------------------------- TEXT_EXTENSIONS = { ".py", ".sql", ".toml", ".ini", ".cfg", ".json", ".yaml", ".yml", ".md", ".txt", } SKIP_DIRS = { ".git", ".hg", ".svn", "__pycache__", ".pytest_cache", ".mypy_cache", ".venv", "venv", "env", "node_modules", "dist", "build", ".idea", ".vscode", } SEARCH_GROUPS = { "NORMALIZATION": [ r"\bmake_valid\b", r"\bpolygonize_full\b", r"\bunary_union\b", r"\bnormalize\b", r"\bMultiPolygon\b", r"\bPolygon\b", r"\bShapely\b", r"\bshapely\b", r"\bWKB\b", r"\bwkb\b", ], "GEOMETRY_ISSUE": [ r"geometry_issue", r"geometry missing", r"geometry_missing", r"invalid geometry", r"geometry_invalid", r"geometry_repaired", ], "GEOTYPE": [ r"raw_geotype", r"geotype", r"geometry_type", r"\bMETA\b", r"\bAREA\b", ], "BBOX": [ r"bbox_min_lon", r"bbox_min_lat", r"bbox_max_lon", r"bbox_max_lat", r"\bbounds\b", r"\bbbox\b", ], "COUNTERS": [ r"geometry_invalid_count", r"geometry_repaired_count", r"invalid_count", r"repaired_count", r"summary_json", r"chart_import_job", ], "DB_WRITE": [ r"chart_area_geometry", r"chart_coverage", r"chart_feature", r"INSERT\s+INTO", r"UPDATE\s+", r"executemany", r"execute\(", ], "RAW_SOURCE": [ r"raw_feature_blob", r"chart_raw_record", r"source_record_index", r"raw_blob", r"raw_metadata_json", ], } # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def write_csv( path: Path, rows: Iterable[Dict[str, Any]], fieldnames: Optional[List[str]] = None, ): rows = list(rows) if fieldnames is None: seen = set() fieldnames = [] for row in rows: for k in row.keys(): if k not in seen: seen.add(k) fieldnames.append(k) path.parent.mkdir(parents=True, exist_ok=True) with path.open("w", encoding="utf-8-sig", newline="") as f: if not fieldnames: return w = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") w.writeheader() for row in rows: out = {} for k in fieldnames: v = row.get(k) if isinstance(v, (dict, list, tuple)): v = json.dumps(v, ensure_ascii=False, default=str) out[k] = "" if v is None else v w.writerow(out) def safe_read_text(path: Path) -> Optional[str]: for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"): try: return path.read_text(encoding=enc) except Exception: pass return None def relpath(path: Path, root: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) except Exception: return str(path) def should_skip(path: Path) -> bool: return any(part in SKIP_DIRS for part in path.parts) def list_source_files(root: Path) -> List[Path]: out = [] for p in root.rglob("*"): if not p.is_file(): continue if should_skip(p): continue if p.suffix.lower() in TEXT_EXTENSIONS: out.append(p) return sorted(out) def open_input(src: Path) -> Tuple[Path, Optional[Path]]: if src.is_dir(): return src, None if src.is_file() and src.suffix.lower() == ".zip": temp = Path(tempfile.mkdtemp(prefix="cm93_v12_v11_")) with zipfile.ZipFile(src, "r") as z: z.extractall(temp) return temp, temp raise ValueError("V11 输入必须是目录或 ZIP") def find_file(root: Path, filename: str) -> Optional[Path]: direct = root / filename if direct.exists(): return direct hits = list(root.rglob(filename)) return hits[0] if hits else None # --------------------------------------------------------------------------- # V11 gate # --------------------------------------------------------------------------- def load_v11_gate(root: Path) -> Dict[str, Any]: result = { "source": None, "implementation_precheck_pass": False, "db_invalid_count": None, "golden_revalidation_pass_count": None, "golden_revalidation_fail_count": None, "area_geometry_column_type": None, "coverage_geometry_column_type": None, "future_area_bbox_update_needed_count": None, "raw_geotype16_meta_count": None, "raw_geotype16_with_issue_count": None, "total_feature_geometry_issue_count": None, "import_job_geometry_invalid_sum": None, "errors": [], } report_path = find_file(root, "implementation_precheck_report_v11.json") summary_path = find_file(root, "implementation_precheck_summary_v11.txt") if report_path: report = json.loads(report_path.read_text(encoding="utf-8")) s = report.get("summary", {}) result["source"] = "implementation_precheck_report_v11.json" result["implementation_precheck_pass"] = bool( s.get("implementation_precheck_pass") ) for key in ( "db_invalid_count", "golden_revalidation_pass_count", "golden_revalidation_fail_count", "area_geometry_column_type", "coverage_geometry_column_type", "future_area_bbox_update_needed_count", "raw_geotype16_meta_count", "raw_geotype16_with_issue_count", "total_feature_geometry_issue_count", "import_job_geometry_invalid_sum", ): result[key] = s.get(key) elif summary_path: text = summary_path.read_text(encoding="utf-8", errors="replace") result["source"] = "implementation_precheck_summary_v11.txt" result["implementation_precheck_pass"] = bool( re.search( r"IMPLEMENTATION\s+PRECHECK\s+PASS:\s*YES", text, flags=re.I, ) ) else: result["errors"].append( "找不到 implementation_precheck_report_v11.json " "或 implementation_precheck_summary_v11.txt" ) return result # --------------------------------------------------------------------------- # Source analysis # --------------------------------------------------------------------------- def scan_hits( path: Path, text: str, root: Path, ) -> List[Dict[str, Any]]: rows = [] lines = text.splitlines() compiled = { group: [re.compile(pat, re.I) for pat in patterns] for group, patterns in SEARCH_GROUPS.items() } for lineno, line in enumerate(lines, start=1): stripped = line.strip() for group, patterns in compiled.items(): matched = [] for pat in patterns: if pat.search(line): matched.append(pat.pattern) if not matched: continue start = max(1, lineno - 2) end = min(len(lines), lineno + 2) context = "\n".join( f"{n:05d}: {lines[n-1]}" for n in range(start, end + 1) ) rows.append({ "file": relpath(path, root), "line": lineno, "group": group, "matched_patterns": matched, "code": stripped, "context": context, }) return rows def python_function_inventory( path: Path, text: str, root: Path, ) -> List[Dict[str, Any]]: if path.suffix.lower() != ".py": return [] try: tree = ast.parse(text) except Exception: return [] rows = [] for node in ast.walk(tree): if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): rows.append({ "file": relpath(path, root), "name": node.name, "line": node.lineno, "end_line": getattr(node, "end_lineno", None), "async": int(isinstance(node, ast.AsyncFunctionDef)), "args": [ arg.arg for arg in ( list(node.args.posonlyargs) + list(node.args.args) + list(node.args.kwonlyargs) ) ], }) return rows def python_dependency_inventory( path: Path, text: str, root: Path, ) -> List[Dict[str, Any]]: if path.suffix.lower() != ".py": return [] try: tree = ast.parse(text) except Exception: return [] rows = [] for node in ast.walk(tree): if isinstance(node, ast.Import): for alias in node.names: rows.append({ "file": relpath(path, root), "line": node.lineno, "module": alias.name, "import_type": "import", }) elif isinstance(node, ast.ImportFrom): rows.append({ "file": relpath(path, root), "line": node.lineno, "module": node.module or "", "import_type": "from", "names": [x.name for x in node.names], }) return rows def candidate_score( path: Path, hits: List[Dict[str, Any]], root: Path, ) -> Dict[str, Any]: groups = Counter(r["group"] for r in hits) score = 0 weights = { "GEOMETRY_ISSUE": 8, "GEOTYPE": 5, "BBOX": 5, "COUNTERS": 7, "DB_WRITE": 7, "RAW_SOURCE": 4, "NORMALIZATION": 3, } for group, count in groups.items(): score += weights.get(group, 1) * min(count, 10) name = path.name.lower() if "import" in name: score += 8 if "geometry" in name or "geom" in name: score += 6 if "cm93" in name: score += 6 if "db" in name or "mysql" in name: score += 3 return { "file": relpath(path, root), "score": score, "hit_count": len(hits), **{f"hits_{g.lower()}": groups.get(g, 0) for g in SEARCH_GROUPS}, } def choose_integration_points( hits: List[Dict[str, Any]] ) -> Dict[str, List[Dict[str, Any]]]: buckets = { "area_geometry_build_or_write": [], "coverage_geometry_build_or_write": [], "geotype16_issue_logic": [], "geometry_invalid_counter": [], "bbox_update": [], "raw_source_preservation": [], } for r in hits: code = r["code"].lower() file = r["file"].lower() if ( "chart_area_geometry" in code or ( r["group"] == "NORMALIZATION" and "area" in file ) ): buckets["area_geometry_build_or_write"].append(r) if "chart_coverage" in code: buckets["coverage_geometry_build_or_write"].append(r) if ( "raw_geotype" in code or "geotype" in code or "geometry missing" in code ): buckets["geotype16_issue_logic"].append(r) if "geometry_invalid_count" in code: buckets["geometry_invalid_counter"].append(r) if ( "bbox_min_lon" in code or "bbox_max_lon" in code or ".bounds" in code ): buckets["bbox_update"].append(r) if ( "raw_feature_blob" in code or "chart_raw_record" in code ): buckets["raw_source_preservation"].append(r) for key in buckets: buckets[key] = buckets[key][:30] return buckets # --------------------------------------------------------------------------- # Report # --------------------------------------------------------------------------- def build_integration_md( gate: Dict[str, Any], candidates: List[Dict[str, Any]], integration: Dict[str, List[Dict[str, Any]]], ) -> str: lines = [] lines.append("# CM93 Importer Normalization 接入点调查 v12") lines.append("") lines.append("## V11 门控") lines.append("") lines.append( f"- IMPLEMENTATION PRECHECK：" f"{'PASS' if gate['implementation_precheck_pass'] else 'FAIL'}" ) for key in ( "db_invalid_count", "golden_revalidation_pass_count", "golden_revalidation_fail_count", "area_geometry_column_type", "coverage_geometry_column_type", "future_area_bbox_update_needed_count", "raw_geotype16_meta_count", "raw_geotype16_with_issue_count", "import_job_geometry_invalid_sum", ): lines.append(f"- {key}: `{gate.get(key)}`") lines.append("") lines.append("## 高优先级候选源码文件") lines.append("") for row in candidates[:15]: lines.append( f"- `{row['file']}` — score={row['score']}, " f"hits={row['hit_count']}" ) title_map = { "area_geometry_build_or_write": "Area geometry 构造/写入", "coverage_geometry_build_or_write": "Coverage geometry 构造/写入", "geotype16_issue_logic": "raw_geotype=16 / geometry missing 判定", "geometry_invalid_counter": "geometry_invalid_count 统计", "bbox_update": "bbox 计算/写入", "raw_source_preservation": "raw source 保留", } for key, title in title_map.items(): lines.append("") lines.append(f"## {title}") lines.append("") rows = integration.get(key, []) if not rows: lines.append("- 未自动定位，需要人工继续查。") continue seen = set() for row in rows: marker = (row["file"], row["line"]) if marker in seen: continue seen.add(marker) lines.append( f"- `{row['file']}:{row['line']}` " f"`{row['code']}`" ) lines.append("") lines.append("## 下一阶段修改边界") lines.append("") lines.append("下一阶段只应修改真实定位到的 importer 代码，不应重新设计数据库。") lines.append("") lines.append("冻结要求：") lines.append("") lines.append("1. valid geometry 原样 PASS-THROUGH。") lines.append("2. invalid Area/Coverage 才执行 odd-even face selection。") lines.append("3. normalization 后执行 GEOS normalize canonicalization。") lines.append("4. 输出必须为 valid Polygon/MultiPolygon。") lines.append("5. Area normalized 后同步重新计算 feature bbox。") lines.append("6. retraced connector 不保存为 Area 面积，但 raw source 必须保留。") lines.append("7. raw_geotype=16 / META 不生成 `geometry missing`。") lines.append("8. raw_geotype=16 不增加 `geometry_invalid_count`。") lines.append("9. 不删除 `raw_feature_blob` 或 `chart_raw_record`。") lines.append("10. 当前仍不进入 PBF。") return "\n".join(lines) def parse_args(): p = argparse.ArgumentParser( description="CM93 importer normalization source audit v12" ) p.add_argument("--v11-input", required=True) p.add_argument("--project-root", default=".") p.add_argument("--output", default=None) return p.parse_args() def main(): args = parse_args() v11_src = Path(args.v11_input).expanduser().resolve() project_root = Path(args.project_root).expanduser().resolve() if not v11_src.exists(): print(f"V11 输入不存在：{v11_src}") sys.exit(2) if not project_root.is_dir(): print(f"项目目录不存在：{project_root}") sys.exit(2) temp_v11 = None try: v11_root, temp_v11 = open_input(v11_src) print("[1/6] V11 implementation precheck gate ...") gate = load_v11_gate(v11_root) print(json.dumps(gate, ensure_ascii=False, indent=2)) if not gate["implementation_precheck_pass"]: print("") print("STOP：V11 IMPLEMENTATION PRECHECK 未通过。") print("V12 不扫描 importer 源码。") sys.exit(3) print(" V11 gate: PASS") stamp = datetime.now().strftime("%Y%m%d_%H%M%S") outdir = Path( args.output or project_root / f"cm93_importer_source_audit_v12_{stamp}" ) outdir.mkdir(parents=True, exist_ok=True) print("[2/6] 扫描项目文本源码 ...") files = list_source_files(project_root) print(f" files={len(files)}") all_hits = [] function_rows = [] dependency_rows = [] candidate_rows = [] for i, path in enumerate(files, start=1): # 避免扫描当前输出目录自身 try: path.resolve().relative_to(outdir.resolve()) continue except Exception: pass text = safe_read_text(path) if text is None: continue hits = scan_hits(path, text, project_root) all_hits.extend(hits) function_rows.extend( python_function_inventory(path, text, project_root) ) dependency_rows.extend( python_dependency_inventory(path, text, project_root) ) if hits: candidate_rows.append( candidate_score(path, hits, project_root) ) if i % 100 == 0: print(f" scanned={i}/{len(files)}") candidate_rows.sort( key=lambda r: (-int(r["score"]), r["file"]) ) print("[3/6] 分类 SQL 命中与函数 ...") sql_hits = [ r for r in all_hits if ( r["group"] == "DB_WRITE" or re.search( r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b", r["code"], re.I, ) ) ] print("[4/6] 推导 normalization 接入点 ...") integration = choose_integration_points(all_hits) print("[5/6] 写报告 ...") write_csv( outdir / "importer_candidate_files_v12.csv", candidate_rows ) write_csv( outdir / "importer_code_hits_v12.csv", all_hits ) write_csv( outdir / "importer_sql_hits_v12.csv", sql_hits ) write_csv( outdir / "importer_function_inventory_v12.csv", function_rows ) write_csv( outdir / "importer_dependency_inventory_v12.csv", dependency_rows ) integration_md = build_integration_md( gate, candidate_rows, integration, ) ( outdir / "importer_normalization_integration_points_v12.md" ).write_text( integration_md, encoding="utf-8" ) report = { "version": "v12.0", "generated_at": datetime.now().isoformat(timespec="seconds"), "project_root": str(project_root), "read_only": True, "v11_gate": gate, "source_file_count": len(files), "candidate_file_count": len(candidate_rows), "code_hit_count": len(all_hits), "sql_hit_count": len(sql_hits), "function_count": len(function_rows), "dependency_count": len(dependency_rows), "top_candidate_files": candidate_rows[:25], "integration_point_counts": { k: len(v) for k, v in integration.items() }, } ( outdir / "importer_source_audit_report_v12.json" ).write_text( json.dumps( report, ensure_ascii=False, indent=2, default=str, ), encoding="utf-8" ) lines = [] lines.append("CM93 Importer Normalization 接入点只读调查 v12") lines.append("=" * 88) lines.append(f"Generated: {report['generated_at']}") lines.append(f"Project: {project_root}") lines.append("") lines.append("V11 GATE") lines.append(" PASS") lines.append("") lines.append("SOURCE SCAN") lines.append(f" files: {len(files)}") lines.append(f" candidate files:{len(candidate_rows)}") lines.append(f" code hits: {len(all_hits)}") lines.append(f" SQL hits: {len(sql_hits)}") lines.append(f" functions: {len(function_rows)}") lines.append("") lines.append("TOP CANDIDATE FILES") for row in candidate_rows[:15]: lines.append( f" {row['score']:4d} {row['file']}" ) lines.append("") lines.append("INTEGRATION POINT COUNTS") for key, rows in integration.items(): lines.append( f" {key:36s} {len(rows)}" ) lines.append("") lines.append( "NO SOURCE WRITE / NO DATABASE WRITE / NO PBF" ) ( outdir / "importer_source_audit_summary_v12.txt" ).write_text( "\n".join(lines), encoding="utf-8" ) print("[6/6] 压缩 ...") zpath = outdir.with_suffix(".zip") with zipfile.ZipFile( zpath, "w", compression=zipfile.ZIP_DEFLATED ) as z: for p in sorted(outdir.rglob("*")): if p.is_file(): z.write( p, p.relative_to(outdir.parent) ) print("") print("SOURCE AUDIT PASS: YES") print(f"ZIP: {zpath.resolve()}") print("把 V12 ZIP 上传给 ChatGPT。") finally: if temp_v11 and temp_v11.exists(): shutil.rmtree(temp_v11, ignore_errors=True) if __name__ == "__main__": main()