#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ CM93 Importer Normalization 独立源码审计 Version: standalone-v1.0 完全独立： - 不读取 V10/V11/V12 历史结果 - 不连接 MySQL - 不修改源码 - 不生成 patch - 不开始 PBF 用途： 从当前 importer 项目源码零开始定位： 1. Area / Coverage geometry 构造 2. chart_area_geometry / chart_coverage / chart_feature 写库 3. geometry_issue / geometry missing 4. raw_geotype / geometry_type / META / AREA 5. geometry_invalid_count / geometry_repaired_count 6. bbox 计算与写入 7. Shapely / GEOS / Polygon / MultiPolygon / WKB 8. raw_feature_blob / chart_raw_record / source_record_index 9. import job / summary / counters 10. 可能的 normalization 接入点 运行： python .\\cm93_importer_normalization_source_audit_standalone_v1.py ` --project-root "." 输出自动生成 ZIP。 """ from __future__ import annotations import argparse import ast import csv import json import re import sys import zipfile from collections import Counter, defaultdict from datetime import datetime from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Tuple TEXT_EXTENSIONS = { ".py", ".sql", ".toml", ".ini", ".cfg", ".json", ".yaml", ".yml", ".md", ".txt", ".env", ".example", } SKIP_DIRS = { ".git", ".hg", ".svn", "__pycache__", ".pytest_cache", ".mypy_cache", ".ruff_cache", ".venv", "venv", "env", "node_modules", "dist", "build", ".idea", ".vscode", } SEARCH_GROUPS = { "GEOMETRY_BUILD": [ r"\bPolygon\s*\(", r"\bMultiPolygon\s*\(", r"\bLineString\s*\(", r"\bPoint\s*\(", r"\bshapely\b", r"\bGEOS\b", r"\bwkb\b", r"\bgeom\b", ], "NORMALIZATION": [ r"\bmake_valid\b", r"\bpolygonize_full\b", r"\bpolygonize\b", r"\bunary_union\b", r"\bnormalize\b", r"\bis_valid\b", r"\bexplain_validity\b", r"\bbuffer\s*\(\s*0", ], "GEOMETRY_ISSUE": [ r"geometry_issue", r"geometry missing", r"geometry_missing", r"invalid geometry", r"geometry_invalid", r"geometry_repaired", ], "GEOTYPE": [ r"raw_geotype", r"\bgeotype\b", r"geometry_type", r"\bMETA\b", r"\bAREA\b", r"\bSOUNDING\b", r"\bPOINT\b", r"\bLINE\b", ], "BBOX": [ r"bbox_min_lon", r"bbox_min_lat", r"bbox_max_lon", r"bbox_max_lat", r"\bbbox\b", r"\bbounds\b", r"min_lon", r"max_lon", r"min_lat", r"max_lat", ], "COUNTERS": [ r"geometry_invalid_count", r"geometry_repaired_count", r"invalid_count", r"repaired_count", r"summary_json", r"chart_import_job", r"import_job", ], "DB_AREA": [r"chart_area_geometry", r"area_geometry"], "DB_COVERAGE": [r"chart_coverage", r"coverage.*geom"], "DB_FEATURE": [r"chart_feature", r"feature_attribute"], "DB_WRITE": [ r"\bINSERT\s+INTO\b", r"\bUPDATE\b", r"\bexecutemany\s*\(", r"\bexecute\s*\(", r"\bcommit\s*\(", r"\bflush\s*\(", r"\badd\s*\(", ], "RAW_SOURCE": [ r"raw_feature_blob", r"chart_raw_record", r"source_record_index", r"raw_blob", r"raw_metadata_json", r"record_kind", ], "TOPOLOGY": [ r"edge_index", r"segment_usage", r"vector_record", r"vector_table", r"n_elements", r"\bring\b", r"\bcontour\b", r"\bedge\b", r"\bnode\b", ], "SRID": [r"normalized_srid", r"\bSRID\b", r"\bsrid\b", r"4326"], } WEIGHTS = { "GEOMETRY_BUILD": 5, "NORMALIZATION": 8, "GEOMETRY_ISSUE": 10, "GEOTYPE": 7, "BBOX": 6, "COUNTERS": 9, "DB_AREA": 10, "DB_COVERAGE": 10, "DB_FEATURE": 7, "DB_WRITE": 6, "RAW_SOURCE": 5, "TOPOLOGY": 5, "SRID": 3, } def write_csv(path: Path, rows: Iterable[Dict[str, Any]]) -> None: rows = list(rows) keys: List[str] = [] seen = set() for row in rows: for key in row: if key not in seen: seen.add(key) keys.append(key) with path.open("w", encoding="utf-8-sig", newline="") as f: if not keys: return w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore") w.writeheader() for row in rows: out = {} for key in keys: val = row.get(key) if isinstance(val, (dict, list, tuple)): val = json.dumps(val, ensure_ascii=False, default=str) out[key] = "" if val is None else val w.writerow(out) def safe_text(path: Path) -> Optional[str]: for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"): try: return path.read_text(encoding=enc) except Exception: pass return None def should_skip(path: Path) -> bool: return any(part in SKIP_DIRS for part in path.parts) def rel(path: Path, root: Path) -> str: try: return str(path.resolve().relative_to(root.resolve())) except Exception: return str(path) def list_files(root: Path, outdir: Path) -> List[Path]: result = [] for p in root.rglob("*"): if not p.is_file() or should_skip(p): continue try: p.resolve().relative_to(outdir.resolve()) continue except Exception: pass if p.suffix.lower() in TEXT_EXTENSIONS: result.append(p) return sorted(result) def context(lines: List[str], line_no: int, radius: int = 3) -> str: start = max(1, line_no - radius) end = min(len(lines), line_no + radius) return "\n".join(f"{i:05d}: {lines[i-1]}" for i in range(start, end + 1)) def scan_hits(path: Path, text: str, root: Path) -> List[Dict[str, Any]]: rows = [] lines = text.splitlines() compiled = { group: [re.compile(p, re.I) for p in pats] for group, pats in SEARCH_GROUPS.items() } for no, line in enumerate(lines, 1): for group, pats in compiled.items(): matched = [p.pattern for p in pats if p.search(line)] if matched: rows.append({ "file": rel(path, root), "line": no, "group": group, "matched_patterns": matched, "code": line.strip(), "context": context(lines, no), }) return rows def ast_inventory(path: Path, text: str, root: Path): funcs, classes, deps, sqls = [], [], [], [] if path.suffix.lower() != ".py": return funcs, classes, deps, sqls try: tree = ast.parse(text) except Exception: return funcs, classes, deps, sqls class V(ast.NodeVisitor): def __init__(self): self.cls: List[str] = [] self.fn: List[str] = [] def visit_ClassDef(self, node): classes.append({ "file": rel(path, root), "class_name": node.name, "line": node.lineno, "end_line": getattr(node, "end_lineno", None), }) self.cls.append(node.name) self.generic_visit(node) self.cls.pop() def visit_FunctionDef(self, node): funcs.append({ "file": rel(path, root), "class_name": ".".join(self.cls), "function_name": node.name, "line": node.lineno, "end_line": getattr(node, "end_lineno", None), "async": 0, "args": [a.arg for a in node.args.args], }) self.fn.append(node.name) self.generic_visit(node) self.fn.pop() def visit_AsyncFunctionDef(self, node): funcs.append({ "file": rel(path, root), "class_name": ".".join(self.cls), "function_name": node.name, "line": node.lineno, "end_line": getattr(node, "end_lineno", None), "async": 1, "args": [a.arg for a in node.args.args], }) self.fn.append(node.name) self.generic_visit(node) self.fn.pop() def visit_Import(self, node): for a in node.names: deps.append({ "file": rel(path, root), "line": node.lineno, "type": "import", "module": a.name, "names": "", }) def visit_ImportFrom(self, node): deps.append({ "file": rel(path, root), "line": node.lineno, "type": "from", "module": node.module or "", "names": [x.name for x in node.names], }) def visit_Constant(self, node): if isinstance(node.value, str) and re.search( r"\b(SELECT|INSERT\s+INTO|UPDATE|DELETE\s+FROM|CREATE\s+TABLE|ALTER\s+TABLE)\b", node.value, re.I ): sqls.append({ "file": rel(path, root), "line": getattr(node, "lineno", None), "class_name": ".".join(self.cls), "function_name": ".".join(self.fn), "sql": node.value.strip(), }) V().visit(tree) return funcs, classes, deps, sqls def score_file(path: Path, hits: List[Dict[str, Any]], root: Path) -> Dict[str, Any]: c = Counter(x["group"] for x in hits) score = sum(WEIGHTS.get(g, 1) * min(n, 15) for g, n in c.items()) name = rel(path, root).lower() for token, bonus in ( ("import", 10), ("cm93", 10), ("geom", 7), ("coverage", 6), ("feature", 5), ("parser", 5), ("decoder", 5), ("mysql", 4), ("database", 4), ("repository", 3), ("model", 2), ): if token in name: score += bonus row = {"file": rel(path, root), "score": score, "hit_count": len(hits)} for g in SEARCH_GROUPS: row[f"hits_{g.lower()}"] = c.get(g, 0) return row def derive_points(hits: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: buckets = { "area_geometry_build": [], "coverage_geometry_build": [], "area_geometry_write": [], "coverage_geometry_write": [], "feature_write": [], "geometry_issue_logic": [], "geotype_logic": [], "invalid_counter_logic": [], "repaired_counter_logic": [], "bbox_logic": [], "raw_source_preservation": [], "srid_logic": [], "normalization_existing": [], } for r in hits: code = r["code"].lower() grp = r["group"] if grp == "GEOMETRY_BUILD": if "coverage" in code or "coverage" in r["file"].lower(): buckets["coverage_geometry_build"].append(r) else: buckets["area_geometry_build"].append(r) if "chart_area_geometry" in code or "area_geometry" in code: buckets["area_geometry_write"].append(r) if "chart_coverage" in code: buckets["coverage_geometry_write"].append(r) if "chart_feature" in code and grp in {"DB_FEATURE", "DB_WRITE"}: buckets["feature_write"].append(r) if grp == "GEOMETRY_ISSUE": buckets["geometry_issue_logic"].append(r) if grp == "GEOTYPE" or "geometry missing" in code: buckets["geotype_logic"].append(r) if "geometry_invalid_count" in code: buckets["invalid_counter_logic"].append(r) if "geometry_repaired_count" in code: buckets["repaired_counter_logic"].append(r) if grp == "BBOX": buckets["bbox_logic"].append(r) if grp == "RAW_SOURCE": buckets["raw_source_preservation"].append(r) if grp == "SRID": buckets["srid_logic"].append(r) if grp == "NORMALIZATION": buckets["normalization_existing"].append(r) for key, rows in buckets.items(): uniq = {(r["file"], int(r["line"])): r for r in rows} buckets[key] = sorted( uniq.values(), key=lambda x: (x["file"], int(x["line"])) )[:60] return buckets def build_md(candidates, points) -> str: titles = { "area_geometry_build": "Area geometry 构造", "coverage_geometry_build": "Coverage geometry 构造", "area_geometry_write": "Area geometry 写库", "coverage_geometry_write": "Coverage geometry 写库", "feature_write": "Feature 写库", "geometry_issue_logic": "geometry_issue 判定", "geotype_logic": "raw_geotype / META / geometry missing", "invalid_counter_logic": "geometry_invalid_count", "repaired_counter_logic": "geometry_repaired_count", "bbox_logic": "bbox 计算与写入", "raw_source_preservation": "raw source 保留", "srid_logic": "SRID / normalized_srid", "normalization_existing": "现有 validity / normalization", } lines = [ "# CM93 Importer Normalization 独立源码审计 v1", "", "本报告仅依据当前项目源码，不依赖任何历史测试结果。", "", "## 高优先级候选文件", "", ] for r in candidates[:20]: lines.append(f"- `{r['file']}` — score={r['score']}, hits={r['hit_count']}") for key, title in titles.items(): lines += ["", f"## {title}", ""] rows = points.get(key, []) if not rows: lines.append("- 未自动定位。") else: for r in rows[:30]: lines.append(f"- `{r['file']}:{r['line']}` — `{r['code']}`") lines += [ "", "## 下一阶段人工确认重点", "", "1. 确认 Area geometry 的唯一最终构造入口。", "2. 确认 Coverage geometry 的唯一最终构造入口。", "3. 确认 validity 检查发生在构造后、写库前还是写库后。", "4. 确认 raw_geotype / META 为什么会进入 geometry missing。", "5. 确认 geometry_invalid_count 的实际累计条件。", "6. 确认 bbox 来自 raw topology 还是最终 geometry。", "7. 确认数据库写入点对 Polygon/MultiPolygon 的处理方式。", "8. 确认 raw_feature_blob / chart_raw_record 始终保留。", "", "本轮不修改源码，不修改数据库，不进入 PBF。", ] return "\n".join(lines) def parse_args(): p = argparse.ArgumentParser(description="CM93 importer 独立源码审计") p.add_argument("--project-root", default=".") p.add_argument("--output", default=None) return p.parse_args() def main(): args = parse_args() root = Path(args.project_root).expanduser().resolve() if not root.is_dir(): print(f"项目目录不存在：{root}") sys.exit(2) stamp = datetime.now().strftime("%Y%m%d_%H%M%S") outdir = Path( args.output or root / f"cm93_importer_source_audit_standalone_v1_{stamp}" ).expanduser().resolve() outdir.mkdir(parents=True, exist_ok=True) print("[1/6] 枚举源码文件 ...") files = list_files(root, outdir) print(f" files={len(files)}") file_rows, hits = [], [] funcs, classes, deps, ast_sql = [], [], [], [] candidates = [] print("[2/6] 关键字 + AST 扫描 ...") for i, path in enumerate(files, 1): text = safe_text(path) file_rows.append({ "file": rel(path, root), "suffix": path.suffix.lower(), "size_bytes": path.stat().st_size, "readable_text": int(text is not None), }) if text is None: continue h = scan_hits(path, text, root) hits.extend(h) f, c, d, s = ast_inventory(path, text, root) funcs.extend(f) classes.extend(c) deps.extend(d) ast_sql.extend(s) if h: candidates.append(score_file(path, h, root)) if i % 100 == 0: print(f" scanned={i}/{len(files)}") candidates.sort(key=lambda r: (-int(r["score"]), r["file"])) print("[3/6] 归类 SQL 与接入点 ...") sql_hits = [ r for r in hits if r["group"] in {"DB_AREA", "DB_COVERAGE", "DB_FEATURE", "DB_WRITE"} or re.search(r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b", r["code"], re.I) ] for r in ast_sql: sql_hits.append({ "file": r["file"], "line": r["line"], "group": "AST_SQL_LITERAL", "code": r["sql"], "context": r["sql"], "class_name": r.get("class_name"), "function_name": r.get("function_name"), }) points = derive_points(hits) print("[4/6] 写结构化输出 ...") write_csv(outdir / "项目文件清单_v1.csv", file_rows) write_csv(outdir / "候选源码文件_v1.csv", candidates) write_csv(outdir / "关键代码命中_v1.csv", hits) write_csv(outdir / "SQL命中_v1.csv", sql_hits) write_csv(outdir / "Python函数清单_v1.csv", funcs) write_csv(outdir / "Python类清单_v1.csv", classes) write_csv(outdir / "Python依赖清单_v1.csv", deps) (outdir / "normalization接入点_v1.md").write_text( build_md(candidates, points), encoding="utf-8" ) print("[5/6] 写摘要 ...") group_counts = Counter(r["group"] for r in hits) report = { "version": "standalone-v1.0", "generated_at": datetime.now().isoformat(timespec="seconds"), "project_root": str(root), "history_dependency": False, "database_dependency": False, "read_only": True, "pbf": "OUT_OF_SCOPE", "source_file_count": len(files), "candidate_file_count": len(candidates), "code_hit_count": len(hits), "sql_hit_count": len(sql_hits), "function_count": len(funcs), "class_count": len(classes), "dependency_count": len(deps), "hit_group_counts": dict(group_counts), "integration_point_counts": {k: len(v) for k, v in points.items()}, "top_candidate_files": candidates[:25], } (outdir / "独立源码审计报告_v1.json").write_text( json.dumps(report, ensure_ascii=False, indent=2, default=str), encoding="utf-8" ) lines = [ "CM93 Importer Normalization 独立源码审计 v1", "=" * 88, f"Generated: {report['generated_at']}", f"Project: {root}", "", "DEPENDENCY", " 历史测试依赖: NO", " 数据库依赖: NO", " 源码修改: NO", " PBF: OUT OF SCOPE", "", "SCAN", f" source files: {len(files)}", f" candidate files: {len(candidates)}", f" code hits: {len(hits)}", f" SQL hits: {len(sql_hits)}", f" Python functions: {len(funcs)}", f" Python classes: {len(classes)}", "", "TOP CANDIDATE FILES", ] for r in candidates[:15]: lines.append(f" {int(r['score']):4d} {r['file']}") lines += [ "", "SOURCE AUDIT COMPLETE: YES", "NO SOURCE WRITE / NO DATABASE WRITE / NO PBF", ] (outdir / "独立源码审计摘要_v1.txt").write_text( "\n".join(lines), encoding="utf-8" ) print("[6/6] 压缩 ...") zpath = outdir.with_suffix(".zip") with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z: for p in sorted(outdir.rglob("*")): if p.is_file(): z.write(p, p.relative_to(outdir.parent)) print("") print("SOURCE AUDIT COMPLETE: YES") print(f"Output: {outdir}") print(f"ZIP: {zpath}") print("把 ZIP 上传给 ChatGPT。") if __name__ == "__main__": main()