570 lines
19 KiB
Python
570 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
"""
|
|
CM93 Importer Normalization 独立源码审计
|
|
Version: standalone-v1.0
|
|
|
|
完全独立：
|
|
- 不读取 V10/V11/V12 历史结果
|
|
- 不连接 MySQL
|
|
- 不修改源码
|
|
- 不生成 patch
|
|
- 不开始 PBF
|
|
|
|
用途：
|
|
从当前 importer 项目源码零开始定位：
|
|
1. Area / Coverage geometry 构造
|
|
2. chart_area_geometry / chart_coverage / chart_feature 写库
|
|
3. geometry_issue / geometry missing
|
|
4. raw_geotype / geometry_type / META / AREA
|
|
5. geometry_invalid_count / geometry_repaired_count
|
|
6. bbox 计算与写入
|
|
7. Shapely / GEOS / Polygon / MultiPolygon / WKB
|
|
8. raw_feature_blob / chart_raw_record / source_record_index
|
|
9. import job / summary / counters
|
|
10. 可能的 normalization 接入点
|
|
|
|
运行：
|
|
python .\\cm93_importer_normalization_source_audit_standalone_v1.py `
|
|
--project-root "."
|
|
|
|
输出自动生成 ZIP。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
import argparse
|
|
import ast
|
|
import csv
|
|
import json
|
|
import re
|
|
import sys
|
|
import zipfile
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
|
|
TEXT_EXTENSIONS = {
|
|
".py", ".sql", ".toml", ".ini", ".cfg", ".json",
|
|
".yaml", ".yml", ".md", ".txt", ".env", ".example",
|
|
}
|
|
|
|
SKIP_DIRS = {
|
|
".git", ".hg", ".svn", "__pycache__", ".pytest_cache",
|
|
".mypy_cache", ".ruff_cache", ".venv", "venv", "env",
|
|
"node_modules", "dist", "build", ".idea", ".vscode",
|
|
}
|
|
|
|
SEARCH_GROUPS = {
|
|
"GEOMETRY_BUILD": [
|
|
r"\bPolygon\s*\(", r"\bMultiPolygon\s*\(", r"\bLineString\s*\(",
|
|
r"\bPoint\s*\(", r"\bshapely\b", r"\bGEOS\b", r"\bwkb\b", r"\bgeom\b",
|
|
],
|
|
"NORMALIZATION": [
|
|
r"\bmake_valid\b", r"\bpolygonize_full\b", r"\bpolygonize\b",
|
|
r"\bunary_union\b", r"\bnormalize\b", r"\bis_valid\b",
|
|
r"\bexplain_validity\b", r"\bbuffer\s*\(\s*0",
|
|
],
|
|
"GEOMETRY_ISSUE": [
|
|
r"geometry_issue", r"geometry missing", r"geometry_missing",
|
|
r"invalid geometry", r"geometry_invalid", r"geometry_repaired",
|
|
],
|
|
"GEOTYPE": [
|
|
r"raw_geotype", r"\bgeotype\b", r"geometry_type",
|
|
r"\bMETA\b", r"\bAREA\b", r"\bSOUNDING\b", r"\bPOINT\b", r"\bLINE\b",
|
|
],
|
|
"BBOX": [
|
|
r"bbox_min_lon", r"bbox_min_lat", r"bbox_max_lon", r"bbox_max_lat",
|
|
r"\bbbox\b", r"\bbounds\b", r"min_lon", r"max_lon", r"min_lat", r"max_lat",
|
|
],
|
|
"COUNTERS": [
|
|
r"geometry_invalid_count", r"geometry_repaired_count",
|
|
r"invalid_count", r"repaired_count", r"summary_json",
|
|
r"chart_import_job", r"import_job",
|
|
],
|
|
"DB_AREA": [r"chart_area_geometry", r"area_geometry"],
|
|
"DB_COVERAGE": [r"chart_coverage", r"coverage.*geom"],
|
|
"DB_FEATURE": [r"chart_feature", r"feature_attribute"],
|
|
"DB_WRITE": [
|
|
r"\bINSERT\s+INTO\b", r"\bUPDATE\b", r"\bexecutemany\s*\(",
|
|
r"\bexecute\s*\(", r"\bcommit\s*\(", r"\bflush\s*\(", r"\badd\s*\(",
|
|
],
|
|
"RAW_SOURCE": [
|
|
r"raw_feature_blob", r"chart_raw_record", r"source_record_index",
|
|
r"raw_blob", r"raw_metadata_json", r"record_kind",
|
|
],
|
|
"TOPOLOGY": [
|
|
r"edge_index", r"segment_usage", r"vector_record", r"vector_table",
|
|
r"n_elements", r"\bring\b", r"\bcontour\b", r"\bedge\b", r"\bnode\b",
|
|
],
|
|
"SRID": [r"normalized_srid", r"\bSRID\b", r"\bsrid\b", r"4326"],
|
|
}
|
|
|
|
WEIGHTS = {
|
|
"GEOMETRY_BUILD": 5, "NORMALIZATION": 8, "GEOMETRY_ISSUE": 10,
|
|
"GEOTYPE": 7, "BBOX": 6, "COUNTERS": 9, "DB_AREA": 10,
|
|
"DB_COVERAGE": 10, "DB_FEATURE": 7, "DB_WRITE": 6,
|
|
"RAW_SOURCE": 5, "TOPOLOGY": 5, "SRID": 3,
|
|
}
|
|
|
|
|
|
def write_csv(path: Path, rows: Iterable[Dict[str, Any]]) -> None:
|
|
rows = list(rows)
|
|
keys: List[str] = []
|
|
seen = set()
|
|
for row in rows:
|
|
for key in row:
|
|
if key not in seen:
|
|
seen.add(key)
|
|
keys.append(key)
|
|
|
|
with path.open("w", encoding="utf-8-sig", newline="") as f:
|
|
if not keys:
|
|
return
|
|
w = csv.DictWriter(f, fieldnames=keys, extrasaction="ignore")
|
|
w.writeheader()
|
|
for row in rows:
|
|
out = {}
|
|
for key in keys:
|
|
val = row.get(key)
|
|
if isinstance(val, (dict, list, tuple)):
|
|
val = json.dumps(val, ensure_ascii=False, default=str)
|
|
out[key] = "" if val is None else val
|
|
w.writerow(out)
|
|
|
|
|
|
def safe_text(path: Path) -> Optional[str]:
|
|
for enc in ("utf-8", "utf-8-sig", "cp932", "latin-1"):
|
|
try:
|
|
return path.read_text(encoding=enc)
|
|
except Exception:
|
|
pass
|
|
return None
|
|
|
|
|
|
def should_skip(path: Path) -> bool:
|
|
return any(part in SKIP_DIRS for part in path.parts)
|
|
|
|
|
|
def rel(path: Path, root: Path) -> str:
|
|
try:
|
|
return str(path.resolve().relative_to(root.resolve()))
|
|
except Exception:
|
|
return str(path)
|
|
|
|
|
|
def list_files(root: Path, outdir: Path) -> List[Path]:
|
|
result = []
|
|
for p in root.rglob("*"):
|
|
if not p.is_file() or should_skip(p):
|
|
continue
|
|
try:
|
|
p.resolve().relative_to(outdir.resolve())
|
|
continue
|
|
except Exception:
|
|
pass
|
|
if p.suffix.lower() in TEXT_EXTENSIONS:
|
|
result.append(p)
|
|
return sorted(result)
|
|
|
|
|
|
def context(lines: List[str], line_no: int, radius: int = 3) -> str:
|
|
start = max(1, line_no - radius)
|
|
end = min(len(lines), line_no + radius)
|
|
return "\n".join(f"{i:05d}: {lines[i-1]}" for i in range(start, end + 1))
|
|
|
|
|
|
def scan_hits(path: Path, text: str, root: Path) -> List[Dict[str, Any]]:
|
|
rows = []
|
|
lines = text.splitlines()
|
|
compiled = {
|
|
group: [re.compile(p, re.I) for p in pats]
|
|
for group, pats in SEARCH_GROUPS.items()
|
|
}
|
|
for no, line in enumerate(lines, 1):
|
|
for group, pats in compiled.items():
|
|
matched = [p.pattern for p in pats if p.search(line)]
|
|
if matched:
|
|
rows.append({
|
|
"file": rel(path, root),
|
|
"line": no,
|
|
"group": group,
|
|
"matched_patterns": matched,
|
|
"code": line.strip(),
|
|
"context": context(lines, no),
|
|
})
|
|
return rows
|
|
|
|
|
|
def ast_inventory(path: Path, text: str, root: Path):
|
|
funcs, classes, deps, sqls = [], [], [], []
|
|
if path.suffix.lower() != ".py":
|
|
return funcs, classes, deps, sqls
|
|
try:
|
|
tree = ast.parse(text)
|
|
except Exception:
|
|
return funcs, classes, deps, sqls
|
|
|
|
class V(ast.NodeVisitor):
|
|
def __init__(self):
|
|
self.cls: List[str] = []
|
|
self.fn: List[str] = []
|
|
|
|
def visit_ClassDef(self, node):
|
|
classes.append({
|
|
"file": rel(path, root),
|
|
"class_name": node.name,
|
|
"line": node.lineno,
|
|
"end_line": getattr(node, "end_lineno", None),
|
|
})
|
|
self.cls.append(node.name)
|
|
self.generic_visit(node)
|
|
self.cls.pop()
|
|
|
|
def visit_FunctionDef(self, node):
|
|
funcs.append({
|
|
"file": rel(path, root),
|
|
"class_name": ".".join(self.cls),
|
|
"function_name": node.name,
|
|
"line": node.lineno,
|
|
"end_line": getattr(node, "end_lineno", None),
|
|
"async": 0,
|
|
"args": [a.arg for a in node.args.args],
|
|
})
|
|
self.fn.append(node.name)
|
|
self.generic_visit(node)
|
|
self.fn.pop()
|
|
|
|
def visit_AsyncFunctionDef(self, node):
|
|
funcs.append({
|
|
"file": rel(path, root),
|
|
"class_name": ".".join(self.cls),
|
|
"function_name": node.name,
|
|
"line": node.lineno,
|
|
"end_line": getattr(node, "end_lineno", None),
|
|
"async": 1,
|
|
"args": [a.arg for a in node.args.args],
|
|
})
|
|
self.fn.append(node.name)
|
|
self.generic_visit(node)
|
|
self.fn.pop()
|
|
|
|
def visit_Import(self, node):
|
|
for a in node.names:
|
|
deps.append({
|
|
"file": rel(path, root), "line": node.lineno,
|
|
"type": "import", "module": a.name, "names": "",
|
|
})
|
|
|
|
def visit_ImportFrom(self, node):
|
|
deps.append({
|
|
"file": rel(path, root), "line": node.lineno,
|
|
"type": "from", "module": node.module or "",
|
|
"names": [x.name for x in node.names],
|
|
})
|
|
|
|
def visit_Constant(self, node):
|
|
if isinstance(node.value, str) and re.search(
|
|
r"\b(SELECT|INSERT\s+INTO|UPDATE|DELETE\s+FROM|CREATE\s+TABLE|ALTER\s+TABLE)\b",
|
|
node.value, re.I
|
|
):
|
|
sqls.append({
|
|
"file": rel(path, root),
|
|
"line": getattr(node, "lineno", None),
|
|
"class_name": ".".join(self.cls),
|
|
"function_name": ".".join(self.fn),
|
|
"sql": node.value.strip(),
|
|
})
|
|
|
|
V().visit(tree)
|
|
return funcs, classes, deps, sqls
|
|
|
|
|
|
def score_file(path: Path, hits: List[Dict[str, Any]], root: Path) -> Dict[str, Any]:
|
|
c = Counter(x["group"] for x in hits)
|
|
score = sum(WEIGHTS.get(g, 1) * min(n, 15) for g, n in c.items())
|
|
name = rel(path, root).lower()
|
|
for token, bonus in (
|
|
("import", 10), ("cm93", 10), ("geom", 7), ("coverage", 6),
|
|
("feature", 5), ("parser", 5), ("decoder", 5), ("mysql", 4),
|
|
("database", 4), ("repository", 3), ("model", 2),
|
|
):
|
|
if token in name:
|
|
score += bonus
|
|
row = {"file": rel(path, root), "score": score, "hit_count": len(hits)}
|
|
for g in SEARCH_GROUPS:
|
|
row[f"hits_{g.lower()}"] = c.get(g, 0)
|
|
return row
|
|
|
|
|
|
def derive_points(hits: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]:
|
|
buckets = {
|
|
"area_geometry_build": [], "coverage_geometry_build": [],
|
|
"area_geometry_write": [], "coverage_geometry_write": [],
|
|
"feature_write": [], "geometry_issue_logic": [],
|
|
"geotype_logic": [], "invalid_counter_logic": [],
|
|
"repaired_counter_logic": [], "bbox_logic": [],
|
|
"raw_source_preservation": [], "srid_logic": [],
|
|
"normalization_existing": [],
|
|
}
|
|
|
|
for r in hits:
|
|
code = r["code"].lower()
|
|
grp = r["group"]
|
|
|
|
if grp == "GEOMETRY_BUILD":
|
|
if "coverage" in code or "coverage" in r["file"].lower():
|
|
buckets["coverage_geometry_build"].append(r)
|
|
else:
|
|
buckets["area_geometry_build"].append(r)
|
|
|
|
if "chart_area_geometry" in code or "area_geometry" in code:
|
|
buckets["area_geometry_write"].append(r)
|
|
if "chart_coverage" in code:
|
|
buckets["coverage_geometry_write"].append(r)
|
|
if "chart_feature" in code and grp in {"DB_FEATURE", "DB_WRITE"}:
|
|
buckets["feature_write"].append(r)
|
|
if grp == "GEOMETRY_ISSUE":
|
|
buckets["geometry_issue_logic"].append(r)
|
|
if grp == "GEOTYPE" or "geometry missing" in code:
|
|
buckets["geotype_logic"].append(r)
|
|
if "geometry_invalid_count" in code:
|
|
buckets["invalid_counter_logic"].append(r)
|
|
if "geometry_repaired_count" in code:
|
|
buckets["repaired_counter_logic"].append(r)
|
|
if grp == "BBOX":
|
|
buckets["bbox_logic"].append(r)
|
|
if grp == "RAW_SOURCE":
|
|
buckets["raw_source_preservation"].append(r)
|
|
if grp == "SRID":
|
|
buckets["srid_logic"].append(r)
|
|
if grp == "NORMALIZATION":
|
|
buckets["normalization_existing"].append(r)
|
|
|
|
for key, rows in buckets.items():
|
|
uniq = {(r["file"], int(r["line"])): r for r in rows}
|
|
buckets[key] = sorted(
|
|
uniq.values(),
|
|
key=lambda x: (x["file"], int(x["line"]))
|
|
)[:60]
|
|
|
|
return buckets
|
|
|
|
|
|
def build_md(candidates, points) -> str:
|
|
titles = {
|
|
"area_geometry_build": "Area geometry 构造",
|
|
"coverage_geometry_build": "Coverage geometry 构造",
|
|
"area_geometry_write": "Area geometry 写库",
|
|
"coverage_geometry_write": "Coverage geometry 写库",
|
|
"feature_write": "Feature 写库",
|
|
"geometry_issue_logic": "geometry_issue 判定",
|
|
"geotype_logic": "raw_geotype / META / geometry missing",
|
|
"invalid_counter_logic": "geometry_invalid_count",
|
|
"repaired_counter_logic": "geometry_repaired_count",
|
|
"bbox_logic": "bbox 计算与写入",
|
|
"raw_source_preservation": "raw source 保留",
|
|
"srid_logic": "SRID / normalized_srid",
|
|
"normalization_existing": "现有 validity / normalization",
|
|
}
|
|
|
|
lines = [
|
|
"# CM93 Importer Normalization 独立源码审计 v1",
|
|
"",
|
|
"本报告仅依据当前项目源码，不依赖任何历史测试结果。",
|
|
"",
|
|
"## 高优先级候选文件",
|
|
"",
|
|
]
|
|
for r in candidates[:20]:
|
|
lines.append(f"- `{r['file']}` — score={r['score']}, hits={r['hit_count']}")
|
|
|
|
for key, title in titles.items():
|
|
lines += ["", f"## {title}", ""]
|
|
rows = points.get(key, [])
|
|
if not rows:
|
|
lines.append("- 未自动定位。")
|
|
else:
|
|
for r in rows[:30]:
|
|
lines.append(f"- `{r['file']}:{r['line']}` — `{r['code']}`")
|
|
|
|
lines += [
|
|
"",
|
|
"## 下一阶段人工确认重点",
|
|
"",
|
|
"1. 确认 Area geometry 的唯一最终构造入口。",
|
|
"2. 确认 Coverage geometry 的唯一最终构造入口。",
|
|
"3. 确认 validity 检查发生在构造后、写库前还是写库后。",
|
|
"4. 确认 raw_geotype / META 为什么会进入 geometry missing。",
|
|
"5. 确认 geometry_invalid_count 的实际累计条件。",
|
|
"6. 确认 bbox 来自 raw topology 还是最终 geometry。",
|
|
"7. 确认数据库写入点对 Polygon/MultiPolygon 的处理方式。",
|
|
"8. 确认 raw_feature_blob / chart_raw_record 始终保留。",
|
|
"",
|
|
"本轮不修改源码，不修改数据库，不进入 PBF。",
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def parse_args():
|
|
p = argparse.ArgumentParser(description="CM93 importer 独立源码审计")
|
|
p.add_argument("--project-root", default=".")
|
|
p.add_argument("--output", default=None)
|
|
return p.parse_args()
|
|
|
|
|
|
def main():
|
|
args = parse_args()
|
|
root = Path(args.project_root).expanduser().resolve()
|
|
if not root.is_dir():
|
|
print(f"项目目录不存在：{root}")
|
|
sys.exit(2)
|
|
|
|
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
outdir = Path(
|
|
args.output or root / f"cm93_importer_source_audit_standalone_v1_{stamp}"
|
|
).expanduser().resolve()
|
|
outdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
print("[1/6] 枚举源码文件 ...")
|
|
files = list_files(root, outdir)
|
|
print(f" files={len(files)}")
|
|
|
|
file_rows, hits = [], []
|
|
funcs, classes, deps, ast_sql = [], [], [], []
|
|
candidates = []
|
|
|
|
print("[2/6] 关键字 + AST 扫描 ...")
|
|
for i, path in enumerate(files, 1):
|
|
text = safe_text(path)
|
|
file_rows.append({
|
|
"file": rel(path, root),
|
|
"suffix": path.suffix.lower(),
|
|
"size_bytes": path.stat().st_size,
|
|
"readable_text": int(text is not None),
|
|
})
|
|
if text is None:
|
|
continue
|
|
|
|
h = scan_hits(path, text, root)
|
|
hits.extend(h)
|
|
f, c, d, s = ast_inventory(path, text, root)
|
|
funcs.extend(f)
|
|
classes.extend(c)
|
|
deps.extend(d)
|
|
ast_sql.extend(s)
|
|
if h:
|
|
candidates.append(score_file(path, h, root))
|
|
|
|
if i % 100 == 0:
|
|
print(f" scanned={i}/{len(files)}")
|
|
|
|
candidates.sort(key=lambda r: (-int(r["score"]), r["file"]))
|
|
|
|
print("[3/6] 归类 SQL 与接入点 ...")
|
|
sql_hits = [
|
|
r for r in hits
|
|
if r["group"] in {"DB_AREA", "DB_COVERAGE", "DB_FEATURE", "DB_WRITE"}
|
|
or re.search(r"\b(SELECT|INSERT|UPDATE|DELETE|CREATE|ALTER)\b", r["code"], re.I)
|
|
]
|
|
for r in ast_sql:
|
|
sql_hits.append({
|
|
"file": r["file"], "line": r["line"],
|
|
"group": "AST_SQL_LITERAL", "code": r["sql"],
|
|
"context": r["sql"], "class_name": r.get("class_name"),
|
|
"function_name": r.get("function_name"),
|
|
})
|
|
|
|
points = derive_points(hits)
|
|
|
|
print("[4/6] 写结构化输出 ...")
|
|
write_csv(outdir / "项目文件清单_v1.csv", file_rows)
|
|
write_csv(outdir / "候选源码文件_v1.csv", candidates)
|
|
write_csv(outdir / "关键代码命中_v1.csv", hits)
|
|
write_csv(outdir / "SQL命中_v1.csv", sql_hits)
|
|
write_csv(outdir / "Python函数清单_v1.csv", funcs)
|
|
write_csv(outdir / "Python类清单_v1.csv", classes)
|
|
write_csv(outdir / "Python依赖清单_v1.csv", deps)
|
|
|
|
(outdir / "normalization接入点_v1.md").write_text(
|
|
build_md(candidates, points), encoding="utf-8"
|
|
)
|
|
|
|
print("[5/6] 写摘要 ...")
|
|
group_counts = Counter(r["group"] for r in hits)
|
|
report = {
|
|
"version": "standalone-v1.0",
|
|
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
|
"project_root": str(root),
|
|
"history_dependency": False,
|
|
"database_dependency": False,
|
|
"read_only": True,
|
|
"pbf": "OUT_OF_SCOPE",
|
|
"source_file_count": len(files),
|
|
"candidate_file_count": len(candidates),
|
|
"code_hit_count": len(hits),
|
|
"sql_hit_count": len(sql_hits),
|
|
"function_count": len(funcs),
|
|
"class_count": len(classes),
|
|
"dependency_count": len(deps),
|
|
"hit_group_counts": dict(group_counts),
|
|
"integration_point_counts": {k: len(v) for k, v in points.items()},
|
|
"top_candidate_files": candidates[:25],
|
|
}
|
|
|
|
(outdir / "独立源码审计报告_v1.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2, default=str),
|
|
encoding="utf-8"
|
|
)
|
|
|
|
lines = [
|
|
"CM93 Importer Normalization 独立源码审计 v1",
|
|
"=" * 88,
|
|
f"Generated: {report['generated_at']}",
|
|
f"Project: {root}",
|
|
"",
|
|
"DEPENDENCY",
|
|
" 历史测试依赖: NO",
|
|
" 数据库依赖: NO",
|
|
" 源码修改: NO",
|
|
" PBF: OUT OF SCOPE",
|
|
"",
|
|
"SCAN",
|
|
f" source files: {len(files)}",
|
|
f" candidate files: {len(candidates)}",
|
|
f" code hits: {len(hits)}",
|
|
f" SQL hits: {len(sql_hits)}",
|
|
f" Python functions: {len(funcs)}",
|
|
f" Python classes: {len(classes)}",
|
|
"",
|
|
"TOP CANDIDATE FILES",
|
|
]
|
|
for r in candidates[:15]:
|
|
lines.append(f" {int(r['score']):4d} {r['file']}")
|
|
lines += [
|
|
"",
|
|
"SOURCE AUDIT COMPLETE: YES",
|
|
"NO SOURCE WRITE / NO DATABASE WRITE / NO PBF",
|
|
]
|
|
|
|
(outdir / "独立源码审计摘要_v1.txt").write_text(
|
|
"\n".join(lines), encoding="utf-8"
|
|
)
|
|
|
|
print("[6/6] 压缩 ...")
|
|
zpath = outdir.with_suffix(".zip")
|
|
with zipfile.ZipFile(zpath, "w", compression=zipfile.ZIP_DEFLATED) as z:
|
|
for p in sorted(outdir.rglob("*")):
|
|
if p.is_file():
|
|
z.write(p, p.relative_to(outdir.parent))
|
|
|
|
print("")
|
|
print("SOURCE AUDIT COMPLETE: YES")
|
|
print(f"Output: {outdir}")
|
|
print(f"ZIP: {zpath}")
|
|
print("把 ZIP 上传给 ChatGPT。")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|