commit
91a8a83f87
11 changed files with 351 additions and 0 deletions
@ -0,0 +1,3 @@ |
|||||
|
/org_tree_matcher/data/table1.csv |
||||
|
/org_tree_matcher/data/table2.csv |
||||
|
/.idea/ |
@ -0,0 +1,34 @@ |
|||||
|
# 机构树智能匹配系统 |
||||
|
|
||||
|
## 目录结构 |
||||
|
- stage1_tree_rebuild/ 机构树重构 |
||||
|
- stage2_hierarchical_clean/ 层级感知清洗 |
||||
|
- stage3_cross_tree_match/ 跨树节点匹配 |
||||
|
- stage4_anomaly_detect/ 异常检测与标记 |
||||
|
- data/ 数据目录(table1.csv, table2.csv) |
||||
|
- scripts/ 执行脚本 |
||||
|
|
||||
|
## 依赖安装 |
||||
|
```bash |
||||
|
pip install -r requirements.txt |
||||
|
``` |
||||
|
|
||||
|
## 数据准备 |
||||
|
将原始数据(如table1.csv, table2.csv)放入data/目录。 |
||||
|
|
||||
|
## 执行流程 |
||||
|
```bash |
||||
|
bash scripts/run_all.sh |
||||
|
# 或 |
||||
|
python3 scripts/run_all.py |
||||
|
``` |
||||
|
|
||||
|
## 输出说明 |
||||
|
- 合并后的机构树 |
||||
|
- 异常节点报告 |
||||
|
- 跨表匹配对照表 |
||||
|
- 未匹配节点分析 |
||||
|
|
||||
|
## 备注 |
||||
|
- 各阶段代码均为可扩展骨架,便于后续完善。 |
||||
|
- 如需样例数据格式,可参考data/目录下的csv模板。 |
@ -0,0 +1,4 @@ |
|||||
|
pandas |
||||
|
networkx |
||||
|
sentence-transformers |
||||
|
tqdm |
@ -0,0 +1,14 @@ |
|||||
|
import os |
||||
|
import subprocess |
||||
|
|
||||
|
steps = [ |
||||
|
("机构树重构", "stage1_tree_rebuild.tree_builder"), |
||||
|
("层级感知清洗", "stage2_hierarchical_clean.hierarchical_cleaner"), |
||||
|
("跨树节点匹配", "stage3_cross_tree_match.matcher"), |
||||
|
("异常检测与标记", "stage4_anomaly_detect.anomaly_detector"), |
||||
|
] |
||||
|
|
||||
|
for name, module in steps: |
||||
|
print(f"[运行] {name} ...") |
||||
|
subprocess.run(["python3", "-m", module], check=True) |
||||
|
print("全部流程完成!") |
@ -0,0 +1,16 @@ |
|||||
|
#!/bin/bash |
||||
|
set -e |
||||
|
|
||||
|
echo "[1] 机构树重构..." |
||||
|
python3 -m stage1_tree_rebuild.tree_builder |
||||
|
|
||||
|
echo "[2] 层级感知清洗..." |
||||
|
python3 -m stage2_hierarchical_clean.hierarchical_cleaner |
||||
|
|
||||
|
echo "[3] 跨树节点匹配..." |
||||
|
python3 -m stage3_cross_tree_match.matcher |
||||
|
|
||||
|
echo "[4] 异常检测与标记..." |
||||
|
python3 -m stage4_anomaly_detect.anomaly_detector |
||||
|
|
||||
|
echo "全部流程完成!" |
@ -0,0 +1,5 @@ |
|||||
|
from difflib import SequenceMatcher |
||||
|
|
||||
|
|
||||
|
def lcs_ratio(a: str, b: str) -> float: |
||||
|
return SequenceMatcher(None, a, b).ratio() |
@ -0,0 +1,34 @@ |
|||||
|
# 机构树重构模块 |
||||
|
from typing import List, Optional |
||||
|
|
||||
|
|
||||
|
class TreeNode: |
||||
|
def __init__(self, name: str, node_id: str, parent: Optional['TreeNode'] = None, source: str = "unknown"): |
||||
|
self.name = name |
||||
|
self.node_id = node_id |
||||
|
self.parent = parent |
||||
|
self.children: List['TreeNode'] = [] |
||||
|
self.source = source |
||||
|
self.normalized_name = name |
||||
|
|
||||
|
def add_child(self, child: 'TreeNode'): |
||||
|
self.children.append(child) |
||||
|
child.parent = self |
||||
|
|
||||
|
|
||||
|
class VirtualRoot(TreeNode): |
||||
|
def __init__(self): |
||||
|
super().__init__(name="GLOBAL_ROOT", node_id="GLOBAL_ROOT", source="virtual") |
||||
|
|
||||
|
|
||||
|
# LCS修复接口 |
||||
|
|
||||
|
def attach_roots(root: TreeNode, table1_roots: List[TreeNode], table2_roots: List[TreeNode]): |
||||
|
for node in table1_roots + table2_roots: |
||||
|
root.add_child(node) |
||||
|
|
||||
|
# 断裂分支修复示例 |
||||
|
# def repair_branch(node1: TreeNode, node2: TreeNode): |
||||
|
# if lcs_ratio(node1.name, node2.name) > 0.8: |
||||
|
# # 合并逻辑 |
||||
|
# pass |
@ -0,0 +1,11 @@ |
|||||
|
# 层级感知清洗模块 |
||||
|
def hierarchical_normalize(node, parent_chain=[]): |
||||
|
# 保留当前层级核心词 |
||||
|
if parent_chain: |
||||
|
for parent in reversed(parent_chain): |
||||
|
node.name = node.name.replace(parent.name, "") |
||||
|
feature_words = ["局", "委员会", "集团", "公司", "银行"] |
||||
|
for word in feature_words: |
||||
|
if word in node.name: |
||||
|
return node.name.split(word)[0] + word |
||||
|
return node.name[:4] # 保底策略 |
@ -0,0 +1,12 @@ |
|||||
|
# 跨树节点匹配模块 |
||||
|
import hashlib |
||||
|
|
||||
|
|
||||
|
def path_signature(node): |
||||
|
path = [] |
||||
|
while getattr(node, 'parent', None): |
||||
|
path.append(getattr(node, 'normalized_name', node.name)[:2]) |
||||
|
node = node.parent |
||||
|
return hashlib.md5("".join(reversed(path)).encode()).hexdigest() |
||||
|
|
||||
|
# 其他匹配方法(如BERT向量、Dijkstra)可后续扩展 |
@ -0,0 +1,8 @@ |
|||||
|
# 异常检测与标记模块 |
||||
|
from difflib import SequenceMatcher |
||||
|
|
||||
|
|
||||
|
def is_duplicate(name1, name2): |
||||
|
return SequenceMatcher(None, name1, name2).ratio() > 0.95 |
||||
|
|
||||
|
# 其他异常检测函数可在此扩展 |
@ -0,0 +1,210 @@ |
|||||
|
|
||||
|
感谢补充的关键信息!基于机构树的特性(节点位置相近、名称包含上级全称/简称、存在错误和重复节点),我重新优化了处理方案。以下是根据新需求设计的**层级感知的机构树匹配方案**: |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **优化后的四阶处理框架** |
||||
|
|
||||
|
```mermaid |
||||
|
graph TD |
||||
|
A[原始数据] --> B[机构树重构] |
||||
|
B --> C[层级感知清洗] |
||||
|
C --> D[跨树节点匹配] |
||||
|
D --> E[异常检测标记] |
||||
|
``` |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **阶段1:机构树重构(1-2天)** |
||||
|
|
||||
|
**目标**:将两个表的9w节点合并为统一的树状结构 |
||||
|
|
||||
|
**关键操作**: |
||||
|
|
||||
|
1. **虚拟根节点创建**: |
||||
|
|
||||
|
```python |
||||
|
# 为两个表创建统一根节点 |
||||
|
class VirtualRoot: |
||||
|
def __init__(self): |
||||
|
self.id = "GLOBAL_ROOT" |
||||
|
self.children = [] |
||||
|
self.source = "virtual" |
||||
|
|
||||
|
# 附加原始数据 |
||||
|
root.attach(table1_root_nodes) |
||||
|
root.attach(table2_root_nodes) |
||||
|
``` |
||||
|
|
||||
|
2. **拓扑结构修复**: |
||||
|
|
||||
|
- 使用**最长公共子序列(LCS)** 算法修复断裂分支 |
||||
|
- 示例:当遇到`北京/海淀区/中关村街`和`北京市/中关村街道`时: |
||||
|
|
||||
|
```python |
||||
|
lcs_ratio = SequenceMatcher(None, "海淀区/中关村街", "北京市/中关村街道").ratio() # => 0.82 |
||||
|
if lcs_ratio > 0.8: merge_nodes() |
||||
|
``` |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **阶段2:层级感知清洗(核心创新)** |
||||
|
|
||||
|
**针对“名称包含上级全称”的特性**: |
||||
|
|
||||
|
```python |
||||
|
def hierarchical_normalize(node, parent_chain=[]): |
||||
|
# 保留当前层级核心词 |
||||
|
if parent_chain: |
||||
|
# 移除父节点名称组件(如父节点=“北京市”,当前节点=“北京市海淀区” → “海淀区”) |
||||
|
for parent in reversed(parent_chain): |
||||
|
node.name = node.name.replace(parent.name, "") |
||||
|
|
||||
|
# 提取机构特征词 |
||||
|
feature_words = ["局", "委员会", "集团", "公司", "银行"] |
||||
|
for word in feature_words: |
||||
|
if word in node.name: |
||||
|
return node.name.split(word)[0] + word |
||||
|
|
||||
|
return node.name[:4] # 保底策略 |
||||
|
``` |
||||
|
|
||||
|
**清洗规则优先级**: |
||||
|
|
||||
|
1. 行政层级优先(省→市→区→街道) |
||||
|
2. 组织类型其次(政府→企业→学校) |
||||
|
3. 长度约束(超过20字符的名称强制截断) |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **阶段3:跨树节点匹配(创新方案)** |
||||
|
|
||||
|
#### **三级匹配策略** |
||||
|
|
||||
|
| 匹配维度 | 技术实现 | 适用场景 | |
||||
|
|----------------|-----------------------------------|----------------------------| |
||||
|
| 1. 路径指纹 | 父节点链MD5哈希比对 | 相同父路径的节点 | |
||||
|
| 2. 层级向量 | 同一深度节点BERT向量聚类 | 兄弟节点匹配 | |
||||
|
| 3. 跨树桥接 | Dijkstra算法寻找最短匹配路径 | 不同分支的相似机构 | |
||||
|
|
||||
|
**路径匹配示例**: |
||||
|
|
||||
|
```python |
||||
|
# 生成节点路径指纹 |
||||
|
def path_signature(node): |
||||
|
path = [] |
||||
|
while node.parent: |
||||
|
path.append(node.normalized_name[:2]) # 取前2字作为标识 |
||||
|
node = node.parent |
||||
|
return hashlib.md5("".join(reversed(path)).encode()).hexdigest() |
||||
|
|
||||
|
# 比对示例 |
||||
|
signature1 = path_signature(node1) # 北京市/海淀区 => "e89a41c" |
||||
|
signature2 = path_signature(node2) # 北京/海淀區 => "e89a41c" # 匹配成功 |
||||
|
``` |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **阶段4:异常检测与标记** |
||||
|
|
||||
|
#### **异常类型检测矩阵** |
||||
|
|
||||
|
| 异常类型 | 检测算法 | 标记方式 | |
||||
|
|------------------|-----------------------------------|-------------------| |
||||
|
| 重复节点 | 同一父节点下名称相似度>95% | `DUPLICATE-[ID]` | |
||||
|
| 位置错误节点 | 地理坐标与父节点距离>50km | `LOC_ERR` | |
||||
|
| 名称错误节点 | 与同级节点编辑距离差异>3个标准差 | `NAME_ERR` | |
||||
|
| 跨树冲突节点 | 在不同树中父路径不一致但名称相同 | `CONFLICT` | |
||||
|
|
||||
|
**空间位置验证**(若有坐标数据): |
||||
|
|
||||
|
```python |
||||
|
from geopy.distance import great_circle |
||||
|
|
||||
|
def validate_location(child, parent): |
||||
|
if great_circle(child.coord, parent.coord).km > 50: |
||||
|
child.add_tag("POS_ERR") |
||||
|
# 自动关联最近父节点 |
||||
|
new_parent = find_nearest_parent(child, parent.siblings()) |
||||
|
child.relink(new_parent) |
||||
|
``` |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **性能优化关键措施** |
||||
|
|
||||
|
1. **层级分片处理**: |
||||
|
|
||||
|
```python |
||||
|
# 按树深度分片处理 |
||||
|
for depth in range(tree.max_depth): |
||||
|
process_layer(depth) # 并行处理同层节点 |
||||
|
``` |
||||
|
|
||||
|
2. **图数据库应用**: |
||||
|
|
||||
|
- 使用Neo4j存储机构树关系 |
||||
|
- Cypher查询示例: |
||||
|
|
||||
|
```cypher |
||||
|
MATCH (p:机构)-[:PARENT*3]->(c:机构) |
||||
|
WHERE p.name CONTAINS '北京' AND c.normalized_name =~ '税务.*' |
||||
|
RETURN p, c |
||||
|
``` |
||||
|
|
||||
|
3. **大模型分级调用**: |
||||
|
|
||||
|
```mermaid |
||||
|
graph LR |
||||
|
A[节点] --> B{匹配置信度<85%?} |
||||
|
B -->|是| C[调用Dify验证] |
||||
|
B -->|否| D[直接输出] |
||||
|
``` |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **时间与资源评估** |
||||
|
|
||||
|
| 阶段 | 工作量 | 时间 | 关键依赖 | |
||||
|
|--------------------|--------|--------|------------------------| |
||||
|
| 机构树重构 | 2人日 | 2天 | 原始数据父子关系完整性 | |
||||
|
| 层级感知清洗 | 3人日 | 3天 | 行政区划词典 | |
||||
|
| 跨树节点匹配 | 5人日 | 5天 | GPU资源(Sentence-BERT) | |
||||
|
| 异常检测与修复 | 4人日 | 4天 | 地理坐标数据(可选) | |
||||
|
| 结果验证与优化 | 3人日 | 3天 | 人工抽样数据集 | |
||||
|
| **总计** | 17人日 | **约17天** | 3人团队并行 | |
||||
|
|
||||
|
--- |
||||
|
|
||||
|
### **实施建议** |
||||
|
|
||||
|
1. **先导实验**: |
||||
|
- 选取1-2个典型子树(约500节点)验证流程 |
||||
|
- 重点测试: |
||||
|
- 名称包含上级的情况(如"北京市海淀区税务局" vs "海淀税务") |
||||
|
- 跨表重复节点(相同机构在不同表的父路径不同) |
||||
|
|
||||
|
2. **容错机制**: |
||||
|
|
||||
|
```python |
||||
|
try: |
||||
|
match_nodes() |
||||
|
except NodeConflictError as e: |
||||
|
log_conflict(e.node_id) |
||||
|
e.node.add_tag("MANUAL_REVIEW") # 打标进入人工审核队列 |
||||
|
``` |
||||
|
|
||||
|
3. **结果输出**: |
||||
|
- 生成四类报告: |
||||
|
- 机构树合并全景图 |
||||
|
- 异常节点明细表 |
||||
|
- 跨表匹配对照表 |
||||
|
- 未匹配节点分析 |
||||
|
|
||||
|
> 最终可达到的预期效果: |
||||
|
> |
||||
|
> - 95%+的标准节点自动匹配 |
||||
|
> - 85%+的异常节点自动标记 |
||||
|
> - 人工复核量控制在总节点数的5%以内 |
||||
|
|
||||
|
需要样本数据或具体代码实现细节,我可提供进一步支持! |
Loading…
Reference in new issue