diff --git a/main.py b/main.py index f8c8877..67a3e3c 100755 --- a/main.py +++ b/main.py @@ -25,6 +25,7 @@ pattern = r"(\d{4}\.\d{2})--(\d{4}\.\d{2}|至今)\s*([^\n]+)" # 查找所有匹配项 matches = re.findall(pattern, text) + # 定义标签规则 def get_label_and_details(content): if "学习" in content: @@ -70,6 +71,7 @@ def get_label_and_details(content): details = {} return "工作经历", details + # 整理结果 results = [] for match in matches: diff --git a/scripts/train.py b/scripts/train.py index 8809e80..5f0ff8a 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -8,12 +8,14 @@ from transformers import ( DataCollatorForTokenClassification ) + # 加载数据 def load_data(file_path): with open(file_path, "r", encoding="utf-8") as f: data = json.load(f) return data + # 数据预处理 def tokenize_and_align_labels(examples, tokenizer): tokenized_inputs = tokenizer(examples["text"], truncation=True, padding=True, is_split_into_words=True) @@ -32,6 +34,7 @@ def tokenize_and_align_labels(examples, tokenizer): tokenized_inputs["labels"] = labels return tokenized_inputs + # 标签映射 label2id = { "TIME": 0,