From 2fdc8ca4a521693dc59975f30dc1509412b56ac5 Mon Sep 17 00:00:00 2001 From: Viviman Date: Sat, 11 Jan 2025 13:48:28 +0800 Subject: [PATCH] =?UTF-8?q?=E6=97=A0=E6=B3=95=E7=BB=A7=E7=BB=AD=E5=AE=9E?= =?UTF-8?q?=E7=8E=B0=EF=BC=8C=E9=9C=80=E8=A6=81=E7=BB=A7=E7=BB=AD=E7=A0=94?= =?UTF-8?q?=E7=A9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 2 ++ scripts/train.py | 3 +++ 2 files changed, 5 insertions(+) diff --git a/main.py b/main.py index f8c8877..67a3e3c 100755 --- a/main.py +++ b/main.py @@ -25,6 +25,7 @@ pattern = r"(\d{4}\.\d{2})--(\d{4}\.\d{2}|至今)\s*([^\n]+)" # 查找所有匹配项 matches = re.findall(pattern, text) + # 定义标签规则 def get_label_and_details(content): if "学习" in content: @@ -70,6 +71,7 @@ def get_label_and_details(content): details = {} return "工作经历", details + # 整理结果 results = [] for match in matches: diff --git a/scripts/train.py b/scripts/train.py index 8809e80..5f0ff8a 100755 --- a/scripts/train.py +++ b/scripts/train.py @@ -8,12 +8,14 @@ from transformers import ( DataCollatorForTokenClassification ) + # 加载数据 def load_data(file_path): with open(file_path, "r", encoding="utf-8") as f: data = json.load(f) return data + # 数据预处理 def tokenize_and_align_labels(examples, tokenizer): tokenized_inputs = tokenizer(examples["text"], truncation=True, padding=True, is_split_into_words=True) @@ -32,6 +34,7 @@ def tokenize_and_align_labels(examples, tokenizer): tokenized_inputs["labels"] = labels return tokenized_inputs + # 标签映射 label2id = { "TIME": 0,