|
|
@ -8,12 +8,14 @@ from transformers import ( |
|
|
|
DataCollatorForTokenClassification |
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
# 加载数据 |
|
|
|
def load_data(file_path): |
|
|
|
with open(file_path, "r", encoding="utf-8") as f: |
|
|
|
data = json.load(f) |
|
|
|
return data |
|
|
|
|
|
|
|
|
|
|
|
# 数据预处理 |
|
|
|
def tokenize_and_align_labels(examples, tokenizer): |
|
|
|
tokenized_inputs = tokenizer(examples["text"], truncation=True, padding=True, is_split_into_words=True) |
|
|
@ -32,6 +34,7 @@ def tokenize_and_align_labels(examples, tokenizer): |
|
|
|
tokenized_inputs["labels"] = labels |
|
|
|
return tokenized_inputs |
|
|
|
|
|
|
|
|
|
|
|
# 标签映射 |
|
|
|
label2id = { |
|
|
|
"TIME": 0, |
|
|
|