machine-learning - 训练空白 spacy 模型后无法获得任何实体
问题描述
我一直在研究法律数据以识别法律文件中的自定义实体。我正在我清理的训练数据集(spacy 文档中提到的 JSON 格式)上训练空 spacy 模型。我试图从训练数据集中删除标点符号、特殊字符、括号等。我已经用名称“dictkey”标记了新实体,并在它的 JSON 中提到了 startindex 和 endindex,以便创建训练数据集。下面是训练数据集的链接和我正在使用的代码。如果该数据集有任何问题或需要进一步清理,您能否查看随附的训练数据集?
https://techmailer.online/TRAIN_DATA3json.txt
https://techmailer.online/train_ner%20-%20Copy.py
from __future__ import unicode_literals, print_function
import plac
import random
from pathlib import Path
import spacy
from spacy.util import minibatch, compounding
import re
#import createtrainingdataset_updated_v2 as traindataset
# training data
#TRAIN_DATA = [
# ("Who is Shaka Khan?", {"entities": [(7, 17, "PERSON")]}),
# ("I like London and Berlin.", {"entities": [(7, 13, "LOC"), (18, 24, "LOC")]}),
#]
with open('/Users/modis1/Desktop/24-01/TRAIN_DATA3json.txt', 'r', encoding='utf-8') as openfile:
TRAIN_DATA = openfile.read()
@plac.annotations(
model=("Model name. Defaults to blank 'en' model.", "option", "m", str),
output_dir=("Optional output directory", "option", "o", Path),
n_iter=("Number of training iterations", "option", "n", int),
)
def main(model=None, output_dir='/Users/A-GUPTA50/Desktop/ShubhamModi/NewSavedModel/', n_iter=100):
"""Load the model, set up the pipeline and train the entity recognizer."""
if model is not None:
nlp = spacy.load(model) # load existing spaCy model
print("Loaded model '%s'" % model)
else:
nlp = spacy.blank("en") # create blank Language class
print("Created blank 'en' model")
# create the built-in pipeline components and add them to the pipeline
# nlp.create_pipe works for built-ins that are registered with spaCy
if "ner" not in nlp.pipe_names:
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner, last=True)
# otherwise, get it so we can add labels
else:
ner = nlp.get_pipe("ner")
# add labels
for _, annotations in TRAIN_DATA:
for ent in annotations.get("entities"):
ner.add_label(ent[2])
# get names of other pipes to disable them during training
pipe_exceptions = ["ner", "trf_wordpiecer", "trf_tok2vec"]
other_pipes = [pipe for pipe in nlp.pipe_names if pipe not in pipe_exceptions]
with nlp.disable_pipes(*other_pipes): # only train NER
# reset and initialize the weights randomly – but only if we're
# training a new model
if model is None:
nlp.begin_training()
for itn in range(n_iter):
random.shuffle(TRAIN_DATA)
losses = {}
# batch up the examples using spaCy's minibatch
batches = minibatch(TRAIN_DATA, size=compounding(4.0, 32.0, 1.001))
for batch in batches:
texts, annotations = zip(*batch)
nlp.update(
texts, # batch of texts
annotations, # batch of annotations
drop=0.5, # dropout - make it harder to memorise data
losses=losses,
)
print("Losses", losses)
# test the trained model
for text, _ in TRAIN_DATA:
# text = re.sub(r'\\u\d{4,}','', text.rstrip())
doc = nlp(text)
print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])
# save model to output directory
if output_dir is not None:
output_dir = Path(output_dir)
if not output_dir.exists():
output_dir.mkdir()
nlp.to_disk(output_dir)
print("Saved model to", output_dir)
# test the saved model
print("Loading from", output_dir)
nlp2 = spacy.load(output_dir)
for text, _ in TRAIN_DATA:
doc = nlp2(text)
print("Entities", [(ent.text, ent.label_) for ent in doc.ents])
print("Tokens", [(t.text, t.ent_type_, t.ent_iob) for t in doc])
if __name__ == "__main__":
plac.call(main)
解决方案
推荐阅读
- c++ - 查找以文件类型打包的图像文件。(二进制)
- jwt - 如何将 jsonwebtoken (JWT) 添加到请求标头?
- java - 如何在 Java 中使用“minOccurs”验证 xml?
- java - Artemis-2.6.3 控制台:服务不可用
- entity-framework-core - 在 EFCore 中使用单个数据库处理多个 DBContext 的通用标识和迁移
- python-3.x - scatter_kws 和 line_kws 在 seaborn lmplot 中做了什么
- oracle - 如何将图像保存在 Oracle 的目录文件夹中(不是数据库中的 Blob)
- hangouts-api - 如何使用谷歌 API 从环聊直接拨打视频电话到另一封电子邮件
- dart - 如何获取有关“libphonenumber”的飞镖语言源代码?
- angular - 如何使用角度 6 中的循环选择下拉列表?