"""This includes function for scoring models applied to a SpaCy corpus."""from__future__importannotationsfromcollections.abcimportIterablefromcopyimportcopyfromfunctoolsimportpartialfromtimeimporttime# type: ignorefromtypingimportCallableimportpandasaspdfromspacy.languageimportLanguagefromspacy.scorerimportScorerfromspacy.tokensimportDoc,Spanfromspacy.trainingimportExamplefromspacy.training.augmentimportdont_augmentfromspacy.training.corpusimportCorpusfrom..utilsimportflatten_dict
[docs]defno_misc_getter(doc:Doc,attr:str)->Iterable[Span]:# type: ignore"""A utility getter for scoring entities without including MISC. Args: doc (Doc): a SpaCy Doc attr (str): attribute to be extracted Returns: Iterable[Span] """spans=getattr(doc,attr)# type: ignoreforspaninspans:ifspan.label_=="MISC":continueyieldspan
[docs]defscore(# noqacorpus:Corpus,apply_fn:Callable[[Iterable[Example],list[Example]]]|Language,# type: ignorescore_fn:list[Callable[[Iterable[Example]],dict]|str]=[# noqa"token","pos","ents","dep",],augmenters:list[Callable[[Language,Example],Iterable[Example]]]=[],# noqak:int=1,nlp:Language|None=None,**kwargs,# noqa)->pd.DataFrame:"""scores a models performance on a given corpus with potentially augmentations applied to it. Args: corpus (Corpus): A spacy Corpus apply_fn (Union[Callable, Language]): A wrapper function for the model you wish to score. The model should take in a list of spacy Examples (Iterable[Example]) and output a tagged version of it (Iterable[Example]). A SpaCy pipeline (Language) can be provided as is. score_fn (list[Union[Callable[[Iterable[Example]], dict], str]], optional): A scoring function which takes in a list of examples (Iterable[Example]) and return a dictionary of performance scores. Four potiential strings are valid. "ents" for measuring the performance of entity spans. "pos" for measuring the performance of fine-grained (tag_acc), and coarse-grained (pos_acc) pos-tags. "token" for measuring the performance of tokenization. "dep" for measuring the performance of dependency parsing. "nlp" for measuring the performance of all components in the specified nlp pipeline. Defaults to ["token", "pos", "ents", "dep"]. augmenters (list[Callable[[Language, Example], Iterable[Example]]], optional): A spaCy style augmenters which should be applied to the corpus or a list thereof. defaults to [], indicating no augmenters. k (int, optional): Number of times it should run the augmentation and test the performance on the corpus. Defaults to 1. nlp (Optional[Language], optional): A spacy processing pipeline. If None it will use an empty Danish pipeline. Defaults to None. Used for loading the calling the corpus. Returns: pandas.DataFrame: returns a pandas dataframe containing the performance metrics. Example: >>> from spacy.training.augment import create_lower_casing_augmenter >>> from dacy.datasets import dane >>> test = dane(splits=["test") >>> nlp = dacy.load("da_dacy_small_tft-0.0.0") >>> scores = score(test, augmenter=[create_lower_casing_augmenter(0.5)], >>> apply_fn = nlp) """ifcallable(augmenters):augmenters=[augmenters]iflen(augmenters)==0:augmenters=[dont_augment]def__apply_nlp(examples):# noqa: ANN001examples=((e.x.text,e.y)foreinexamples)doc_tuples=nlp_.pipe(examples,as_tuples=True)return[Example(x,y)forx,yindoc_tuples]ifisinstance(apply_fn,Language):nlp_=apply_fnapply_fn=__apply_nlp# type: ignoreifnlpisNone:fromspacy.lang.daimportDanishnlp=Danish()scorer=Scorer(nlp)defents_scorer(examples):# noqa: ANN001scores=Scorer.score_spans(examples,attr="ents")scores_no_misc=Scorer.score_spans(examples,attr="ents",getter=no_misc_getter,)scores["ents_excl_MISC"]={k:scores_no_misc[k]forkin["ents_p","ents_r","ents_f"]}returnscoresdefpos_scorer(examples):# noqa: ANN001scores=Scorer.score_token_attr(examples,attr="pos")scores_=Scorer.score_token_attr(examples,attr="tag")forkinscores_:scores[k]=scores_[k]returnscoresdef_scorers={"ents":ents_scorer,"pos":pos_scorer,"token":Scorer.score_tokenization,"nlp":scorer.score,"dep":partial(Scorer.score_deps,attr="dep",getter=dep_getter,ignore_labels=("p","punct"),),}def__score(augmenter):# noqa: ANN001corpus_=copy(corpus)corpus_.augmenter=augmenterscores_ls=[]for_iinrange(k):# type: ignores=time()examples=apply_fn(corpus_(nlp))# type: ignorespeed=time()-sscores={"wall_time":speed}forfninscore_fn:ifisinstance(fn,str):fn=def_scorers[fn]# noqascores.update(fn(examples))# type: ignorescores=flatten_dict(scores)scores_ls.append(scores)# and collapse list to dictforkeyinscores:# type: ignorescores[key]=[s.get(key,None)forsinscores_ls]# type: ignorescores["k"]=list(range(k))# type: ignorereturnpd.DataFrame(scores)# type: ignorefori,auginenumerate(augmenters):scores_=__score(aug)scores=pd.concat([scores,scores_])ifi!=0elsescores_# type: ignore # noqareturnscores# type: ignore