{ "benchmark_name": "NER OSS Reference Benchmark", "datasets": [ { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 3.3849170431494713, "latency_p95_ms": 7.938541006296873, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.4376092275428172, "gold": 5648, "matched": 3130, "note": "", "precision": 0.36155712140464363, "predicted": 8657, "recall": 0.5541784702549575, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.4376092275428172, "gold": 5648, "matched": 3130, "note": "", "precision": 0.36155712140464363, "predicted": 8657, "recall": 0.5541784702549575, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 3453, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 1.3830829411745071, "latency_p95_ms": 2.8907079249620438, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.6042296072507553, "gold": 5648, "matched": 3400, "note": "", "precision": 0.6064930431680342, "predicted": 5606, "recall": 0.6019830028328612, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.6042296072507553, "gold": 5648, "matched": 3400, "note": "", "precision": 0.6064930431680342, "predicted": 5606, "recall": 0.6019830028328612, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 42.436250019818544, "latency_p95_ms": 183.0060000065714, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.9117021276595746, "gold": 5648, "matched": 5142, "note": "", "precision": 0.9129971590909091, "predicted": 5632, "recall": 0.9104107648725213, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.9117021276595746, "gold": 5648, "matched": 5142, "note": "", "precision": 0.9129971590909091, "predicted": 5632, "recall": 0.9104107648725213, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 42.74445818737149, "latency_p95_ms": 177.3395841009915, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.5457319288049401, "gold": 5648, "matched": 3756, "note": "", "precision": 0.46273253665147224, "predicted": 8117, "recall": 0.6650141643059491, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.5457319288049401, "gold": 5648, "matched": 3756, "note": "", "precision": 0.46273253665147224, "predicted": 8117, "recall": 0.6650141643059491, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 46.85716610401869, "latency_p95_ms": 190.9068338572979, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.49450183272242587, "gold": 5648, "matched": 3710, "note": "", "precision": 0.3964946029710377, "predicted": 9357, "recall": 0.6568696883852692, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.49450183272242587, "gold": 5648, "matched": 3710, "note": "", "precision": 0.3964946029710377, "predicted": 9357, "recall": 0.6568696883852692, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 3453, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "conll2003-en-test", "dataset_languages": [ "en" ], "dataset_name": "CoNLL-2003 EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 59.9567499011755, "latency_p95_ms": 83.46858387812972, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 3453, "status": "completed", "tracks": { "flat": { "f1": 0.35747797230804434, "gold": 5648, "matched": 1562, "note": "", "precision": 0.505338078291815, "predicted": 3091, "recall": 0.27655807365439095, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.35747797230804434, "gold": 5648, "matched": 1562, "note": "", "precision": 0.505338078291815, "predicted": 3091, "recall": 0.27655807365439095, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 5.295709008350968, "latency_p95_ms": 9.09312511794269, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.40965618141916604, "gold": 31216, "matched": 14560, "note": "", "precision": 0.36520517708437844, "predicted": 39868, "recall": 0.4664274730907227, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.4099375386866243, "gold": 31216, "matched": 14570, "note": "", "precision": 0.36545600481589247, "predicted": 39868, "recall": 0.4667478216299334, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 14423, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 2.0453750621527433, "latency_p95_ms": 3.3804578706622124, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.8108549471661862, "gold": 31216, "matched": 25323, "note": "", "precision": 0.8104916143899629, "predicted": 31244, "recall": 0.8112186058431573, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.8111431316042268, "gold": 31216, "matched": 25332, "note": "", "precision": 0.8107796696965818, "predicted": 31244, "recall": 0.811506919528447, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 108.38295891880989, "latency_p95_ms": 219.67795793898404, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.7573634797172166, "gold": 31216, "matched": 23515, "note": "", "precision": 0.7614714549399307, "predicted": 30881, "recall": 0.7532995899538698, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.7578465948435512, "gold": 31216, "matched": 23530, "note": "", "precision": 0.7619571905054888, "predicted": 30881, "recall": 0.7537801127626858, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 107.96900000423193, "latency_p95_ms": 219.30591599084437, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.5414600375817686, "gold": 31216, "matched": 19162, "note": "", "precision": 0.48434143012410585, "predicted": 39563, "recall": 0.613851870835469, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.541714350301643, "gold": 31216, "matched": 19171, "note": "", "precision": 0.48456891540075325, "predicted": 39563, "recall": 0.6141401845207586, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 117.3701249063015, "latency_p95_ms": 234.4315410591662, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.5365933885203662, "gold": 31216, "matched": 19576, "note": "", "precision": 0.46890869023665804, "predicted": 41748, "recall": 0.6271143003587903, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.5371141933008059, "gold": 31216, "matched": 19595, "note": "", "precision": 0.4693638018587717, "predicted": 41748, "recall": 0.6277229625832906, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 14423, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "wikiner-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiNER EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 68.63075005821884, "latency_p95_ms": 90.83445789292455, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 14423, "status": "completed", "tracks": { "flat": { "f1": 0.35147800208763635, "gold": 31216, "matched": 7913, "note": "", "precision": 0.5729490985446384, "predicted": 13811, "recall": 0.2534917990773962, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.3517445088502454, "gold": 31216, "matched": 7919, "note": "", "precision": 0.5733835348635146, "predicted": 13811, "recall": 0.25368400820092263, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 2.868624869734049, "latency_p95_ms": 4.7079159412533045, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.3109824730664094, "gold": 13958, "matched": 4835, "note": "", "precision": 0.28213806383847817, "predicted": 17137, "recall": 0.34639633185270097, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.3111754301334619, "gold": 13958, "matched": 4838, "note": "", "precision": 0.2823131236505806, "predicted": 17137, "recall": 0.3466112623585041, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 1.2060001026839018, "latency_p95_ms": 1.9213331397622824, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.4465504942423316, "gold": 13958, "matched": 6573, "note": "", "precision": 0.4245849751308055, "predicted": 15481, "recall": 0.4709127382146439, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.446618431332586, "gold": 13958, "matched": 6574, "note": "", "precision": 0.42464957044118595, "predicted": 15481, "recall": 0.4709843817165783, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 34.81470886617899, "latency_p95_ms": 86.46016707643867, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.49129124969701166, "gold": 13958, "matched": 7094, "note": "", "precision": 0.475437303129817, "predicted": 14921, "recall": 0.508239002722453, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.49136050417258215, "gold": 13958, "matched": 7095, "note": "", "precision": 0.4755043227665706, "predicted": 14921, "recall": 0.5083106462243875, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 35.193457966670394, "latency_p95_ms": 88.95029197447002, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.3661160317508964, "gold": 13958, "matched": 5973, "note": "", "precision": 0.3199078785281988, "predicted": 18671, "recall": 0.4279266370540192, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.36617732691777255, "gold": 13958, "matched": 5974, "note": "", "precision": 0.31996143752343204, "predicted": 18671, "recall": 0.42799828055595357, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 39.84199999831617, "latency_p95_ms": 97.02687500976026, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.379364409334861, "gold": 13958, "matched": 6291, "note": "", "precision": 0.3275197834235735, "predicted": 19208, "recall": 0.4507092706691503, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.3794247120545136, "gold": 13958, "matched": 6292, "note": "", "precision": 0.32757184506455644, "predicted": 19208, "recall": 0.4507809141710847, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are pl but dataset languages are en.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are pl but dataset languages are en." ] }, { "dataset_id": "wikiann-en-test", "dataset_languages": [ "en" ], "dataset_name": "WikiANN EN Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 59.95599995367229, "latency_p95_ms": 72.4445830564946, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.26857654431512984, "gold": 13958, "matched": 3000, "note": "", "precision": 0.35790980672870437, "predicted": 8382, "recall": 0.21493050580312364, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.26857654431512984, "gold": 13958, "matched": 3000, "note": "", "precision": 0.35790980672870437, "predicted": 8382, "recall": 0.21493050580312364, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 20456, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 6.377791054546833, "latency_p95_ms": 10.64100000075996, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 20456, "status": "completed", "tracks": { "flat": { "f1": 0.5269768769626035, "gold": 37119, "matched": 21229, "note": "", "precision": 0.48858457997698507, "predicted": 43450, "recall": 0.5719173469112854, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.5271009941789025, "gold": 37119, "matched": 21234, "note": "", "precision": 0.4886996547756041, "predicted": 43450, "recall": 0.5720520488159703, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 1.6792500391602516, "latency_p95_ms": 2.8093750588595867, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 20456, "status": "completed", "tracks": { "flat": { "f1": 0.8569497390975208, "gold": 37119, "matched": 31696, "note": "", "precision": 0.8600189933523267, "predicted": 36855, "recall": 0.8539023141787225, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.8569767756238679, "gold": 37119, "matched": 31697, "note": "", "precision": 0.8600461267127933, "predicted": 36855, "recall": 0.8539292545596595, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 20456, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 20456, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 20456, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 104.33958400972188, "latency_p95_ms": 234.2710830271244, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 20456, "status": "completed", "tracks": { "flat": { "f1": 0.6656775348546344, "gold": 37119, "matched": 27144, "note": "", "precision": 0.6108835576360445, "predicted": 44434, "recall": 0.7312697001535602, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.6657511066423063, "gold": 37119, "matched": 27147, "note": "", "precision": 0.6109510735022731, "predicted": 44434, "recall": 0.7313505212963711, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiner-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiNER PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 68.18033312447369, "latency_p95_ms": 92.0476249884814, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 20456, "status": "completed", "tracks": { "flat": { "f1": 0.5242986562489476, "gold": 37119, "matched": 15568, "note": "", "precision": 0.6991512103112228, "predicted": 22267, "recall": 0.419407850427005, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.5242986562489476, "gold": 37119, "matched": 15568, "note": "", "precision": 0.6991512103112228, "predicted": 22267, "recall": 0.419407850427005, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 4.66324994340539, "latency_p95_ms": 7.1573329623788595, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.46310821128618035, "gold": 14960, "matched": 6672, "note": "", "precision": 0.4815937635339974, "predicted": 13854, "recall": 0.4459893048128342, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.4631776219893108, "gold": 14960, "matched": 6673, "note": "", "precision": 0.4816659448534719, "predicted": 13854, "recall": 0.4460561497326203, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 1.2502921745181084, "latency_p95_ms": 1.9956659525632858, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.6103453974549662, "gold": 14960, "matched": 9233, "note": "", "precision": 0.6036613272311213, "predicted": 15295, "recall": 0.6171791443850267, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.6103453974549662, "gold": 14960, "matched": 9233, "note": "", "precision": 0.6036613272311213, "predicted": 15295, "recall": 0.6171791443850267, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 10000, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 43.88662497512996, "latency_p95_ms": 107.17587498947978, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.655582419766171, "gold": 14960, "matched": 9897, "note": "", "precision": 0.6497078710693889, "predicted": 15233, "recall": 0.6615641711229947, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.655582419766171, "gold": 14960, "matched": 9897, "note": "", "precision": 0.6497078710693889, "predicted": 15233, "recall": 0.6615641711229947, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "wikiann-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "WikiANN PL Test", "dataset_supported_tracks": [ "flat", "surface" ], "latency_p50_ms": 60.57558301836252, "latency_p95_ms": 74.47354099713266, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 10000, "status": "completed", "tracks": { "flat": { "f1": 0.425506402295405, "gold": 14960, "matched": 4968, "note": "", "precision": 0.5920629245620308, "predicted": 8391, "recall": 0.3320855614973262, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Dataset does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.425506402295405, "gold": 14960, "matched": 4968, "note": "", "precision": 0.5920629245620308, "predicted": 8391, "recall": 0.3320855614973262, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "spacy-en", "provider_name": "spaCy EntityRecognizer EN", "sample_count": 1828, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 61.39574991539121, "latency_p95_ms": 74.51149984262884, "provider_id": "spacy-pl", "provider_name": "spaCy EntityRecognizer PL", "sample_count": 1828, "status": "completed", "tracks": { "flat": { "f1": 0.6315023140524519, "gold": 33984, "matched": 18216, "note": "", "precision": 0.7683806470662673, "predicted": 23707, "recall": 0.5360169491525424, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.632334332911546, "gold": 33984, "matched": 18240, "note": "", "precision": 0.7693930062850635, "predicted": 23707, "recall": 0.536723163841808, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 16.010332852602005, "latency_p95_ms": 19.34016589075327, "provider_id": "spacy-xx", "provider_name": "spaCy EntityRecognizer XX", "sample_count": 1828, "status": "completed", "tracks": { "flat": { "f1": 0.37211658724253116, "gold": 33984, "matched": 11752, "note": "", "precision": 0.4027554062853422, "predicted": 29179, "recall": 0.3458097928436911, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.3728131975998607, "gold": 33984, "matched": 11774, "note": "", "precision": 0.4035093731793413, "predicted": 29179, "recall": 0.3464571563088512, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en", "provider_name": "Flair NER EN", "sample_count": 1828, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "flair-en-ontonotes", "provider_name": "Flair NER EN OntoNotes", "sample_count": 1828, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 0.0, "latency_p95_ms": 0.0, "provider_id": "stanza-en", "provider_name": "Stanza NER EN", "sample_count": 1828, "status": "skipped", "tracks": { "flat": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": null, "gold": null, "matched": null, "note": "Skipped: provider languages are en but dataset languages are pl.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "surface" } }, "warnings": [ "Skipped: provider languages are en but dataset languages are pl." ] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 581.5466251224279, "latency_p95_ms": 974.2245830129832, "provider_id": "stanza-pl", "provider_name": "Stanza NER PL", "sample_count": 1828, "status": "completed", "tracks": { "flat": { "f1": 0.7317553559087768, "gold": 33984, "matched": 21177, "note": "", "precision": 0.8862152661533311, "predicted": 23896, "recall": 0.623146186440678, "supported": true, "track": "flat" }, "nested": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "nested" }, "overlap": { "f1": null, "gold": null, "matched": null, "note": "Provider does not support this track.", "precision": null, "predicted": null, "recall": null, "supported": false, "track": "overlap" }, "surface": { "f1": 0.7323082239115409, "gold": 33984, "matched": 21193, "note": "", "precision": 0.8868848342818881, "predicted": 23896, "recall": 0.6236169962335216, "supported": true, "track": "surface" } }, "warnings": [] }, { "dataset_id": "poleval2018-pl-test", "dataset_languages": [ "pl" ], "dataset_name": "PolEval-2018 PL Test", "dataset_supported_tracks": [ "flat", "overlap", "nested", "surface" ], "latency_p50_ms": 307.2775830514729, "latency_p95_ms": 449.4479577988386, "provider_id": "gliner-multi", "provider_name": "GLiNER Multi", "sample_count": 1828, "status": "completed", "tracks": { "flat": { "f1": 0.16610637687957677, "gold": 33984, "matched": 3187, "note": "", "precision": 0.7261335156071999, "predicted": 4389, "recall": 0.09377942561205273, "supported": true, "track": "flat" }, "nested": { "f1": 0.0, "gold": 14897, "matched": 0, "note": "", "precision": 0.0, "predicted": 0, "recall": 0.0, "supported": true, "track": "nested" }, "overlap": { "f1": 0.17554009329476453, "gold": 33984, "matched": 3368, "note": "", "precision": 0.7673729778992937, "predicted": 4389, "recall": 0.09910546139359698, "supported": true, "track": "overlap" }, "surface": { "f1": 0.16621061683996563, "gold": 33984, "matched": 3189, "note": "", "precision": 0.7265892002734108, "predicted": 4389, "recall": 0.0938382768361582, "supported": true, "track": "surface" } }, "warnings": [ "Sentence of length 1217 has been truncated to 384", "Sentence of length 1222 has been truncated to 384", "Sentence of length 1243 has been truncated to 384", "Sentence of length 1250 has been truncated to 384", "Sentence of length 1252 has been truncated to 384", "Sentence of length 1259 has been truncated to 384", "Sentence of length 1295 has been truncated to 384", "Sentence of length 1303 has been truncated to 384", "Sentence of length 1355 has been truncated to 384", "Sentence of length 1378 has been truncated to 384", "Sentence of length 1579 has been truncated to 384", "Sentence of length 1681 has been truncated to 384", "Sentence of length 1763 has been truncated to 384", "Sentence of length 1967 has been truncated to 384", "Sentence of length 1995 has been truncated to 384", "Sentence of length 2025 has been truncated to 384", "Sentence of length 2104 has been truncated to 384", "Sentence of length 2390 has been truncated to 384", "Sentence of length 2535 has been truncated to 384", "Sentence of length 2642 has been truncated to 384", "Sentence of length 2645 has been truncated to 384" ] } ], "generated_at": "2026-06-05T14:33:28.697085+00:00", "machine": { "cpu_count": 8, "hostname": "MacBookPro", "machine": "arm64", "platform": "macOS-26.1-arm64-arm-64bit", "processor": "Apple M1 Pro", "python_version": "3.12.10" }, "task": "ner", "tracks": [ "flat", "overlap", "nested", "surface" ] }