How to convert annotated text in XML to CONLL?

Question

I need to preprocess XML files for a NER task and I am struggling with the conversion of the XML files. I guess there is a nice and easy way to solve the following problem.

Given an annotated text in XML with the following structure as input:

<doc>
   Some <tag1>annotated text</tag1> in <tag2>XML</tag2>.
</doc>

I want a CoNLL file in IOB2 tagging format as follows as output:

Some          O
annotated     B-TAG1
text          I-TAG1
in            O
XML           B-TAG2
.             O

Answer 1

Let's convert your XML file to a TXT (called 'read.txt') as follows:

<doc>
   Some <tag1>annotated text</tag1> in <tag2>Tag2 entity</tag2> <tag1>tag1 entity</tag1>.
   Some <tag3>annotated text</tag3> in <tag2>XML</tag2>!
</doc>

Then using regex and several if-else conditions the below code return 'output.txt' file in CONNL format as you want.

import re

sentences, connl = [], []

with open('read.txt', 'r', encoding='utf-8') as file:
    for line in file:
        line = line.strip()
        if line not in ['<doc>', '</doc>']:
            sentences.append(line)

for sentence in sentences:
    tag1 = re.findall(r'<tag1>(.+?)</tag1>', sentence)
    tag2 = re.findall(r'<tag2>(.+?)</tag2>', sentence)
    tag3 = re.findall(r'<tag3>(.+?)</tag3>', sentence)
    splitted = re.split('<tag1>|</tag1>|<tag2>|</tag2>|<tag3>|</tag3>', sentence)  # splitted considering tags
    if tag1 or tag2 or tag3:  # if any tag in sentence
        for split in splitted:  # search each index
            if split in tag1:
                counter = 0
                for token in split.split():
                    if counter > 0:
                        connl.append(token + ' I-TAG1')
                    else:
                        connl.append(token + ' B-TAG1')
                    counter += 1

            elif split in tag2:
                counter = 0
                for token in split.split():
                    if counter > 0:
                        connl.append(token + ' I-TAG2')
                    else:
                        connl.append(token + ' B-TAG2')
                    counter += 1

            elif split in tag3:
                counter = 0
                for token in split.split():
                    if counter > 0:
                        connl.append(token + ' I-TAG3')
                    else:
                        connl.append(token + ' B-TAG3')
                    counter += 1

            else:  # current word is not an entity
                for token in split.split():
                    connl.append(token + ' O')

    else:  # if no entity in sentence
        for word in sentence.split():
            connl.append(word + ' O')

    connl.append('')

with open('output.txt', 'w', encoding='utf-8') as output:
    for element in connl:
        output.write(element + "\n")

output.txt:

Some O
annotated B-TAG1
text I-TAG1
in O
XML B-TAG2
other B-TAG1
tag I-TAG1
. O

Some O
annotated B-TAG3
text I-TAG3
in O
XML B-TAG2
! O

Answer 2

Alternatively, you could use a XML parser such as lxml or BeautifulSoup and spaCy as tokenizer.

pip install lxml
pip install spacy
python3 -m spacy download en_core_web_sm

An example:

from lxml import etree
from io import StringIO
import re
import spacy

DISABLED = [
    "ner", "tok2vec", "tagger", "parser", "attribute_ruler", "lemmatizer"]

def type_to_iob(enttype, idx):
    mapping = {
        "tag1": "TAG1",
        "tag2": "TAG2",
        "tag3": "TAG3",
    }
    iob = 'B' if idx == 0 else 'I'
    return '{}_{}'.format(iob, mapping.get(enttype))

def transform_to_iob(item):
    tokens = list(nlp(item.text, disable=DISABLED))
    return [
        (ent, type_to_iob(item.tag, idx))
        for idx, ent in enumerate(tokens)
    ]

xmltext = """<doc>
   Some example of <tag1>annotated text</tag1> in <tag2>XML</tag2>.
   Some other sample of <tag3>another annotated text</tag3> in <tag2>XML</tag2>!
</doc>"""

nlp = spacy.load("en_core_web_sm")
nlp.add_pipe('sentencizer')

tree = etree.parse(StringIO(xmltext))
for item in tree.getroot().xpath('/doc/node()'):
    if isinstance(item, etree._ElementUnicodeResult):
        doc = nlp(str(item).replace("\n", "").strip(), disable=DISABLED)
        for sentence in doc.sents:
            for token in sentence:
                if re.match(r'\s*$', str(token)):
                    print()
                    continue
                print(f"{token} O")
    elif isinstance(item, etree._Element):
        for iob_tag in transform_to_iob(item):
            print(f'{iob_tag[0]} {iob_tag[1]}')

Result:

❯ python3 test.py
Some O
example O
of O
annotated B_TAG1
text I_TAG1
in O
XML B_TAG2
. O

Some O
other O
sample O
of O
another B_TAG3
annotated I_TAG3
text I_TAG3
in O
XML B_TAG2
! O