Files: opus2text.py

#!/bin/env python

import os, sys, re
from popen2 import *

prefix = 'xml'
lang_re = re.compile('^(\w*)-(\w*)$')

if len(sys.argv) == 2:
    steps = eval(sys.argv[1])   
else:
    steps = range(1, 8)

# Searches path for .gz files and decompresses them.
def gunzip(path):
    for item in os.listdir(path):
        item = os.path.join(path, item)
        if os.path.isdir(item):
            gunzip(item)
        else:
            if item[-3:] == '.gz':
                command = 'gunzip %s' % (item,)
                print command
                os.system(command)

# Converts list fo XCES files to single plain text file.
def xces2text(files, corpus, lang1, lang2):
    lang1_file = open('%s.%s' % (corpus, lang1), 'w+')
    lang2_file = open('%s.%s' % (corpus, lang2), 'w+')
    for file in files:
        command = '../bin/xces2text.perl < %s' % (file)
        print command
        in_file, out_file = popen2(command)
        is_lang1 = True
        for line in in_file.readlines():
            if line.startswith('- '):
                line = line[2:]
            if is_lang1:
                lang1_file.write(line.lower())
            else:
                lang2_file.write(line.lower())
            is_lang1 = not is_lang1
        in_file.close()
        out_file.close()
    lang1_file.close()
    lang2_file.close()

for lang in os.listdir(prefix):
    match = lang_re.match(lang)
    if match:
        lang1 = match.group(1)
        lang2 = match.group(2)

        # Decompress sentence align files.
        gunzip(os.path.join(prefix, lang))

        # Build list of sentence align files between two languages.
        files = []
        os.chdir(os.path.join(prefix, lang))
        for genre in os.listdir('.'):
            os.chdir(genre)
            for year in os.listdir('.'):
                os.chdir(year)
                for file in os.listdir('.'):
                  files.append(os.path.join(prefix, lang, genre, year, file))
                os.chdir('..')
            os.chdir('..')
        os.chdir('../..')

        xces2text(files, '%s' % (lang,), lang1, lang2)