Files: train.py

#!/bin/env python

import os, sys, re

if len(sys.argv) == 2:
    steps = eval(sys.argv[1])   
else:
    steps = range(1, 9)

lang_re = re.compile('^(\w*)-(\w*)$')
prefix = '/pub/tmp/xbarto33'
bin_prefix = os.path.join(prefix, 'bin')
corpus_prefix = os.path.join(prefix, 'opus')
languages = ['cze-eng']

def tokenize(file):     
    command = '%s -l en < %s > tmp' % (os.path.join(bin_prefix,
      'tokenizer.perl'), file)
    print command
    os.system(command)
    command = 'mv tmp %s' % (file,)
    print command
    os.system(command)

def detokenize(file):
    command = '%s -l en < %s > tmp' % (os.path.join(bin_prefix,
      'tokenizer.perl'), file)
    print command
    os.system(command)
    command = 'mv tmp %s' % (file,)
    print command
    os.system(command)

def clean(corpus, lang1, lang2):
    command = '%s %s %s %s tmp 1 50' % (os.path.join(bin_prefix,
      'clean-corpus-n.perl'), corpus, lang1, lang2)
    print command
    os.system(command)
    command = 'mv tmp.%s %s.%s' % (lang1, corpus, lang1)
    print command
    os.system(command)
    command = 'mv tmp.%s %s.%s' % (lang2, corpus, lang2)
    print command
    os.system(command)

for lang in languages:
    match = lang_re.match(lang)
    if match:
        lang1 = match.group(1)
        lang2 = match.group(2)

        # Prepare corpus.
        if 1 in steps:
            lang1_file = open('%s.%s' % (lang, lang1), 'r')
            line_count = 0
            for line in lang1_file.readlines():
                line_count = line_count + 1
            lang1_file.close()
            
            devel_index = int(0.90 * line_count)
            test_index = int(0.95 * line_count)
            
            lang1_file = open('%s.%s' % (lang, lang1), 'r')
            lang2_file = open('%s.%s' % (lang, lang2), 'r')
                    
            # Build traing dataset
            out1_file = open('%s-train.%s' % (lang, lang1), 'w+')
            out2_file = open('%s-train.%s' % (lang, lang2), 'w+')
            for I in xrange(0, devel_index):
                out1_file.write(lang1_file.readline())
                out2_file.write(lang2_file.readline())
            out1_file.close()
            out2_file.close()

            # Build development dataset
            out1_file = open('%s-devel.%s' % (lang, lang1), 'w+')
            out2_file = open('%s-devel.%s' % (lang, lang2), 'w+')
            for I in xrange(devel_index, test_index):
                out1_file.write(lang1_file.readline())
                out2_file.write(lang2_file.readline())
            out1_file.close()
            out2_file.close()

            # Build test dataset
            out1_file = open('%s-test.%s' % (lang, lang1), 'w+')
            out2_file = open('%s-test.%s' % (lang, lang2), 'w+')
            for I in xrange(test_index, line_count):
                out1_file.write(lang1_file.readline())
                out2_file.write(lang2_file.readline())
            out1_file.close()
            out2_file.close()

            lang1_file.close()
            lang2_file.close()

        # Separate words in sentences.
        if 2 in steps:
            tokenize('%s-train.%s' % (lang, lang1))
            tokenize('%s-train.%s' % (lang, lang2))
            tokenize('%s-devel.%s' % (lang, lang1))
            tokenize('%s-devel.%s' % (lang, lang2))
            tokenize('%s-test.%s' % (lang, lang1))
            tokenize('%s-test.%s' % (lang, lang2))

        # Clean corpus from empty lines and sentences with more than 50 words.
        if 3 in steps:
            clean('%s-train' % (lang,), lang1, lang2)
            clean('%s-devel' % (lang,), lang1, lang2)
            clean('%s-test' % (lang,), lang1, lang2)

        # Create n-gram language models
        if 4 in steps:
            command = '%s -order 5 -interpolate -kndiscount -text %s-train.%s' \
              ' -lm %s.%s.lm' % (os.path.join(bin_prefix, 'ngram-count'), 
              lang, lang2, lang, lang2)
            print command
            os.system(command)
            command = '%s -order 5 -interpolate -kndiscount -text %s-train.%s' \
              ' -lm %s.%s.lm' % (os.path.join(bin_prefix, 'ngram-count'), 
              lang, lang1, lang, lang1)
            print command
            os.system(command)

        # Train model with Moses.
        if 5 in steps:
             command = '%s -scripts-root-dir %s -root-dir %s -corpus %s-train' \
               ' -f %s -e %s -alignment grow-diag-final-and -reordering' \
               ' msd-bidirectional-fe -lm 0:5:%s.%s.lm:0 -parallel' % \
               (os.path.join(bin_prefix, 'training', 'train-factored-phrase-model.perl'),
               bin_prefix, os.path.join(corpus_prefix, 'working-dir'),  lang,
               lang1, lang2, os.path.join(os.getcwd(), lang, lang2))
             print command
             os.system(command)

        # Tune parameters.
        if 6 in steps:
            command = '%s %s-devel.%s %s-devel.%s %s %s --working-dir %s' \
              ' --rootdir %s' % (os.path.join(bin_prefix, 'training',
              'mert-moses.pl'), lang, lang1, lang, lang2,
              os.path.join(bin_prefix, 'moses'), os.path.join(corpus_prefix,
             'working-dir', 'model', 'moses.ini'), os.path.join(corpus_prefix,
             'working-dir', 'tuning'), bin_prefix)
            print command
            os.system(command)
            command = '%s %s < %s > %s' % (os.path.join(bin_prefix,
              'reuse-weights.perl'), os.path.join(corpus_prefix, 'working-dir',
              'tuning', 'moses.ini'), os.path.join(corpus_prefix, 'working-dir',
              'model', 'moses.ini'), os.path.join(corpus_prefix, 'working-dir',
              'model', 'moses-tuned.ini'))
            print command
            os.system(command)

        # Translate test dataset.
        if 7 in steps:
            command = '%s -f %s -input-file %s-test.%s > %s-translated-normal.%s' \
              % (os.path.join(bin_prefix, 'moses'), os.path.join(corpus_prefix,
              'working-dir', 'model', 'moses.ini'), lang, lang1, lang, lang2)
            print command
            os.system(command)
            command = '%s -f %s -input-file %s-test.%s > %s-translated-tuned.%s' \
              % (os.path.join(bin_prefix, 'moses'), os.path.join(corpus_prefix,
              'working-dir', 'model', 'moses-tuned.ini'), lang, lang1, lang, lang2)
            print command
            os.system(command)
               
        # Compute BLEU
        if 8 in steps:
            command = '%s %s-test.%s < %s-translated-normal.%s' % (
              os.path.join(bin_prefix, 'multi-bleu.perl'), lang, lang2, lang, lang2)
            print command
            os.system(command)
            command = '%s %s-test.%s < %s-translated-tuned.%s' % (
              os.path.join(bin_prefix, 'multi-bleu.perl'), lang, lang2, lang, lang2)
            print command
            os.system(command)