<filename>cse_bert.py
# -*- coding: utf-8 -*-
"""CSE_BERT.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1a0xAe44WFTPpOhY233DJJ39IunnyYphf
"""

#from google.colab import drive
#drive.mount('/content/drive')

#!pip install simpletransformers

from simpletransformers.classification import ClassificationModel, ClassificationArgs

import pandas as pd
import logging
import statistics

import pandas as pd
import numpy as np

from sklearn.model_selection import train_test_split, KFold
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score, accuracy_score, precision_recall_fscore_support

DATA_COLUMN = 'Text'
LABEL_COLUMN = 'HandLabels'
label_list = ['Positive', 'Negative', 'Neutral']

path ='data/clean/'
#path = '/content/drive/My Drive/clean/'
languages = ['Hungarian', 'Portuguese', 'Bosnian',
             'Croatian', 'Polish', 'Russian',
             'Serbian', 'Slovak', 'Slovenian',
             'English', 'German', 'Swedish']

def to_number(text_labels):
    labels = []
    for l in text_labels:
        n = 0
        if l == 'Neutral': n = 1
        elif l == 'Negative': n = 2
        labels.append(n)
    return labels


def load_single_lang(lang):
    df = pd.read_csv(path + lang + '.csv')
    df.drop('Unnamed: 0', axis=1, inplace=True)
    df.dropna(axis=0, inplace=True)
    df.reset_index(drop=True, inplace=True)
    df['HandLabels'] = to_number(df['HandLabels'])
    df.columns = ['text', 'labels']
    return df


def load_dataset(train_langs, test_langs, train_on_test_lang=False):
    
    # Load train languages
    df_train = pd.DataFrame()
    for lang in train_langs:
        df = pd.read_csv(path + lang + '.csv')
        df.drop('Unnamed: 0', axis=1, inplace=True)
        df.dropna(axis=0, inplace=True)
        df_train = pd.concat([df_train, df], ignore_index=True)
        print(df.shape)
    
    df_train.reset_index(drop=True, inplace=True)
    print(df_train.shape)
    
    # Load test language
    df_test = pd.read_csv(path + test_lang + '.csv')
    df_test.drop('Unnamed: 0', axis=1, inplace=True)
    df_test.dropna(axis=0, inplace=True)
    df_test.reset_index(drop=True, inplace=True)
    print(df_test.shape)
    
    if train_on_test_lang:
        train_size = 0.3
        num_train = int(df_test.shape[0] * train_size)
        df_train = pd.concat([df_train, df_test.head(num_train)], ignore_index=True)
        df_test = df_test.tail(df_test.shape[0]-num_train)
    
    df_train['HandLabels'] = to_number(df_train['HandLabels'])
    df_test['HandLabels'] = to_number(df_test['HandLabels'])

    #train_texts = list(df_train['Text'])
    #train_labels = to_number(df_train['HandLabels'])

    #test_texts = list(df_test['Text'])
    #test_labels = to_number(df_test['HandLabels'])

    df_train.columns = ['text', 'labels']
    df_test.columns = ['text', 'labels']

    print(df_train.head())
    print(df_test.head())
    
    return df_train, df_test

# define evaluation metrics average F1 score
def avg_f1_score(y_true, y_pred):
    scores = f1_score(y_true, y_pred, average=None)
    print(y_true[:10])
    print(y_pred[:10])
    print(scores)
    # get average F1 for postive and negative F1 scores
    f1_negative = scores[2] # Negative
    f1_positive = scores[0] # Positive
    return (f1_negative + f1_positive) / 2.0

def eval(y_true, y_pred):
    acc = accuracy_score(y_true, y_pred)
    avg_f1 = avg_f1_score(y_true, y_pred)
    print("acc:{} f1:{}".format(acc, avg_f1))
    return {"acc":acc, "avg_f1":avg_f1}

def fit_and_evaluate(train_langs, test_lang):
    print(train_langs, test_lang)
    model_name = 'EMBEDDIA/crosloengual-bert'

    logging.basicConfig(level=logging.INFO)
    transformers_logger = logging.getLogger('transformers')
    transformers_logger.setLevel(logging.WARNING)

    # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns.
    # If the Dataframe has a header, it should contain a 'text' and a 'labels' column.
    # If no header is present, the Dataframe should contain at least two columns,
    # with the first column is the text with type str, and the second column in the label with type int.
    df_train, df_test = load_dataset(train_langs, test_lang, train_on_test_lang=True)


    # hyperparameters
    model_args = ClassificationArgs()
    model_args.logging_steps = 1000000
    model_args.save_eval_checkpoints = False
    model_args.save_steps = 1000000
    model_args.no_cache = True
    model_args.save_model_every_epoch = False

    model_args.num_train_epochs = 1
    model_args.learning_rate = 2e-4
    model_args.train_batch_size = 32
    model_args.overwrite_output_dir = True
    '''
    model_args.train_custom_parameters_only = True
    model_args.custom_parameter_groups = [
        {
            "params": ["classifier.weight"],
            "lr": 2e-4,
        },
        {
            "params": ["classifier.bias"],
            "lr": 2e-4,
            "weight_decay": 0.0,
        },
    ]
    '''


    # Create a ClassificationModel
    model = ClassificationModel('bert', model_name, num_labels=3, args=model_args)
    print(model.get_named_parameters())

    # Train the model
    print('Training ...')
    model.train_model(df_train)

    # Evaluate the model
    print('Evaluating ...')
    predictions, raw_outputs = model.predict(df_test['text'].values)
    out = eval(df_test['labels'].values, predictions)

    # write results to file
    with open('results_csebert.txt', 'a+') as f:
        f.write("{} {} {}\n".format(train_langs, test_lang, out))

    del model

def cross_validation(lang):
    print(lang)
    model_name = 'EMBEDDIA/crosloengual-bert'

    logging.basicConfig(level=logging.INFO)
    transformers_logger = logging.getLogger('transformers')
    transformers_logger.setLevel(logging.WARNING)

    # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns.
    # If the Dataframe has a header, it should contain a 'text' and a 'labels' column.
    # If no header is present, the Dataframe should contain at least two columns,
    # with the first column is the text with type str, and the second column in the label with type int.
    accs = []
    f1s = []
    df = load_single_lang(lang)
    

    kf = KFold(n_splits=10)
    for train_index, test_index in kf.split(df.index):
        df_train = df.iloc[train_index]
        df_test = df.iloc[test_index]

        # hyperparameters
        model_args = ClassificationArgs()
        model_args.logging_steps = 1000000
        model_args.save_eval_checkpoints = False
        model_args.save_steps = 1000000
        model_args.no_cache = True
        model_args.save_model_every_epoch = False

        model_args.num_train_epochs = 1
        model_args.learning_rate = 2e-4
        model_args.train_batch_size = 32
        model_args.overwrite_output_dir = True
        '''
        model_args.train_custom_parameters_only = True
        model_args.custom_parameter_groups = [
            {
                "params": ["classifier.weight"],
                "lr": 2e-4,
            },
            {
                "params": ["classifier.bias"],
                "lr": 2e-4,
                "weight_decay": 0.0,
            },
        ]
        '''


        # Create a ClassificationModel
        model = ClassificationModel('bert', model_name, num_labels=3, args=model_args)
        print(model.get_named_parameters())

        # Train the model
        print('Training ...')
        model.train_model(df_train)

        # Evaluate the model
        print('Evaluating ...')
        predictions, raw_outputs = model.predict(df_test['text'].values)
        out = eval(df_test['labels'].values, predictions)
        accs.append(out['acc'])
        f1s.append(out['avg_f1'])
        
        del model

    # write results to file
    with open('results_csebert.txt', 'a+') as f:
        f.write("{} {} {}\n".format(lang, statistics.mean(accs), statistics.mean(f1s)))





languages = ['Bosnian', 'Bulgarian', 'Croatian', 'English', 'German', 'Hungarian', 'Polish', 'Portuguese', 'Russian', 'Serbian', 'Slovak', 'Slovenian', 'Swedish']

for lang in languages:
    cross_validation(lang) 





'''
more_experiments = [
    #(['English'], 'Slovenian'),
    (['English'], 'Croatian'),
    (['Slovenian'], 'English'),
    #(['Slovenian'], 'Croatian'),
    #(['Croatian'], 'Slovenian'),
    (['Croatian'], 'English'),
    (['Croatian', 'English'], 'Slovenian'),
    (['Croatian', 'Slovenian'], 'English'),
    (['English', 'Slovenian'], 'Croatian'),
]


for (train_langs, test_lang) in more_experiments:
    fit_and_evaluate(train_langs, test_lang)

'''


'''
experiments_same_fam = [
    (['German'], 'English'),            
    (['English'], 'German'),
    (['Polish'], 'Russian'),
    (['Polish'], 'Slovak'),
    (['German'], 'Swedish'),
    (['German', 'Swedish'], 'English'),
    (['Slovenian', 'Serbian'], 'Russian'),
    (['Slovenian', 'Serbian'], 'Slovak'),

    (['Serbian'], 'Slovenian'),
    (['Serbian'], 'Croatian'),
    (['Serbian'], 'Bosnian'),
    (['Polish'], 'Slovenian'),
    (['Slovak'], 'Slovenian'),
    (['Croatian'], 'Slovenian'),
    (['Croatian'], 'Serbian'),
    (['Croatian'], 'Bosnian'),
    (['Slovenian'], 'Croatian'),
    (['Slovenian'], 'Serbian'),
    (['Slovenian'], 'Bosnian'),
]

experiments_diff_lang_fam = [
    (['Russian'], 'English'),
    (['English'], 'Russian'),
    (['English'], 'Slovak'),
    (['Polish', 'Slovenian'], 'English'),
    (['German', 'Swedish'], 'Russian'),
    (['English', 'German'], 'Slovak'),                
    
    (['German'], 'Slovenian'),
    (['English'], 'Slovenian'),
    (['Swedish'], 'Slovenian'),
    (['Hungarian'], 'Slovenian'),
    (['Portuguese'], 'Slovenian'),
]

experiments_large_train_dataset = [
    (['English', 'Croatian'], 'Slovenian'),
    (['English', 'Croatian', 'Serbian'], 'Slovak'),
    (['Hungarian', 'Slovak', 'English', 'Croatian'], 'Russian'),
    (['Russian', 'Swedish'], 'English'),

    (['Croatian', 'Serbian', 'Bosnian'], 'Slovenian'),
    (['English', 'Swedish'], 'German'),
]



for (train_langs, test_lang) in experiments_same_fam:
    fit_and_evaluate(train_langs, test_lang)

for (train_langs, test_lang) in experiments_diff_lang_fam:
    fit_and_evaluate(train_langs, test_lang)

for (train_langs, test_lang) in experiments_large_train_dataset:
    fit_and_evaluate(train_langs, test_lang)
'''



<filename>cse_bert.py
# -*- coding: utf-8 -*-
"""CSE_BERT.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1a0xAe44WFTPpOhY233DJJ39IunnyYphf
"""

#from google.colab import drive
#drive.mount('/content/drive')

#!pip install simpletransformers

from simpletransformers.classification import ClassificationModel, ClassificationArgs

import pandas as pd
import logging
import statistics

import pandas as pd
import numpy as np

from sklearn.model_selection import train_test_split, KFold
from sklearn.preprocessing import LabelEncoder
from sklearn.metrics import f1_score, accuracy_score, precision_recall_fscore_support

DATA_COLUMN = 'Text'
LABEL_COLUMN = 'HandLabels'
label_list = ['Positive', 'Negative', 'Neutral']

path ='data/clean/'
#path = '/content/drive/My Drive/clean/'
languages = ['Hungarian', 'Portuguese', 'Bosnian',
             'Croatian', 'Polish', 'Russian',
             'Serbian', 'Slovak', 'Slovenian',
             'English', 'German', 'Swedish']

def to_number(text_labels):
    labels = []
    for l in text_labels:
        n = 0
        if l == 'Neutral': n = 1
        elif l == 'Negative': n = 2
        labels.append(n)
    return labels


def load_single_lang(lang):
    df = pd.read_csv(path + lang + '.csv')
    df.drop('Unnamed: 0', axis=1, inplace=True)
    df.dropna(axis=0, inplace=True)
    df.reset_index(drop=True, inplace=True)
    df['HandLabels'] = to_number(df['HandLabels'])
    df.columns = ['text', 'labels']
    return df


def load_dataset(train_langs, test_langs, train_on_test_lang=False):
    
    # Load train languages
    df_train = pd.DataFrame()
    for lang in train_langs:
        df = pd.read_csv(path + lang + '.csv')
        df.drop('Unnamed: 0', axis=1, inplace=True)
        df.dropna(axis=0, inplace=True)
        df_train = pd.concat([df_train, df], ignore_index=True)
        print(df.shape)
    
    df_train.reset_index(drop=True, inplace=True)
    print(df_train.shape)
    
    # Load test language
    df_test = pd.read_csv(path + test_lang + '.csv')
    df_test.drop('Unnamed: 0', axis=1, inplace=True)
    df_test.dropna(axis=0, inplace=True)
    df_test.reset_index(drop=True, inplace=True)
    print(df_test.shape)
    
    if train_on_test_lang:
        train_size = 0.3
        num_train = int(df_test.shape[0] * train_size)
        df_train = pd.concat([df_train, df_test.head(num_train)], ignore_index=True)
        df_test = df_test.tail(df_test.shape[0]-num_train)
    
    df_train['HandLabels'] = to_number(df_train['HandLabels'])
    df_test['HandLabels'] = to_number(df_test['HandLabels'])

    #train_texts = list(df_train['Text'])
    #train_labels = to_number(df_train['HandLabels'])

    #test_texts = list(df_test['Text'])
    #test_labels = to_number(df_test['HandLabels'])

    df_train.columns = ['text', 'labels']
    df_test.columns = ['text', 'labels']

    print(df_train.head())
    print(df_test.head())
    
    return df_train, df_test

# define evaluation metrics average F1 score
def avg_f1_score(y_true, y_pred):
    scores = f1_score(y_true, y_pred, average=None)
    print(y_true[:10])
    print(y_pred[:10])
    print(scores)
    # get average F1 for postive and negative F1 scores
    f1_negative = scores[2] # Negative
    f1_positive = scores[0] # Positive
    return (f1_negative + f1_positive) / 2.0

def eval(y_true, y_pred):
    acc = accuracy_score(y_true, y_pred)
    avg_f1 = avg_f1_score(y_true, y_pred)
    print("acc:{} f1:{}".format(acc, avg_f1))
    return {"acc":acc, "avg_f1":avg_f1}

def fit_and_evaluate(train_langs, test_lang):
    print(train_langs, test_lang)
    model_name = 'EMBEDDIA/crosloengual-bert'

    logging.basicConfig(level=logging.INFO)
    transformers_logger = logging.getLogger('transformers')
    transformers_logger.setLevel(logging.WARNING)

    # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns.
    # If the Dataframe has a header, it should contain a 'text' and a 'labels' column.
    # If no header is present, the Dataframe should contain at least two columns,
    # with the first column is the text with type str, and the second column in the label with type int.
    df_train, df_test = load_dataset(train_langs, test_lang, train_on_test_lang=True)


    # hyperparameters
    model_args = ClassificationArgs()
    model_args.logging_steps = 1000000
    model_args.save_eval_checkpoints = False
    model_args.save_steps = 1000000
    model_args.no_cache = True
    model_args.save_model_every_epoch = False

    model_args.num_train_epochs = 1
    model_args.learning_rate = 2e-4
    model_args.train_batch_size = 32
    model_args.overwrite_output_dir = True
    '''
    model_args.train_custom_parameters_only = True
    model_args.custom_parameter_groups = [
        {
            "params": ["classifier.weight"],
            "lr": 2e-4,
        },
        {
            "params": ["classifier.bias"],
            "lr": 2e-4,
            "weight_decay": 0.0,
        },
    ]
    '''


    # Create a ClassificationModel
    model = ClassificationModel('bert', model_name, num_labels=3, args=model_args)
    print(model.get_named_parameters())

    # Train the model
    print('Training ...')
    model.train_model(df_train)

    # Evaluate the model
    print('Evaluating ...')
    predictions, raw_outputs = model.predict(df_test['text'].values)
    out = eval(df_test['labels'].values, predictions)

    # write results to file
    with open('results_csebert.txt', 'a+') as f:
        f.write("{} {} {}\n".format(train_langs, test_lang, out))

    del model

def cross_validation(lang):
    print(lang)
    model_name = 'EMBEDDIA/crosloengual-bert'

    logging.basicConfig(level=logging.INFO)
    transformers_logger = logging.getLogger('transformers')
    transformers_logger.setLevel(logging.WARNING)

    # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns.
    # If the Dataframe has a header, it should contain a 'text' and a 'labels' column.
    # If no header is present, the Dataframe should contain at least two columns,
    # with the first column is the text with type str, and the second column in the label with type int.
    accs = []
    f1s = []
    df = load_single_lang(lang)
    

    kf = KFold(n_splits=10)
    for train_index, test_index in kf.split(df.index):
        df_train = df.iloc[train_index]
        df_test = df.iloc[test_index]

        # hyperparameters
        model_args = ClassificationArgs()
        model_args.logging_steps = 1000000
        model_args.save_eval_checkpoints = False
        model_args.save_steps = 1000000
        model_args.no_cache = True
        model_args.save_model_every_epoch = False

        model_args.num_train_epochs = 1
        model_args.learning_rate = 2e-4
        model_args.train_batch_size = 32
        model_args.overwrite_output_dir = True
        '''
        model_args.train_custom_parameters_only = True
        model_args.custom_parameter_groups = [
            {
                "params": ["classifier.weight"],
                "lr": 2e-4,
            },
            {
                "params": ["classifier.bias"],
                "lr": 2e-4,
                "weight_decay": 0.0,
            },
        ]
        '''


        # Create a ClassificationModel
        model = ClassificationModel('bert', model_name, num_labels=3, args=model_args)
        print(model.get_named_parameters())

        # Train the model
        print('Training ...')
        model.train_model(df_train)

        # Evaluate the model
        print('Evaluating ...')
        predictions, raw_outputs = model.predict(df_test['text'].values)
        out = eval(df_test['labels'].values, predictions)
        accs.append(out['acc'])
        f1s.append(out['avg_f1'])
        
        del model

    # write results to file
    with open('results_csebert.txt', 'a+') as f:
        f.write("{} {} {}\n".format(lang, statistics.mean(accs), statistics.mean(f1s)))





languages = ['Bosnian', 'Bulgarian', 'Croatian', 'English', 'German', 'Hungarian', 'Polish', 'Portuguese', 'Russian', 'Serbian', 'Slovak', 'Slovenian', 'Swedish']

for lang in languages:
    cross_validation(lang) 





'''
more_experiments = [
    #(['English'], 'Slovenian'),
    (['English'], 'Croatian'),
    (['Slovenian'], 'English'),
    #(['Slovenian'], 'Croatian'),
    #(['Croatian'], 'Slovenian'),
    (['Croatian'], 'English'),
    (['Croatian', 'English'], 'Slovenian'),
    (['Croatian', 'Slovenian'], 'English'),
    (['English', 'Slovenian'], 'Croatian'),
]


for (train_langs, test_lang) in more_experiments:
    fit_and_evaluate(train_langs, test_lang)

'''


'''
experiments_same_fam = [
    (['German'], 'English'),            
    (['English'], 'German'),
    (['Polish'], 'Russian'),
    (['Polish'], 'Slovak'),
    (['German'], 'Swedish'),
    (['German', 'Swedish'], 'English'),
    (['Slovenian', 'Serbian'], 'Russian'),
    (['Slovenian', 'Serbian'], 'Slovak'),

    (['Serbian'], 'Slovenian'),
    (['Serbian'], 'Croatian'),
    (['Serbian'], 'Bosnian'),
    (['Polish'], 'Slovenian'),
    (['Slovak'], 'Slovenian'),
    (['Croatian'], 'Slovenian'),
    (['Croatian'], 'Serbian'),
    (['Croatian'], 'Bosnian'),
    (['Slovenian'], 'Croatian'),
    (['Slovenian'], 'Serbian'),
    (['Slovenian'], 'Bosnian'),
]

experiments_diff_lang_fam = [
    (['Russian'], 'English'),
    (['English'], 'Russian'),
    (['English'], 'Slovak'),
    (['Polish', 'Slovenian'], 'English'),
    (['German', 'Swedish'], 'Russian'),
    (['English', 'German'], 'Slovak'),                
    
    (['German'], 'Slovenian'),
    (['English'], 'Slovenian'),
    (['Swedish'], 'Slovenian'),
    (['Hungarian'], 'Slovenian'),
    (['Portuguese'], 'Slovenian'),
]

experiments_large_train_dataset = [
    (['English', 'Croatian'], 'Slovenian'),
    (['English', 'Croatian', 'Serbian'], 'Slovak'),
    (['Hungarian', 'Slovak', 'English', 'Croatian'], 'Russian'),
    (['Russian', 'Swedish'], 'English'),

    (['Croatian', 'Serbian', 'Bosnian'], 'Slovenian'),
    (['English', 'Swedish'], 'German'),
]



for (train_langs, test_lang) in experiments_same_fam:
    fit_and_evaluate(train_langs, test_lang)

for (train_langs, test_lang) in experiments_diff_lang_fam:
    fit_and_evaluate(train_langs, test_lang)

for (train_langs, test_lang) in experiments_large_train_dataset:
    fit_and_evaluate(train_langs, test_lang)
'''



# -*- coding: utf-8 -*- CSE_BERT.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/1a0xAe44WFTPpOhY233DJJ39IunnyYphf #from google.colab import drive #drive.mount('/content/drive') #!pip install simpletransformers #path = '/content/drive/My Drive/clean/' # Load train languages # Load test language #train_texts = list(df_train['Text']) #train_labels = to_number(df_train['HandLabels']) #test_texts = list(df_test['Text']) #test_labels = to_number(df_test['HandLabels']) # define evaluation metrics average F1 score # get average F1 for postive and negative F1 scores # Negative # Positive # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns. # If the Dataframe has a header, it should contain a 'text' and a 'labels' column. # If no header is present, the Dataframe should contain at least two columns, # with the first column is the text with type str, and the second column in the label with type int. # hyperparameters model_args.train_custom_parameters_only = True     model_args.custom_parameter_groups = [         {             "params": ["classifier.weight"],             "lr": 2e-4,         },         {             "params": ["classifier.bias"],             "lr": 2e-4,             "weight_decay": 0.0,         },     ] # Create a ClassificationModel # Train the model # Evaluate the model # write results to file # Train and Evaluation data needs to be in a Pandas Dataframe containing at least two columns. # If the Dataframe has a header, it should contain a 'text' and a 'labels' column. # If no header is present, the Dataframe should contain at least two columns, # with the first column is the text with type str, and the second column in the label with type int. # hyperparameters model_args.train_custom_parameters_only = True         model_args.custom_parameter_groups = [             {                 "params": ["classifier.weight"],                 "lr": 2e-4,             },             {                 "params": ["classifier.bias"],                 "lr": 2e-4,                 "weight_decay": 0.0,             },         ] # Create a ClassificationModel # Train the model # Evaluate the model # write results to file more_experiments = [     #(['English'], 'Slovenian'),     (['English'], 'Croatian'),     (['Slovenian'], 'English'),     #(['Slovenian'], 'Croatian'),     #(['Croatian'], 'Slovenian'),     (['Croatian'], 'English'),     (['Croatian', 'English'], 'Slovenian'),     (['Croatian', 'Slovenian'], 'English'),     (['English', 'Slovenian'], 'Croatian'), ]   for (train_langs, test_lang) in more_experiments:     fit_and_evaluate(train_langs, test_lang) experiments_same_fam = [     (['German'], 'English'),                 (['English'], 'German'),     (['Polish'], 'Russian'),     (['Polish'], 'Slovak'),     (['German'], 'Swedish'),     (['German', 'Swedish'], 'English'),     (['Slovenian', 'Serbian'], 'Russian'),     (['Slovenian', 'Serbian'], 'Slovak'),      (['Serbian'], 'Slovenian'),     (['Serbian'], 'Croatian'),     (['Serbian'], 'Bosnian'),     (['Polish'], 'Slovenian'),     (['Slovak'], 'Slovenian'),     (['Croatian'], 'Slovenian'),     (['Croatian'], 'Serbian'),     (['Croatian'], 'Bosnian'),     (['Slovenian'], 'Croatian'),     (['Slovenian'], 'Serbian'),     (['Slovenian'], 'Bosnian'), ]  experiments_diff_lang_fam = [     (['Russian'], 'English'),     (['English'], 'Russian'),     (['English'], 'Slovak'),     (['Polish', 'Slovenian'], 'English'),     (['German', 'Swedish'], 'Russian'),     (['English', 'German'], 'Slovak'),                          (['German'], 'Slovenian'),     (['English'], 'Slovenian'),     (['Swedish'], 'Slovenian'),     (['Hungarian'], 'Slovenian'),     (['Portuguese'], 'Slovenian'), ]  experiments_large_train_dataset = [     (['English', 'Croatian'], 'Slovenian'),     (['English', 'Croatian', 'Serbian'], 'Slovak'),     (['Hungarian', 'Slovak', 'English', 'Croatian'], 'Russian'),     (['Russian', 'Swedish'], 'English'),      (['Croatian', 'Serbian', 'Bosnian'], 'Slovenian'),     (['English', 'Swedish'], 'German'), ]    for (train_langs, test_lang) in experiments_same_fam:     fit_and_evaluate(train_langs, test_lang)  for (train_langs, test_lang) in experiments_diff_lang_fam:     fit_and_evaluate(train_langs, test_lang)  for (train_langs, test_lang) in experiments_large_train_dataset:     fit_and_evaluate(train_langs, test_lang)