<s> from __future__ import absolute_import , division , print_function , unicode_literals <EOL> import os <EOL> import shutil <EOL> import subprocess <EOL> import sys <EOL> import tempfile <EOL> import unittest <EOL> sys . dont_write_bytecode = True <EOL> from benchexec import util <EOL> here = os . path . dirname ( __file__ ) <EOL> base_dir = os . path . join ( here , '<STR_LIT:..>' , '<STR_LIT:..>' , '<STR_LIT:..>' ) <EOL> bin_dir = os . path . join ( base_dir , '<STR_LIT>' ) <EOL> tablegenerator = os . path . join ( bin_dir , '<STR_LIT>' ) <EOL> OVERWRITE_MODE = False <EOL> def result_file ( name ) : <EOL> return os . path . join ( here , '<STR_LIT>' , name ) <EOL> class TableGeneratorIntegrationTests ( unittest . TestCase ) : <EOL> @ classmethod <EOL> def setUpClass ( cls ) : <EOL> cls . longMessage = True <EOL> cls . maxDiff = None <EOL> def setUp ( self ) : <EOL> self . tmp = tempfile . mkdtemp ( prefix = "<STR_LIT>" , <EOL> dir = os . path . dirname ( __file__ ) ) <EOL> def tearDown ( self ) : <EOL> shutil . rmtree ( self . tmp ) <EOL> def run_cmd ( self , * args ) : <EOL> try : <EOL> output = subprocess . check_output ( args = args , stderr = subprocess . STDOUT ) . decode ( ) <EOL> except subprocess . CalledProcessError as e : <EOL> print ( e . output . decode ( ) ) <EOL> raise e <EOL> print ( output ) <EOL> return output <EOL> def generate_tables_and_check_produced_files ( self , args , table_prefix , <EOL> diff_prefix = None , <EOL> formats = [ '<STR_LIT:html>' , '<STR_LIT>' ] , <EOL> output_path = None ) : <EOL> output = self . run_cmd ( * [ tablegenerator ] + list ( args ) + [ '<STR_LIT>' , output_path or self . tmp ] ) <EOL> generated_files = set ( os . path . join ( self . tmp , x ) for x in os . listdir ( self . tmp ) ) <EOL> csv_file = os . path . join ( self . tmp , table_prefix + '<STR_LIT>' ) if '<STR_LIT>' in formats else None <EOL> html_file = os . path . join ( self . tmp , table_prefix + '<STR_LIT>' ) if '<STR_LIT:html>' in formats else None <EOL> expected_files = { csv_file , html_file } <EOL> if diff_prefix : <EOL> csv_diff_file = os . path . join ( self . tmp , diff_prefix + '<STR_LIT>' ) if '<STR_LIT>' in formats else None <EOL> html_diff_file = os . path . join ( self . tmp , diff_prefix + '<STR_LIT>' ) if '<STR_LIT:html>' in formats else None <EOL> expected_files |= { csv_diff_file , html_diff_file } <EOL> else : <EOL> csv_diff_file = None <EOL> html_diff_file = None <EOL> expected_files . discard ( None ) <EOL> self . assertSetEqual ( generated_files , expected_files , '<STR_LIT>' ) <EOL> return output , html_file , html_diff_file , csv_file , csv_diff_file <EOL> def generate_tables_and_compare_content ( self , args , table_prefix , result_prefix = None , <EOL> diff_prefix = None , result_diff_prefix = None , <EOL> expected_counts = None ) : <EOL> def expected_file_name ( ending ) : <EOL> return [ here , '<STR_LIT>' , ( result_prefix or table_prefix ) + '<STR_LIT:.>' + ending ] <EOL> def expected_diff_file_name ( ending ) : <EOL> return [ here , '<STR_LIT>' , ( result_diff_prefix or diff_prefix ) + '<STR_LIT:.>' + ending ] <EOL> output , html_file , html_diff_file , csv_file , csv_diff_file = self . generate_tables_and_check_produced_files ( args , table_prefix , diff_prefix ) <EOL> generated_csv = util . read_file ( csv_file ) <EOL> self . assert_file_content_equals ( generated_csv , expected_file_name ( '<STR_LIT>' ) ) <EOL> generated_html = self . read_table_from_html ( html_file ) <EOL> self . assert_file_content_equals ( generated_html , expected_file_name ( '<STR_LIT:html>' ) ) <EOL> if diff_prefix : <EOL> generated_csv_diff = util . read_file ( csv_diff_file ) <EOL> self . assert_file_content_equals ( generated_csv_diff , expected_diff_file_name ( '<STR_LIT>' ) ) <EOL> generated_html_diff = self . read_table_from_html ( html_diff_file ) <EOL> self . assert_file_content_equals ( generated_html_diff , expected_diff_file_name ( '<STR_LIT:html>' ) ) <EOL> if expected_counts : <EOL> counts = output [ output . find ( '<STR_LIT>' ) : ] . strip ( ) <EOL> self . assertMultiLineEqual ( expected_counts , counts ) <EOL> else : <EOL> self . assertNotIn ( '<STR_LIT>' , output ) <EOL> self . assertNotIn ( '<STR_LIT>' , output ) <EOL> def assert_file_content_equals ( self , content , file ) : <EOL> if OVERWRITE_MODE : <EOL> util . write_file ( content , * file ) <EOL> else : <EOL> self . assertMultiLineEqual ( content , util . read_file ( * file ) ) <EOL> def read_table_from_html ( self , file ) : <EOL> content = util . read_file ( file ) <EOL> content = content [ content . index ( '<STR_LIT>' ) : content . index ( '<STR_LIT>' ) + <NUM_LIT:8> ] <EOL> return content <EOL> def test_simple_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_correct_only ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , result_file ( '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_all_columns ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , result_file ( '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_xml ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_xml_with_columns ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_xml_with_links ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_xml_with_numberOfDigits ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_xml_with_scaling ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_reverse ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_no_diff ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_differing_files ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_differing_files_common ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_differing_files_reverse ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_differing_files_correct_only ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_big_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count_single_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> expected_counts = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count_single_table2 ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> expected_counts = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count_multi_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> expected_counts = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count_multi_table_reverse ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> expected_counts = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count2 ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) , '<STR_LIT>' , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> expected_counts = '<STR_LIT>' , <EOL> ) <EOL> def test_dump_count_big_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> expected_counts = "<STR_LIT>" , <EOL> ) <EOL> def test_dump_count_big_table_ignore_flapping_timeout_regressions ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> expected_counts = "<STR_LIT>" , <EOL> ) <EOL> def test_multi_table_xml ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_xml_with_columns ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_multi_table_xml_with_wildcards ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table2 ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table_different_result_order ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table_duplicate_results ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table_multiple_results ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_union_table_order_of_results ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_simple_table_legacy ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> result_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_error_result ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_error_result_ignored ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> result_file ( '<STR_LIT>' ) , <EOL> result_file ( '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_compressed_table_gzip ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_compressed_table_bzip2 ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ result_file ( '<STR_LIT>' ) ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_output_stdout ( self ) : <EOL> output , _ , _ , _ , _ = self . generate_tables_and_check_produced_files ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> formats = [ ] , <EOL> output_path = '<STR_LIT:->' , <EOL> ) <EOL> expected = util . read_file ( here , '<STR_LIT>' , '<STR_LIT>' + '<STR_LIT>' ) <EOL> self . assertMultiLineEqual ( output . strip ( ) , expected ) <EOL> def test_format_csv ( self ) : <EOL> self . generate_tables_and_check_produced_files ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> formats = [ '<STR_LIT>' ] , <EOL> ) <EOL> def test_format_html ( self ) : <EOL> self . generate_tables_and_check_produced_files ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT:html>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> formats = [ '<STR_LIT:html>' ] , <EOL> ) <EOL> def test_format_csv_html ( self ) : <EOL> self . generate_tables_and_check_produced_files ( <EOL> [ result_file ( '<STR_LIT>' ) , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:html>' ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> formats = [ '<STR_LIT>' , '<STR_LIT:html>' ] , <EOL> ) <EOL> def test_smt_results ( self ) : <EOL> self . generate_tables_and_compare_content ( <EOL> [ '<STR_LIT>' , os . path . join ( here , '<STR_LIT>' ) , <EOL> ] , <EOL> table_prefix = '<STR_LIT>' , <EOL> diff_prefix = '<STR_LIT>' , <EOL> ) <EOL> def test_results_via_url ( self ) : <EOL> try : <EOL> self . generate_tables_and_compare_content ( <EOL> [ "<STR_LIT>" ] , <EOL> table_prefix = "<STR_LIT>" , <EOL> ) <EOL> except subprocess . CalledProcessError as e : <EOL> if "<STR_LIT>" in e . output . decode ( ) : <EOL> self . skipTest ( "<STR_LIT>" ) <EOL> else : <EOL> raise </s>
