<s> '''<STR_LIT>''' <EOL> import unittest <EOL> from unittest import TestCase <EOL> from mock import MagicMock <EOL> import sys <EOL> from os import path <EOL> sys . path . append ( path . dirname ( path . dirname ( path . abspath ( __file__ ) ) ) ) <EOL> from kafka_monitor import KafkaMonitor <EOL> from plugins . scraper_handler import ScraperHandler <EOL> from plugins . base_handler import BaseHandler <EOL> from plugins . action_handler import ActionHandler <EOL> from plugins . stats_handler import StatsHandler <EOL> import copy <EOL> from kafka . common import OffsetOutOfRangeError <EOL> from jsonschema import Draft4Validator <EOL> import tldextract <EOL> class ExampleHandler ( BaseHandler ) : <EOL> schema = "<STR_LIT>" <EOL> def setup ( self , settings ) : <EOL> pass <EOL> class TestKafkaMonitor ( TestCase ) : <EOL> def setUp ( self ) : <EOL> self . kafka_monitor = KafkaMonitor ( "<STR_LIT>" , True ) <EOL> self . kafka_monitor . settings = self . kafka_monitor . wrapper . load ( "<STR_LIT>" ) <EOL> self . kafka_monitor . logger = MagicMock ( ) <EOL> def test_load_plugins ( self ) : <EOL> assert_keys = [ <NUM_LIT:100> , <NUM_LIT:200> , <NUM_LIT> ] <EOL> self . kafka_monitor . _load_plugins ( ) <EOL> self . assertEqual ( self . kafka_monitor . plugins_dict . keys ( ) , assert_keys ) <EOL> assert_keys = [ <NUM_LIT:200> , <NUM_LIT> ] <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] = None <EOL> self . kafka_monitor . _load_plugins ( ) <EOL> self . assertEqual ( self . kafka_monitor . plugins_dict . keys ( ) , assert_keys ) <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT:100> <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT> <EOL> self . assertRaises ( ImportError , self . kafka_monitor . _load_plugins ) <EOL> del self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] = <NUM_LIT> , <EOL> self . assertRaises ( IOError , self . kafka_monitor . _load_plugins ) <EOL> del self . kafka_monitor . settings [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> def test_load_stats_total ( self ) : <EOL> self . kafka_monitor . stats_dict = { } <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] = [ ] <EOL> self . kafka_monitor . _setup_stats_total ( MagicMock ( ) ) <EOL> self . assertEquals ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) , [ '<STR_LIT>' ] ) <EOL> self . assertEquals ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) , [ '<STR_LIT>' ] ) <EOL> self . kafka_monitor . stats_dict = { } <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> good = [ <EOL> '<STR_LIT>' , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> ] <EOL> self . kafka_monitor . _setup_stats_total ( MagicMock ( ) ) <EOL> self . assertEquals ( <EOL> sorted ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) ) , <EOL> sorted ( good ) ) <EOL> self . assertEquals ( <EOL> sorted ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) ) , <EOL> sorted ( good ) ) <EOL> k1 = '<STR_LIT>' <EOL> k2 = '<STR_LIT>' <EOL> for time_key in self . kafka_monitor . stats_dict [ '<STR_LIT>' ] : <EOL> if time_key == <NUM_LIT:0> : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ <NUM_LIT:0> ] . key , <EOL> '<STR_LIT>' . format ( k = k1 ) <EOL> ) <EOL> else : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ time_key ] . key , <EOL> '<STR_LIT>' . format ( k = k1 , t = time_key ) <EOL> ) <EOL> for time_key in self . kafka_monitor . stats_dict [ '<STR_LIT>' ] : <EOL> if time_key == <NUM_LIT:0> : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ <NUM_LIT:0> ] . key , <EOL> '<STR_LIT>' . format ( k = k2 ) <EOL> ) <EOL> else : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ time_key ] . key , <EOL> '<STR_LIT>' . format ( k = k2 , t = time_key ) <EOL> ) <EOL> def test_load_stats_plugins ( self ) : <EOL> self . kafka_monitor . _load_plugins ( ) <EOL> self . kafka_monitor . stats_dict = { } <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] = [ ] <EOL> self . kafka_monitor . _setup_stats_plugins ( MagicMock ( ) ) <EOL> defaults = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> self . assertEquals ( <EOL> sorted ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) ) , <EOL> sorted ( defaults ) ) <EOL> for key in self . kafka_monitor . plugins_dict : <EOL> plugin_name = self . kafka_monitor . plugins_dict [ key ] [ '<STR_LIT>' ] . __class__ . __name__ <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ plugin_name ] . keys ( ) , <EOL> [ '<STR_LIT>' ] ) <EOL> self . kafka_monitor . stats_dict = { } <EOL> self . kafka_monitor . settings [ '<STR_LIT>' ] = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ] <EOL> good = [ <EOL> '<STR_LIT>' , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> ] <EOL> self . kafka_monitor . _setup_stats_plugins ( MagicMock ( ) ) <EOL> self . assertEquals ( <EOL> sorted ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] . keys ( ) ) , <EOL> sorted ( defaults ) ) <EOL> for key in self . kafka_monitor . plugins_dict : <EOL> plugin_name = self . kafka_monitor . plugins_dict [ key ] [ '<STR_LIT>' ] . __class__ . __name__ <EOL> self . assertEquals ( <EOL> sorted ( self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ plugin_name ] . keys ( ) ) , <EOL> sorted ( good ) ) <EOL> for plugin_key in self . kafka_monitor . stats_dict [ '<STR_LIT>' ] : <EOL> k1 = '<STR_LIT>' . format ( p = plugin_key ) <EOL> for time_key in self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ plugin_key ] : <EOL> if time_key == <NUM_LIT:0> : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ plugin_key ] [ <NUM_LIT:0> ] . key , <EOL> '<STR_LIT>' . format ( k = k1 ) <EOL> ) <EOL> else : <EOL> self . assertEquals ( <EOL> self . kafka_monitor . stats_dict [ '<STR_LIT>' ] [ plugin_key ] [ time_key ] . key , <EOL> '<STR_LIT>' . format ( k = k1 , t = time_key ) <EOL> ) <EOL> def test_process_messages ( self ) : <EOL> self . kafka_monitor . consumer = MagicMock ( ) <EOL> self . kafka_monitor . stats_dict = { } <EOL> self . kafka_monitor . consumer . get_messages = MagicMock ( <EOL> side_effect = OffsetOutOfRangeError ( "<STR_LIT:1>" ) ) <EOL> try : <EOL> self . kafka_monitor . _process_messages ( ) <EOL> except OffsetOutOfRangeError : <EOL> self . fail ( "<STR_LIT>" ) <EOL> message_string = "<STR_LIT>" <EOL> class a : <EOL> pass <EOL> m = a ( ) <EOL> m . message = a ( ) <EOL> m . message . value = message_string <EOL> messages = [ m ] <EOL> self . kafka_monitor . consumer . get_messages = MagicMock ( <EOL> return_value = messages ) <EOL> try : <EOL> self . kafka_monitor . _process_messages ( ) <EOL> except OffsetOutOfRangeError : <EOL> self . fail ( "<STR_LIT>" ) <EOL> self . kafka_monitor . consumer . get_messages = MagicMock ( <EOL> return_value = messages ) <EOL> self . kafka_monitor . _load_plugins ( ) <EOL> self . kafka_monitor . validator = self . kafka_monitor . extend_with_default ( Draft4Validator ) <EOL> self . kafka_monitor . plugins_dict . items ( ) [ <NUM_LIT:0> ] [ <NUM_LIT:1> ] [ '<STR_LIT>' ] . handle = MagicMock ( side_effect = AssertionError ( "<STR_LIT>" ) ) <EOL> self . kafka_monitor . plugins_dict . items ( ) [ <NUM_LIT:1> ] [ <NUM_LIT:1> ] [ '<STR_LIT>' ] . handle = MagicMock ( side_effect = AssertionError ( "<STR_LIT:action>" ) ) <EOL> message_string = "<STR_LIT>" "<STR_LIT>" <EOL> m . message . value = message_string <EOL> messages = [ m ] <EOL> try : <EOL> self . kafka_monitor . _process_messages ( ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT>" , e . message ) <EOL> message_string = "<STR_LIT>" "<STR_LIT>" <EOL> m . message . value = message_string <EOL> messages = [ m ] <EOL> try : <EOL> self . kafka_monitor . _process_messages ( ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT:action>" , e . message ) <EOL> class TestPlugins ( TestCase ) : <EOL> def test_default_handler ( self ) : <EOL> handler = BaseHandler ( ) <EOL> try : <EOL> handler . setup ( "<STR_LIT:s>" ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except NotImplementedError : <EOL> pass <EOL> try : <EOL> handler . handle ( { } ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except NotImplementedError : <EOL> pass <EOL> def test_scrape_handler ( self ) : <EOL> valid = { <EOL> "<STR_LIT:url>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : <NUM_LIT:5> , <EOL> } <EOL> handler = ScraperHandler ( ) <EOL> handler . extract = tldextract . TLDExtract ( ) <EOL> handler . redis_conn = MagicMock ( ) <EOL> handler . redis_conn . zadd = MagicMock ( side_effect = AssertionError ( "<STR_LIT>" ) ) <EOL> try : <EOL> handler . handle ( valid ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT>" , e . message ) <EOL> handler . redis_conn . zadd = MagicMock ( ) <EOL> handler . redis_conn . set = MagicMock ( side_effect = AssertionError ( "<STR_LIT>" ) ) <EOL> valid [ '<STR_LIT>' ] = <NUM_LIT> <EOL> try : <EOL> handler . handle ( valid ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT>" , e . message ) <EOL> def test_action_handler ( self ) : <EOL> handler = ActionHandler ( ) <EOL> handler . redis_conn = MagicMock ( ) <EOL> handler . redis_conn . set = MagicMock ( side_effect = AssertionError ( "<STR_LIT>" ) ) <EOL> valid = { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT:action>" : "<STR_LIT:info>" , <EOL> } <EOL> try : <EOL> handler . handle ( valid ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT>" , e . message ) <EOL> def test_stats_handler ( self ) : <EOL> handler = StatsHandler ( ) <EOL> handler . redis_conn = MagicMock ( ) <EOL> handler . redis_conn . set = MagicMock ( side_effect = AssertionError ( "<STR_LIT>" ) ) <EOL> valid = { <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT>" , <EOL> "<STR_LIT>" : "<STR_LIT:all>" , <EOL> } <EOL> try : <EOL> handler . handle ( valid ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except AssertionError as e : <EOL> self . assertEquals ( "<STR_LIT>" , e . message ) <EOL> def test_bad_plugins ( self ) : <EOL> class ForgotSchema ( BaseHandler ) : <EOL> def handle ( self , d ) : <EOL> pass <EOL> class ForgotHandle ( BaseHandler ) : <EOL> schema = "<STR_LIT>" <EOL> handler = ForgotSchema ( ) <EOL> try : <EOL> handler . setup ( "<STR_LIT:s>" ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except NotImplementedError as e : <EOL> pass <EOL> handler . handle ( { } ) <EOL> handler = ForgotHandle ( ) <EOL> handler . setup ( "<STR_LIT:s>" ) <EOL> try : <EOL> handler . handle ( { } ) <EOL> self . fail ( "<STR_LIT>" ) <EOL> except NotImplementedError : <EOL> pass <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> unittest . main ( ) </s>
