<s> import six <EOL> import mock <EOL> import pytest <EOL> from six . moves import xrange <EOL> from freezegun import freeze_time <EOL> from datetime import date , timedelta <EOL> from scrapi import tasks <EOL> from scrapi import settings <EOL> from scrapi . linter import RawDocument <EOL> settings . USE_FLUENT = False <EOL> BLACKHOLE = lambda * _ , ** __ : None <EOL> @ pytest . fixture <EOL> def raw_doc ( ) : <EOL> return RawDocument ( { <EOL> '<STR_LIT>' : '<STR_LIT:bar>' , <EOL> '<STR_LIT>' : u'<STR_LIT:foo>' , <EOL> '<STR_LIT:source>' : u'<STR_LIT:test>' , <EOL> '<STR_LIT>' : u'<STR_LIT>' , <EOL> } ) <EOL> @ pytest . fixture <EOL> def raw_docs ( ) : <EOL> return [ <EOL> RawDocument ( { <EOL> '<STR_LIT>' : str ( x ) . encode ( '<STR_LIT:utf-8>' ) , <EOL> '<STR_LIT>' : six . text_type ( x ) , <EOL> '<STR_LIT:source>' : u'<STR_LIT:test>' , <EOL> '<STR_LIT>' : u'<STR_LIT>' , <EOL> } ) <EOL> for x in xrange ( <NUM_LIT:11> ) <EOL> ] <EOL> @ freeze_time ( "<STR_LIT>" ) <EOL> def test_run_harvester_calls ( monkeypatch ) : <EOL> mock_harvest = mock . MagicMock ( ) <EOL> mock_begin_norm = mock . MagicMock ( ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_harvest ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_begin_norm ) <EOL> tasks . run_harvester ( '<STR_LIT:test>' ) <EOL> assert mock_harvest . si . called <EOL> assert mock_begin_norm . s . called <EOL> end_date = date ( <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT:16> ) <EOL> start_date = end_date - timedelta ( settings . DAYS_BACK ) <EOL> mock_begin_norm . s . assert_called_once_with ( '<STR_LIT:test>' ) <EOL> mock_harvest . si . assert_called_once_with ( '<STR_LIT:test>' , '<STR_LIT>' , start_date = start_date , end_date = end_date ) <EOL> def test_run_harvester_daysback ( monkeypatch ) : <EOL> mock_harvest = mock . MagicMock ( ) <EOL> mock_begin_norm = mock . MagicMock ( ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_harvest ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_begin_norm ) <EOL> start_date = date ( <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ) <EOL> end_date = date ( <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT:16> ) <EOL> tasks . run_harvester ( '<STR_LIT:test>' , start_date = start_date , end_date = end_date ) <EOL> assert mock_harvest . si . called <EOL> assert mock_begin_norm . s . called <EOL> mock_begin_norm . s . assert_called_once_with ( '<STR_LIT:test>' ) <EOL> mock_harvest . si . assert_called_once_with ( '<STR_LIT:test>' , '<STR_LIT>' , start_date = start_date , end_date = end_date ) <EOL> @ pytest . mark . usefixtures ( '<STR_LIT>' ) <EOL> def test_harvest_runs_harvest ( harvester ) : <EOL> tasks . harvest ( '<STR_LIT:test>' , '<STR_LIT>' ) <EOL> assert harvester . harvest . called <EOL> @ pytest . mark . usefixtures ( '<STR_LIT>' ) <EOL> def test_harvest_days_back ( harvester ) : <EOL> start_date = date ( <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT> ) <EOL> end_date = date ( <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT:16> ) <EOL> _ , timestamps = tasks . harvest ( '<STR_LIT:test>' , '<STR_LIT>' , start_date = start_date , end_date = end_date ) <EOL> keys = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> for key in keys : <EOL> assert key in timestamps . keys ( ) <EOL> assert harvester . harvest . called <EOL> harvester . harvest . assert_called_once_with ( start_date = start_date , end_date = end_date ) <EOL> @ pytest . mark . usefixtures ( '<STR_LIT>' ) <EOL> def test_harvest_raises ( harvester ) : <EOL> harvester . harvest . side_effect = KeyError ( '<STR_LIT>' ) <EOL> with pytest . raises ( KeyError ) as e : <EOL> tasks . harvest ( '<STR_LIT:test>' , '<STR_LIT>' ) <EOL> assert e . value . args [ <NUM_LIT:0> ] == '<STR_LIT>' <EOL> def test_begin_normalize_starts ( raw_docs , monkeypatch ) : <EOL> mock_norm = mock . MagicMock ( ) <EOL> mock_praw = mock . MagicMock ( ) <EOL> mock_pnorm = mock . MagicMock ( ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_norm ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_praw ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , mock_pnorm ) <EOL> timestamps = { } <EOL> tasks . begin_normalization ( ( raw_docs , timestamps ) , '<STR_LIT:test>' ) <EOL> assert mock_norm . si . call_count == <NUM_LIT:11> <EOL> assert mock_pnorm . s . call_count == <NUM_LIT:11> <EOL> assert mock_praw . delay . call_count == <NUM_LIT:11> <EOL> for x in raw_docs : <EOL> mock_pnorm . s . assert_any_call ( x ) <EOL> mock_praw . delay . assert_any_call ( x ) <EOL> mock_norm . si . assert_any_call ( x , '<STR_LIT:test>' ) <EOL> def test_process_raw_calls ( raw_doc , monkeypatch ) : <EOL> pmock = mock . Mock ( ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , pmock ) <EOL> tasks . process_raw ( raw_doc ) <EOL> pmock . assert_called_once_with ( raw_doc , { } ) <EOL> def test_process_norm_calls ( raw_doc , monkeypatch ) : <EOL> pmock = mock . Mock ( ) <EOL> monkeypatch . setattr ( '<STR_LIT>' , pmock ) <EOL> tasks . process_normalized ( raw_doc , raw_doc ) <EOL> pmock . assert_called_once_with ( raw_doc , raw_doc , { } ) </s>
