<s> import shutil <EOL> import json <EOL> from rest_framework import routers , serializers , viewsets , parsers , filters <EOL> from rest_framework . views import APIView <EOL> from rest_framework . exceptions import APIException <EOL> from rest_framework . response import Response <EOL> from django . core . exceptions import ValidationError <EOL> from django . core . files . uploadedfile import SimpleUploadedFile , InMemoryUploadedFile <EOL> from django . core . validators import URLValidator <EOL> from base . models import Project , SeedsList <EOL> from apps . crawl_space . models import Crawl , CrawlModel <EOL> from elasticsearch import Elasticsearch <EOL> from elasticsearch . exceptions import ConnectionError , NotFoundError <EOL> class DataWakeIndexUnavailable ( APIException ) : <EOL> status_code = <NUM_LIT> <EOL> default_detail = "<STR_LIT>" <EOL> class SlugModelSerializer ( serializers . ModelSerializer ) : <EOL> slug = serializers . SlugField ( required = False , read_only = True ) <EOL> class ProjectSerializer ( SlugModelSerializer ) : <EOL> url = serializers . CharField ( read_only = True ) <EOL> class Meta : <EOL> model = Project <EOL> class CrawlSerializer ( SlugModelSerializer ) : <EOL> id = serializers . ReadOnlyField ( ) <EOL> seeds_list = serializers . FileField ( read_only = True , use_url = False ) <EOL> status = serializers . CharField ( read_only = True ) <EOL> config = serializers . CharField ( read_only = True ) <EOL> index_name = serializers . CharField ( read_only = True ) <EOL> url = serializers . CharField ( read_only = True ) <EOL> pages_crawled = serializers . IntegerField ( read_only = True ) <EOL> harvest_rate = serializers . FloatField ( read_only = True ) <EOL> location = serializers . CharField ( read_only = True ) <EOL> def validate_crawler ( self , value ) : <EOL> if value == "<STR_LIT>" and not self . initial_data . get ( "<STR_LIT>" ) : <EOL> raise serializers . ValidationError ( "<STR_LIT>" ) <EOL> return value <EOL> class Meta : <EOL> model = Crawl <EOL> class CrawlModelSerializer ( SlugModelSerializer ) : <EOL> model = serializers . FileField ( use_url = False ) <EOL> features = serializers . FileField ( use_url = False ) <EOL> url = serializers . CharField ( read_only = True ) <EOL> def validate_model ( self , value ) : <EOL> if value . name != "<STR_LIT>" : <EOL> raise serializers . ValidationError ( "<STR_LIT>" ) <EOL> return value <EOL> def validate_features ( self , value ) : <EOL> if value . name != "<STR_LIT>" : <EOL> raise serializers . ValidationError ( "<STR_LIT>" ) <EOL> return value <EOL> class Meta : <EOL> model = CrawlModel <EOL> class SeedsListSerializer ( SlugModelSerializer ) : <EOL> url = serializers . CharField ( read_only = True ) <EOL> file_string = serializers . CharField ( read_only = True ) <EOL> def validate_seeds ( self , value ) : <EOL> try : <EOL> seeds = json . loads ( value ) <EOL> except ValueError : <EOL> raise serializers . ValidationError ( "<STR_LIT>" ) <EOL> if type ( seeds ) != list : <EOL> raise serializers . ValidationError ( "<STR_LIT>" ) <EOL> validator = URLValidator ( ) <EOL> errors = [ ] <EOL> for index , x in enumerate ( seeds ) : <EOL> try : <EOL> validator ( x ) <EOL> except ValidationError : <EOL> errors . append ( { index : x } ) <EOL> if errors : <EOL> errors . insert ( <NUM_LIT:0> , "<STR_LIT>" ) <EOL> errors . append ( { "<STR_LIT:list>" : "<STR_LIT:\n>" . join ( seeds ) } ) <EOL> raise serializers . ValidationError ( errors ) <EOL> return value <EOL> class Meta : <EOL> model = SeedsList <EOL> """<STR_LIT>""" <EOL> class ProjectViewSet ( viewsets . ModelViewSet ) : <EOL> queryset = Project . objects . all ( ) <EOL> serializer_class = ProjectSerializer <EOL> filter_fields = ( '<STR_LIT:id>' , '<STR_LIT>' , '<STR_LIT:name>' , ) <EOL> class CrawlViewSet ( viewsets . ModelViewSet ) : <EOL> queryset = Crawl . objects . all ( ) <EOL> serializer_class = CrawlSerializer <EOL> filter_fields = ( '<STR_LIT:id>' , '<STR_LIT>' , '<STR_LIT:name>' , '<STR_LIT:description>' , '<STR_LIT:status>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> class CrawlModelViewSet ( viewsets . ModelViewSet ) : <EOL> queryset = CrawlModel . objects . all ( ) <EOL> serializer_class = CrawlModelSerializer <EOL> filter_fields = ( '<STR_LIT:id>' , '<STR_LIT>' , '<STR_LIT:name>' , '<STR_LIT>' , ) <EOL> def destroy ( self , request , pk = None ) : <EOL> model = CrawlModel . objects . get ( pk = pk ) <EOL> crawls = Crawl . objects . all ( ) . filter ( crawl_model = pk ) <EOL> if crawls : <EOL> message = "<STR_LIT>" <EOL> raise serializers . ValidationError ( { <EOL> "<STR_LIT:message>" : message , <EOL> "<STR_LIT>" : [ x . name for x in crawls ] , <EOL> } ) <EOL> else : <EOL> shutil . rmtree ( model . get_model_path ( ) ) <EOL> return super ( CrawlModelViewSet , self ) . destroy ( request ) <EOL> class SeedsListViewSet ( viewsets . ModelViewSet ) : <EOL> queryset = SeedsList . objects . all ( ) <EOL> serializer_class = SeedsListSerializer <EOL> filter_fields = ( '<STR_LIT:id>' , '<STR_LIT:name>' , '<STR_LIT>' , '<STR_LIT>' , ) <EOL> def create ( self , request ) : <EOL> seeds_list = request . FILES . get ( "<STR_LIT>" , False ) <EOL> textseeds = request . data . get ( "<STR_LIT>" , False ) <EOL> if seeds_list : <EOL> request . data [ "<STR_LIT>" ] = json . dumps ( map ( str . strip , seeds_list . readlines ( ) ) ) <EOL> elif textseeds : <EOL> if type ( textseeds ) == unicode : <EOL> request . data [ "<STR_LIT>" ] = json . dumps ( map ( unicode . strip , textseeds . split ( "<STR_LIT:\n>" ) ) ) <EOL> elif type ( textseeds ) == str : <EOL> request . data [ "<STR_LIT>" ] = json . dumps ( map ( str . strip , textseeds . split ( "<STR_LIT:\n>" ) ) ) <EOL> return super ( SeedsListViewSet , self ) . create ( request ) <EOL> def destroy ( self , request , pk = None ) : <EOL> seeds = SeedsList . objects . get ( pk = pk ) <EOL> crawls = Crawl . objects . all ( ) . filter ( seeds_object = pk ) <EOL> if crawls : <EOL> message = "<STR_LIT>" <EOL> raise serializers . ValidationError ( { <EOL> "<STR_LIT:message>" : message , <EOL> "<STR_LIT>" : [ x . name for x in crawls ] , <EOL> } ) <EOL> else : <EOL> return super ( SeedsListViewSet , self ) . destroy ( request ) <EOL> class DataWakeView ( APIView ) : <EOL> index = "<STR_LIT>" <EOL> es = Elasticsearch ( ) <EOL> def create_trails ( self , trail_ids ) : <EOL> trails = [ ] <EOL> for x in trail_ids : <EOL> url_search = self . es . search ( index = self . index , q = "<STR_LIT>" % x , <EOL> fields = "<STR_LIT:url>" , size = <NUM_LIT:1000> ) [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> new_trail = { "<STR_LIT>" : x , "<STR_LIT>" : [ ] , "<STR_LIT>" : url_search [ <NUM_LIT:0> ] [ "<STR_LIT>" ] } <EOL> for y in url_search : <EOL> new_trail [ "<STR_LIT>" ] . append ( y [ "<STR_LIT>" ] [ "<STR_LIT:url>" ] [ <NUM_LIT:0> ] ) <EOL> new_trail . update ( { "<STR_LIT>" : "<STR_LIT:\n>" . join ( new_trail [ "<STR_LIT>" ] ) } ) <EOL> trails . append ( new_trail ) <EOL> return trails <EOL> def get ( self , request , format = None ) : <EOL> try : <EOL> trail_ids = [ x [ "<STR_LIT:key>" ] for x in self . es . search ( index = self . index , body = { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : { "<STR_LIT>" : "<STR_LIT>" } <EOL> } <EOL> } <EOL> } ) [ "<STR_LIT>" ] [ "<STR_LIT>" ] [ "<STR_LIT>" ] ] <EOL> response = self . create_trails ( trail_ids ) <EOL> except ConnectionError as e : <EOL> raise OSError ( "<STR_LIT>" ) <EOL> except NotFoundError : <EOL> raise DataWakeIndexUnavailable <EOL> return Response ( response ) <EOL> router = routers . DefaultRouter ( ) <EOL> router . register ( r"<STR_LIT>" , ProjectViewSet ) <EOL> router . register ( r"<STR_LIT>" , CrawlViewSet ) <EOL> router . register ( r"<STR_LIT>" , CrawlModelViewSet ) <EOL> router . register ( r"<STR_LIT>" , SeedsListViewSet ) </s>
