<s> import json <EOL> import logging <EOL> import os <EOL> import re <EOL> import socket <EOL> import salt . cloud <EOL> import yaml <EOL> from django . conf import settings <EOL> from django . contrib . contenttypes . fields import GenericRelation <EOL> from django . core . cache import cache <EOL> from django . db import models , transaction <EOL> from django . core . files . base import ContentFile <EOL> from django . core . files . storage import FileSystemStorage <EOL> from django . utils . timezone import now <EOL> from django_extensions . db . models import ( <EOL> TimeStampedModel , <EOL> TitleSlugDescriptionModel , <EOL> ) <EOL> from guardian . shortcuts import get_users_with_perms <EOL> from model_utils import Choices <EOL> from model_utils . models import StatusModel <EOL> from stackdio . core . fields import DeletingFileField <EOL> from stackdio . core . utils import recursive_update <EOL> from stackdio . api . cloud . models import SecurityGroup <EOL> PROTOCOL_CHOICES = [ <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) , <EOL> ] <EOL> logger = logging . getLogger ( __name__ ) <EOL> HOST_INDEX_PATTERN = re . compile ( r'<STR_LIT>' ) <EOL> def get_hostnames_from_hostdefs ( hostdefs , username = '<STR_LIT>' , namespace = '<STR_LIT>' ) : <EOL> hostnames = [ ] <EOL> for hostdef in hostdefs : <EOL> for i in xrange ( hostdef . count ) : <EOL> hostnames . append ( <EOL> hostdef . hostname_template . format ( <EOL> namespace = namespace , <EOL> username = username , <EOL> index = i <EOL> ) <EOL> ) <EOL> return hostnames <EOL> class StackCreationException ( Exception ) : <EOL> def __init__ ( self , errors , * args , ** kwargs ) : <EOL> self . errors = errors <EOL> super ( StackCreationException , self ) . __init__ ( * args , ** kwargs ) <EOL> class Level ( object ) : <EOL> DEBUG = '<STR_LIT>' <EOL> INFO = '<STR_LIT>' <EOL> WARN = '<STR_LIT>' <EOL> ERROR = '<STR_LIT>' <EOL> class StatusDetailModel ( StatusModel ) : <EOL> status_detail = models . TextField ( blank = True ) <EOL> class Meta : <EOL> abstract = True <EOL> default_permissions = ( ) <EOL> def set_status ( self , status , detail = '<STR_LIT>' ) : <EOL> self . status = status <EOL> self . status_detail = detail <EOL> return self . save ( ) <EOL> class StackQuerySet ( models . QuerySet ) : <EOL> def create ( self , ** kwargs ) : <EOL> new_properties = kwargs . pop ( '<STR_LIT>' , { } ) <EOL> with transaction . atomic ( using = self . db ) : <EOL> stack = super ( StackQuerySet , self ) . create ( ** kwargs ) <EOL> properties = stack . blueprint . properties <EOL> recursive_update ( properties , new_properties ) <EOL> stack . properties = properties <EOL> stack . create_security_groups ( ) <EOL> stack . create_hosts ( ) <EOL> return stack <EOL> _stack_model_permissions = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> _stack_object_permissions = ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT:start>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> ) <EOL> stack_storage = FileSystemStorage ( location = os . path . join ( settings . FILE_STORAGE_DIRECTORY , '<STR_LIT>' ) ) <EOL> def get_local_file_path ( instance , filename ) : <EOL> return '<STR_LIT>' . format ( instance . pk , instance . slug , filename ) <EOL> def get_orchestrate_file_path ( instance , filename ) : <EOL> return '<STR_LIT>' . format ( instance . pk , instance . slug , filename ) <EOL> class Stack ( TimeStampedModel , TitleSlugDescriptionModel , StatusModel ) : <EOL> PENDING = '<STR_LIT>' <EOL> LAUNCHING = '<STR_LIT>' <EOL> CONFIGURING = '<STR_LIT>' <EOL> SYNCING = '<STR_LIT>' <EOL> PROVISIONING = '<STR_LIT>' <EOL> ORCHESTRATING = '<STR_LIT>' <EOL> FINALIZING = '<STR_LIT>' <EOL> FINISHED = '<STR_LIT>' <EOL> DESTROYING = '<STR_LIT>' <EOL> STARTING = '<STR_LIT>' <EOL> STOPPING = '<STR_LIT>' <EOL> TERMINATING = '<STR_LIT>' <EOL> EXECUTING_ACTION = '<STR_LIT>' <EOL> ERROR = '<STR_LIT:error>' <EOL> SAFE_STATES = [ FINISHED , ERROR ] <EOL> OK = '<STR_LIT>' <EOL> RUNNING = '<STR_LIT>' <EOL> REBOOTING = '<STR_LIT>' <EOL> STATUS = Choices ( PENDING , LAUNCHING , CONFIGURING , SYNCING , PROVISIONING , <EOL> ORCHESTRATING , FINALIZING , DESTROYING , FINISHED , <EOL> STARTING , STOPPING , TERMINATING , EXECUTING_ACTION , ERROR ) <EOL> model_permissions = _stack_model_permissions <EOL> object_permissions = _stack_object_permissions <EOL> searchable_fields = ( '<STR_LIT:title>' , '<STR_LIT:description>' , '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:title>' , ) <EOL> default_permissions = tuple ( set ( _stack_model_permissions + _stack_object_permissions ) ) <EOL> unique_together = ( '<STR_LIT:title>' , ) <EOL> blueprint = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> formula_versions = GenericRelation ( '<STR_LIT>' ) <EOL> labels = GenericRelation ( '<STR_LIT>' ) <EOL> namespace = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:64> ) <EOL> create_users = models . BooleanField ( '<STR_LIT>' ) <EOL> map_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_local_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> top_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = FileSystemStorage ( location = settings . STACKDIO_CONFIG . salt_core_states ) ) <EOL> orchestrate_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_orchestrate_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> global_orchestrate_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_orchestrate_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> pillar_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_local_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> global_pillar_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_local_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> props_file = DeletingFileField ( <EOL> max_length = <NUM_LIT:255> , <EOL> upload_to = get_local_file_path , <EOL> null = True , <EOL> blank = True , <EOL> default = None , <EOL> storage = stack_storage ) <EOL> objects = StackQuerySet . as_manager ( ) <EOL> def __unicode__ ( self ) : <EOL> return u'<STR_LIT>' . format ( self . title , self . id ) <EOL> def set_status ( self , event , status , detail , level = Level . INFO ) : <EOL> self . status = status <EOL> self . save ( ) <EOL> self . history . create ( event = event , status = status , <EOL> status_detail = detail , level = level ) <EOL> def get_driver_hosts_map ( self , host_ids = None ) : <EOL> """<STR_LIT>""" <EOL> host_queryset = self . get_hosts ( host_ids ) <EOL> accounts = { } <EOL> for h in host_queryset : <EOL> accounts . setdefault ( h . get_account ( ) , [ ] ) . append ( h ) <EOL> result = { } <EOL> for account , hosts in accounts . items ( ) : <EOL> result [ account . get_driver ( ) ] = host_queryset . filter ( id__in = [ h . id for h in hosts ] ) <EOL> return result <EOL> def get_hosts ( self , host_ids = None ) : <EOL> """<STR_LIT>""" <EOL> if not host_ids : <EOL> return self . hosts . all ( ) <EOL> return self . hosts . filter ( id__in = host_ids ) <EOL> def get_formulas ( self ) : <EOL> return self . blueprint . get_formulas ( ) <EOL> def get_tags ( self ) : <EOL> tags = { } <EOL> for label in self . labels . all ( ) : <EOL> tags [ label . key ] = label . value <EOL> tags [ '<STR_LIT>' ] = self . id <EOL> if '<STR_LIT:Name>' in tags : <EOL> del tags [ '<STR_LIT:Name>' ] <EOL> return tags <EOL> @ property <EOL> def properties ( self ) : <EOL> if not self . props_file : <EOL> return { } <EOL> with open ( self . props_file . path , '<STR_LIT:r>' ) as f : <EOL> return json . load ( f ) <EOL> @ properties . setter <EOL> def properties ( self , props ) : <EOL> props_json = json . dumps ( props , indent = <NUM_LIT:4> ) <EOL> if not self . props_file : <EOL> self . props_file . save ( '<STR_LIT>' , ContentFile ( props_json ) ) <EOL> else : <EOL> with open ( self . props_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( props_json ) <EOL> def create_security_groups ( self ) : <EOL> for hostdef in self . blueprint . host_definitions . all ( ) : <EOL> sg_name = '<STR_LIT>' . format ( <EOL> hostdef . slug , <EOL> self . pk <EOL> ) <EOL> sg_description = '<STR_LIT>' <EOL> account = hostdef . cloud_image . account <EOL> if not account . create_security_groups : <EOL> logger . debug ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( sg_name ) ) <EOL> continue <EOL> driver = account . get_driver ( ) <EOL> try : <EOL> sg_id = driver . create_security_group ( sg_name , <EOL> sg_description , <EOL> delete_if_exists = True ) <EOL> except Exception as e : <EOL> err_msg = '<STR_LIT>' . format ( str ( e ) ) <EOL> self . set_status ( '<STR_LIT>' , self . ERROR , <EOL> err_msg , Level . ERROR ) <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> sg_name , <EOL> sg_id <EOL> ) ) <EOL> for access_rule in hostdef . access_rules . all ( ) : <EOL> driver . authorize_security_group ( sg_id , { <EOL> '<STR_LIT>' : access_rule . protocol , <EOL> '<STR_LIT>' : access_rule . from_port , <EOL> '<STR_LIT>' : access_rule . to_port , <EOL> '<STR_LIT>' : access_rule . rule , <EOL> } ) <EOL> self . security_groups . create ( <EOL> account = account , <EOL> blueprint_host_definition = hostdef , <EOL> name = sg_name , <EOL> description = sg_description , <EOL> group_id = sg_id , <EOL> is_managed = True <EOL> ) <EOL> def create_hosts ( self , host_definition = None , count = None , backfill = False ) : <EOL> """<STR_LIT>""" <EOL> created_hosts = [ ] <EOL> if host_definition is None : <EOL> host_definitions = self . blueprint . host_definitions . all ( ) <EOL> else : <EOL> host_definitions = [ host_definition ] <EOL> for hostdef in host_definitions : <EOL> hosts = self . hosts . all ( ) <EOL> if count is None : <EOL> start , end = <NUM_LIT:0> , hostdef . count <EOL> indexes = range ( start , end ) <EOL> elif not hosts : <EOL> start , end = <NUM_LIT:0> , count <EOL> indexes = range ( start , end ) <EOL> else : <EOL> if backfill : <EOL> hosts = hosts . order_by ( '<STR_LIT:index>' ) <EOL> host_indexes = set ( [ h . index for h in hosts ] ) <EOL> last_index = sorted ( host_indexes ) [ - <NUM_LIT:1> ] <EOL> expected_indexes = set ( range ( last_index + <NUM_LIT:1> ) ) <EOL> gaps = expected_indexes - host_indexes <EOL> indexes = [ ] <EOL> if gaps : <EOL> indexes = list ( gaps ) <EOL> count -= len ( indexes ) <EOL> start = sorted ( host_indexes ) [ - <NUM_LIT:1> ] + <NUM_LIT:1> <EOL> end = start + count <EOL> indexes += range ( start , end ) <EOL> else : <EOL> start = hosts . order_by ( '<STR_LIT>' ) [ <NUM_LIT:0> ] . index + <NUM_LIT:1> <EOL> end = start + count <EOL> indexes = xrange ( start , end ) <EOL> components = hostdef . formula_components . all ( ) <EOL> for i in indexes : <EOL> hostname = hostdef . hostname_template . format ( <EOL> namespace = self . namespace , <EOL> index = i <EOL> ) <EOL> kwargs = dict ( <EOL> index = i , <EOL> cloud_image = hostdef . cloud_image , <EOL> blueprint_host_definition = hostdef , <EOL> instance_size = hostdef . size , <EOL> hostname = hostname , <EOL> sir_price = hostdef . spot_price , <EOL> state = Host . PENDING <EOL> ) <EOL> if hostdef . cloud_image . account . vpc_enabled : <EOL> kwargs [ '<STR_LIT>' ] = hostdef . subnet_id <EOL> else : <EOL> kwargs [ '<STR_LIT>' ] = hostdef . zone <EOL> host = self . hosts . create ( ** kwargs ) <EOL> account = host . cloud_image . account <EOL> account_groups = set ( list ( <EOL> account . security_groups . filter ( <EOL> is_default = True <EOL> ) <EOL> ) ) <EOL> host . security_groups . add ( * account_groups ) <EOL> if account . create_security_groups : <EOL> security_group = SecurityGroup . objects . get ( <EOL> stack = self , <EOL> blueprint_host_definition = hostdef <EOL> ) <EOL> host . security_groups . add ( security_group ) <EOL> host . formula_components . add ( * components ) <EOL> for volumedef in hostdef . volumes . all ( ) : <EOL> self . volumes . create ( <EOL> host = host , <EOL> snapshot = volumedef . snapshot , <EOL> hostname = hostname , <EOL> device = volumedef . device , <EOL> mount_point = volumedef . mount_point <EOL> ) <EOL> created_hosts . append ( host ) <EOL> return created_hosts <EOL> def generate_cloud_map ( self ) : <EOL> master = socket . getfqdn ( ) <EOL> images = { } <EOL> hosts = self . hosts . all ( ) <EOL> cluster_size = len ( hosts ) <EOL> for host in hosts : <EOL> cloud_account = host . cloud_image . account <EOL> cloud_account_yaml = yaml . safe_load ( cloud_account . yaml ) [ cloud_account . slug ] <EOL> roles = [ c . sls_path for c in host . formula_components . all ( ) ] <EOL> instance_size = host . instance_size . title <EOL> security_groups = set ( [ <EOL> sg . group_id for sg in host . security_groups . all ( ) <EOL> ] ) <EOL> volumes = host . volumes . all ( ) <EOL> domain = cloud_account_yaml [ '<STR_LIT>' ] <EOL> fqdn = '<STR_LIT>' . format ( host . hostname , domain ) <EOL> map_volumes = [ ] <EOL> for vol in volumes : <EOL> v = { <EOL> '<STR_LIT>' : vol . device , <EOL> '<STR_LIT>' : vol . mount_point , <EOL> '<STR_LIT>' : vol . snapshot . filesystem_type , <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> } <EOL> if vol . volume_id : <EOL> v [ '<STR_LIT>' ] = vol . volume_id <EOL> else : <EOL> v [ '<STR_LIT>' ] = vol . snapshot . snapshot_id <EOL> map_volumes . append ( v ) <EOL> host_metadata = { <EOL> '<STR_LIT:name>' : host . hostname , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : master , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : [ ] <EOL> } , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : roles , <EOL> '<STR_LIT>' : int ( self . pk ) , <EOL> '<STR_LIT>' : fqdn , <EOL> '<STR_LIT>' : domain , <EOL> '<STR_LIT>' : cluster_size , <EOL> '<STR_LIT>' : self . pillar_file . path , <EOL> '<STR_LIT>' : map_volumes , <EOL> '<STR_LIT>' : host . cloud_image . account . slug , <EOL> '<STR_LIT>' : host . cloud_image . slug , <EOL> '<STR_LIT>' : self . namespace , <EOL> } , <EOL> } , <EOL> '<STR_LIT:size>' : instance_size , <EOL> '<STR_LIT>' : list ( security_groups ) , <EOL> '<STR_LIT>' : map_volumes , <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : True , <EOL> } <EOL> if cloud_account . vpc_enabled : <EOL> host_metadata [ '<STR_LIT>' ] = host . subnet_id <EOL> else : <EOL> host_metadata [ '<STR_LIT>' ] = host . availability_zone . title <EOL> if host . sir_price : <EOL> host_metadata [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : str ( host . sir_price ) <EOL> } <EOL> images . setdefault ( host . cloud_image . slug , { } ) [ host . hostname ] = host_metadata <EOL> return images <EOL> def generate_map_file ( self ) : <EOL> images = self . generate_cloud_map ( ) <EOL> map_file_yaml = yaml . safe_dump ( images , default_flow_style = False ) <EOL> if not self . map_file : <EOL> self . map_file . save ( '<STR_LIT>' , ContentFile ( map_file_yaml ) ) <EOL> else : <EOL> with open ( self . map_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( map_file_yaml ) <EOL> def generate_top_file ( self ) : <EOL> top_file_data = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' . format ( self . pk ) : [ <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> '<STR_LIT>' , <EOL> ] <EOL> } <EOL> } <EOL> top_file_yaml = yaml . safe_dump ( top_file_data , default_flow_style = False ) <EOL> if not self . top_file : <EOL> self . top_file . save ( '<STR_LIT>' . format ( self . pk ) , ContentFile ( top_file_yaml ) ) <EOL> else : <EOL> with open ( self . top_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( top_file_yaml ) <EOL> def generate_orchestrate_file ( self ) : <EOL> hosts = self . hosts . all ( ) <EOL> stack_target = '<STR_LIT>' . format ( self . pk ) <EOL> def _matcher ( sls_set ) : <EOL> return '<STR_LIT>' . join ( <EOL> [ stack_target ] + [ '<STR_LIT>' . format ( i ) for i in sls_set ] <EOL> ) <EOL> groups = { } <EOL> for host in hosts : <EOL> for component in host . formula_components . all ( ) : <EOL> groups . setdefault ( component . order , set ( ) ) . add ( component . sls_path ) <EOL> orchestrate = { } <EOL> for order in sorted ( groups . keys ( ) ) : <EOL> for role in groups [ order ] : <EOL> orchestrate [ role ] = { <EOL> '<STR_LIT>' : [ <EOL> { '<STR_LIT>' : _matcher ( [ role ] ) } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : role } , <EOL> ] <EOL> } <EOL> depend = order - <NUM_LIT:1> <EOL> while depend >= <NUM_LIT:0> : <EOL> if depend in groups . keys ( ) : <EOL> orchestrate [ role ] [ '<STR_LIT>' ] . append ( <EOL> { '<STR_LIT>' : [ { '<STR_LIT>' : req } for req in groups [ depend ] ] } <EOL> ) <EOL> break <EOL> depend -= <NUM_LIT:1> <EOL> yaml_data = yaml . safe_dump ( orchestrate , default_flow_style = False ) <EOL> if not self . orchestrate_file : <EOL> self . orchestrate_file . save ( '<STR_LIT>' , ContentFile ( yaml_data ) ) <EOL> else : <EOL> with open ( self . orchestrate_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( yaml_data ) <EOL> def generate_global_orchestrate_file ( self ) : <EOL> accounts = set ( [ host . cloud_image . account for host in self . hosts . all ( ) ] ) <EOL> orchestrate = { } <EOL> for account in accounts : <EOL> target = '<STR_LIT>' . format ( <EOL> self . id , <EOL> account . slug ) <EOL> groups = { } <EOL> for component in account . formula_components . all ( ) : <EOL> groups . setdefault ( component . order , set ( ) ) . add ( component . sls_path ) <EOL> for order in sorted ( groups . keys ( ) ) : <EOL> for role in groups [ order ] : <EOL> state_title = '<STR_LIT>' . format ( account . slug , role ) <EOL> orchestrate [ state_title ] = { <EOL> '<STR_LIT>' : [ <EOL> { '<STR_LIT>' : target } , <EOL> { '<STR_LIT>' : '<STR_LIT>' } , <EOL> { '<STR_LIT>' : role } , <EOL> ] <EOL> } <EOL> depend = order - <NUM_LIT:1> <EOL> while depend >= <NUM_LIT:0> : <EOL> if depend in groups . keys ( ) : <EOL> orchestrate [ role ] [ '<STR_LIT>' ] . append ( <EOL> { '<STR_LIT>' : [ { '<STR_LIT>' : req } for req in groups [ depend ] ] } <EOL> ) <EOL> break <EOL> depend -= <NUM_LIT:1> <EOL> yaml_data = yaml . safe_dump ( orchestrate , default_flow_style = False ) <EOL> if not self . global_orchestrate_file : <EOL> self . global_orchestrate_file . save ( '<STR_LIT>' , ContentFile ( yaml_data ) ) <EOL> else : <EOL> with open ( self . global_orchestrate_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( yaml_data ) <EOL> def generate_pillar_file ( self , update_formulas = False ) : <EOL> from stackdio . api . formulas . models import FormulaVersion <EOL> from stackdio . api . formulas . tasks import update_formula <EOL> users = [ ] <EOL> if self . create_users : <EOL> user_permissions_map = get_users_with_perms ( <EOL> self , attach_perms = True , with_superusers = True , with_group_users = True <EOL> ) <EOL> for user , perms in user_permissions_map . items ( ) : <EOL> if '<STR_LIT>' in perms : <EOL> if user . settings . public_key : <EOL> logger . debug ( '<STR_LIT>' . format ( <EOL> user . username , <EOL> self . title , <EOL> ) ) <EOL> users . append ( { <EOL> '<STR_LIT:username>' : user . username , <EOL> '<STR_LIT>' : user . settings . public_key , <EOL> '<STR_LIT:id>' : user . id , <EOL> } ) <EOL> else : <EOL> logger . debug ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( <EOL> user . username , <EOL> self . title , <EOL> ) <EOL> ) <EOL> pillar_props = { <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT>' : users <EOL> } <EOL> } <EOL> formulas = set ( ) <EOL> for host in self . hosts . all ( ) : <EOL> formulas . update ( [ c . formula for c in host . formula_components . all ( ) ] ) <EOL> if update_formulas : <EOL> for formula in formulas : <EOL> if formula . private_git_repo : <EOL> logger . debug ( '<STR_LIT>' . format ( formula . uri ) ) <EOL> continue <EOL> try : <EOL> version = self . formula_versions . get ( formula = formula ) . version <EOL> except FormulaVersion . DoesNotExist : <EOL> version = formula . default_version <EOL> update_formula . si ( formula . id , None , version , raise_exception = False ) ( ) <EOL> for formula in formulas : <EOL> recursive_update ( pillar_props , formula . properties ) <EOL> recursive_update ( pillar_props , self . properties ) <EOL> pillar_file_yaml = yaml . safe_dump ( pillar_props , default_flow_style = False ) <EOL> if not self . pillar_file : <EOL> self . pillar_file . save ( '<STR_LIT>' , ContentFile ( pillar_file_yaml ) ) <EOL> else : <EOL> with open ( self . pillar_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( pillar_file_yaml ) <EOL> def generate_global_pillar_file ( self , update_formulas = False ) : <EOL> from stackdio . api . formulas . models import FormulaVersion <EOL> from stackdio . api . formulas . tasks import update_formula <EOL> pillar_props = { } <EOL> accounts = set ( <EOL> [ host . cloud_image . account for <EOL> host in self . hosts . all ( ) ] <EOL> ) <EOL> global_formulas = [ ] <EOL> for account in accounts : <EOL> global_formulas . extend ( account . get_formulas ( ) ) <EOL> if update_formulas : <EOL> for formula in global_formulas : <EOL> if formula . private_git_repo : <EOL> logger . debug ( '<STR_LIT>' . format ( formula . uri ) ) <EOL> continue <EOL> try : <EOL> version = self . formula_versions . get ( formula = formula ) . version <EOL> except FormulaVersion . DoesNotExist : <EOL> version = formula . default_version <EOL> update_formula . si ( formula . id , None , version , raise_exception = False ) ( ) <EOL> for formula in set ( global_formulas ) : <EOL> recursive_update ( pillar_props , formula . properties ) <EOL> for account in accounts : <EOL> recursive_update ( pillar_props , <EOL> account . global_orchestration_properties ) <EOL> pillar_file_yaml = yaml . safe_dump ( pillar_props , default_flow_style = False ) <EOL> if not self . global_pillar_file : <EOL> self . global_pillar_file . save ( '<STR_LIT>' , ContentFile ( pillar_file_yaml ) ) <EOL> else : <EOL> with open ( self . global_pillar_file . path , '<STR_LIT:w>' ) as f : <EOL> f . write ( pillar_file_yaml ) <EOL> def query_hosts ( self , force = False ) : <EOL> """<STR_LIT>""" <EOL> CACHE_KEY = '<STR_LIT>' <EOL> cached_result = cache . get ( CACHE_KEY ) <EOL> if cached_result and not force : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> result = cached_result <EOL> else : <EOL> logger . debug ( '<STR_LIT>' ) <EOL> logger . info ( '<STR_LIT>' . format ( self ) ) <EOL> salt_cloud = salt . cloud . CloudClient ( settings . STACKDIO_CONFIG . salt_cloud_config ) <EOL> result = salt_cloud . full_query ( ) <EOL> cache . set ( CACHE_KEY , result , <NUM_LIT> ) <EOL> host_result = { } <EOL> for host in self . hosts . all ( ) : <EOL> account = host . get_account ( ) <EOL> provider = account . provider <EOL> host_map = result . get ( account . slug , { } ) . get ( provider . name , { } ) <EOL> host_result [ host . hostname ] = host_map . get ( host . hostname , None ) <EOL> return host_result <EOL> def get_root_directory ( self ) : <EOL> if self . map_file : <EOL> return os . path . dirname ( self . map_file . path ) <EOL> if self . props_file : <EOL> return os . path . dirname ( self . props_file . path ) <EOL> return None <EOL> def get_log_directory ( self ) : <EOL> root_dir = self . get_root_directory ( ) <EOL> log_dir = os . path . join ( root_dir , '<STR_LIT>' ) <EOL> if not os . path . isdir ( log_dir ) : <EOL> os . makedirs ( log_dir ) <EOL> return log_dir <EOL> def get_security_groups ( self ) : <EOL> return SecurityGroup . objects . filter ( is_managed = True , <EOL> hosts__stack = self ) . distinct ( ) <EOL> def get_role_list ( self ) : <EOL> roles = set ( ) <EOL> for bhd in self . blueprint . host_definitions . all ( ) : <EOL> for formula_component in bhd . formula_components . all ( ) : <EOL> roles . add ( formula_component . sls_path ) <EOL> return list ( roles ) <EOL> class StackHistory ( TimeStampedModel , StatusDetailModel ) : <EOL> class Meta : <EOL> verbose_name_plural = '<STR_LIT>' <EOL> ordering = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> default_permissions = ( ) <EOL> STATUS = Stack . STATUS <EOL> stack = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> event = models . CharField ( max_length = <NUM_LIT> ) <EOL> level = models . CharField ( max_length = <NUM_LIT:16> , choices = ( <EOL> ( Level . DEBUG , Level . DEBUG ) , <EOL> ( Level . INFO , Level . INFO ) , <EOL> ( Level . WARN , Level . WARN ) , <EOL> ( Level . ERROR , Level . ERROR ) , <EOL> ) ) <EOL> class StackCommand ( TimeStampedModel , StatusModel ) : <EOL> WAITING = '<STR_LIT>' <EOL> RUNNING = '<STR_LIT>' <EOL> FINISHED = '<STR_LIT>' <EOL> ERROR = '<STR_LIT:error>' <EOL> STATUS = Choices ( WAITING , RUNNING , FINISHED , ERROR ) <EOL> class Meta : <EOL> verbose_name_plural = '<STR_LIT>' <EOL> default_permissions = ( ) <EOL> stack = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> start = models . DateTimeField ( '<STR_LIT>' , blank = True , default = now ) <EOL> host_target = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:255> ) <EOL> command = models . TextField ( '<STR_LIT>' ) <EOL> std_out_storage = models . TextField ( ) <EOL> std_err_storage = models . TextField ( ) <EOL> @ property <EOL> def std_out ( self ) : <EOL> if self . std_out_storage != "<STR_LIT>" : <EOL> return json . loads ( self . std_out_storage ) <EOL> else : <EOL> return [ ] <EOL> @ property <EOL> def std_err ( self ) : <EOL> return self . std_err_storage <EOL> @ property <EOL> def submit_time ( self ) : <EOL> return self . created <EOL> @ property <EOL> def start_time ( self ) : <EOL> if self . status in ( self . RUNNING , self . FINISHED ) : <EOL> return self . start <EOL> else : <EOL> return '<STR_LIT>' <EOL> @ property <EOL> def finish_time ( self ) : <EOL> if self . status == self . FINISHED : <EOL> return self . modified <EOL> else : <EOL> return '<STR_LIT>' <EOL> class Host ( TimeStampedModel , StatusDetailModel ) : <EOL> PENDING = '<STR_LIT>' <EOL> OK = '<STR_LIT>' <EOL> DELETING = '<STR_LIT>' <EOL> STATUS = Choices ( PENDING , OK , DELETING ) <EOL> class Meta : <EOL> ordering = [ '<STR_LIT>' , '<STR_LIT>' ] <EOL> default_permissions = ( ) <EOL> stack = models . ForeignKey ( '<STR_LIT>' , <EOL> related_name = '<STR_LIT>' ) <EOL> cloud_image = models . ForeignKey ( '<STR_LIT>' , <EOL> related_name = '<STR_LIT>' ) <EOL> instance_size = models . ForeignKey ( '<STR_LIT>' , <EOL> related_name = '<STR_LIT>' ) <EOL> availability_zone = models . ForeignKey ( '<STR_LIT>' , <EOL> null = True , <EOL> related_name = '<STR_LIT>' ) <EOL> subnet_id = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:32> , blank = True , default = '<STR_LIT>' ) <EOL> blueprint_host_definition = models . ForeignKey ( <EOL> '<STR_LIT>' , <EOL> related_name = '<STR_LIT>' ) <EOL> hostname = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:64> ) <EOL> index = models . IntegerField ( '<STR_LIT>' ) <EOL> security_groups = models . ManyToManyField ( '<STR_LIT>' , <EOL> related_name = '<STR_LIT>' ) <EOL> state = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:32> , default = '<STR_LIT>' ) <EOL> state_reason = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:255> , default = '<STR_LIT>' , blank = True ) <EOL> provider_dns = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:64> , blank = True ) <EOL> provider_private_dns = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:64> , blank = True ) <EOL> provider_private_ip = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:64> , blank = True ) <EOL> fqdn = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:255> , blank = True ) <EOL> instance_id = models . CharField ( '<STR_LIT>' , max_length = <NUM_LIT:32> , blank = True ) <EOL> sir_id = models . CharField ( '<STR_LIT>' , <EOL> max_length = <NUM_LIT:32> , <EOL> default = '<STR_LIT>' ) <EOL> sir_price = models . DecimalField ( '<STR_LIT>' , <EOL> max_digits = <NUM_LIT:5> , <EOL> decimal_places = <NUM_LIT:2> , <EOL> null = True ) <EOL> def __unicode__ ( self ) : <EOL> return self . hostname <EOL> @ property <EOL> def provider_metadata ( self ) : <EOL> metadata = self . stack . query_hosts ( ) <EOL> return metadata [ self . hostname ] <EOL> @ property <EOL> def formula_components ( self ) : <EOL> return self . blueprint_host_definition . formula_components <EOL> def get_account ( self ) : <EOL> return self . cloud_image . account <EOL> def get_provider ( self ) : <EOL> return self . get_account ( ) . provider <EOL> def get_driver ( self ) : <EOL> return self . cloud_image . get_driver ( ) </s>
