original_docstring: 
      Main task to run a full site build process.
  
      All values needed for the build are loaded from
      environment variables.
      
docstring: Main task to run a full site build process.
  All values needed for the build are loaded from
  environment variables.
docstring_tokens: Main task to run a full site build process . All values needed for the build are loaded from environment variables .
code: def main(ctx):
      '''
      Main task to run a full site build process.
  
      All values needed for the build are loaded from
      environment variables.
      '''
      # (variable naming)
      # pylint: disable=C0103
  
      # keep track of total time
      start_time = datetime.now()
  
      # Load from .env for development
      load_dotenv()
  
      # These environment variables will be set into the environment
      # by federalist-builder.
  
      # During development, we can use a `.env` file (loaded above)
      # to make it easier to specify variables.
  
      AWS_DEFAULT_REGION = os.environ['AWS_DEFAULT_REGION']
      BUCKET = os.environ['BUCKET']
      BASEURL = os.environ['BASEURL']
      CACHE_CONTROL = os.environ['CACHE_CONTROL']
      BRANCH = os.environ['BRANCH']
      CONFIG = os.environ['CONFIG']
      REPOSITORY = os.environ['REPOSITORY']
      OWNER = os.environ['OWNER']
      SITE_PREFIX = os.environ['SITE_PREFIX']
      GENERATOR = os.environ['GENERATOR']
  
      AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
      AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
  
      # List of private strings to be removed from any posted logs
      private_values = [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY]
  
      # Optional environment variables
      SOURCE_REPO = os.getenv('SOURCE_REPO', '')
      SOURCE_OWNER = os.getenv('SOURCE_OWNER', '')
  
      # GITHUB_TOKEN can be empty if a non-Federalist user
      # makes a commit to a repo and thus initiates a build for a
      # Federalist site
      GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
      if GITHUB_TOKEN:
          # only include it in list of values to strip from log output
          # if it exists
          private_values.append(GITHUB_TOKEN)
  
      # Ex: https://federalist-builder.fr.cloud.gov/builds/<token>/callback
      FEDERALIST_BUILDER_CALLBACK = os.environ['FEDERALIST_BUILDER_CALLBACK']
  
      # Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/status/<token>
      STATUS_CALLBACK = os.environ['STATUS_CALLBACK']
  
      # Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/log/<token>
      LOG_CALLBACK = os.environ['LOG_CALLBACK']
  
      try:
          # throw a timeout exception after TIMEOUT_SECONDS
          with Timeout(TIMEOUT_SECONDS, swallow_exc=False):
              LOGGER.info(f'Running build for {OWNER}/{REPOSITORY}/{BRANCH}')
  
              # Unfortunately, pyinvoke doesn't have a great way to call tasks
              # from within other tasks. If you call them directly,
              # their pre- and post-dependencies are not executed.
              #
              # Here's the GitHub issue about this:
              # https://github.com/pyinvoke/invoke/issues/170
              #
              # So rather than calling the task functions through python, we'll
              # call them instead using `ctx.run('invoke the_task ...')` via the
              # helper `run_task` method.
  
              ##
              # CLONE
              #
              if SOURCE_OWNER and SOURCE_REPO:
                  # First clone the source (ie, template) repository
                  clone_source_flags = {
                      '--owner': SOURCE_OWNER,
                      '--repository': SOURCE_REPO,
                      '--branch': BRANCH,
                  }
  
                  run_task(ctx, 'clone-repo',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values,
                           flags_dict=clone_source_flags,
                           env={'GITHUB_TOKEN': GITHUB_TOKEN})
  
                  # Then push the cloned source repo up to the destination repo.
                  # Note that the dest repo must already exist but be empty.
                  # The Federalist web app takes care of that operation.
                  push_repo_flags = {
                      '--owner': OWNER,
                      '--repository': REPOSITORY,
                      '--branch': BRANCH,
                  }
  
                  run_task(ctx, 'push-repo-remote',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values,
                           flags_dict=push_repo_flags,
                           env={'GITHUB_TOKEN': GITHUB_TOKEN})
              else:
                  # Just clone the given repository
                  clone_flags = {
                      '--owner': OWNER,
                      '--repository': REPOSITORY,
                      '--branch': BRANCH,
                  }
  
                  run_task(ctx, 'clone-repo',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values,
                           flags_dict=clone_flags,
                           env={'GITHUB_TOKEN': GITHUB_TOKEN})
  
              ##
              # BUILD
              #
              build_flags = {
                  '--branch': BRANCH,
                  '--owner': OWNER,
                  '--repository': REPOSITORY,
                  '--site-prefix': SITE_PREFIX,
                  '--base-url': BASEURL,
              }
  
              # Run the npm `federalist` task (if it is defined)
              run_task(ctx, 'run-federalist-script',
                       log_callback=LOG_CALLBACK,
                       private_values=private_values,
                       flags_dict=build_flags)
  
              # Run the appropriate build engine based on GENERATOR
              if GENERATOR == 'jekyll':
                  build_flags['--config'] = CONFIG
                  run_task(ctx, 'build-jekyll',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values,
                           flags_dict=build_flags)
              elif GENERATOR == 'hugo':
                  # extra: --hugo-version (not yet used)
                  run_task(ctx, 'build-hugo',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values,
                           flags_dict=build_flags)
              elif GENERATOR == 'static':
                  # no build arguments are needed
                  run_task(ctx, 'build-static',
                           log_callback=LOG_CALLBACK,
                           private_values=private_values)
              else:
                  raise ValueError(f'Invalid GENERATOR: {GENERATOR}')
  
              ##
              # PUBLISH
              #
              publish_flags = {
                  '--base-url': BASEURL,
                  '--site-prefix': SITE_PREFIX,
                  '--bucket': BUCKET,
                  '--cache-control': CACHE_CONTROL,
                  '--aws-region': AWS_DEFAULT_REGION,
              }
  
              publish_env = {
                  'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID,
                  'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY,
              }
  
              run_task(ctx, 'publish',
                       log_callback=LOG_CALLBACK,
                       private_values=private_values,
                       flags_dict=publish_flags,
                       env=publish_env)
  
              delta_string = delta_to_mins_secs(datetime.now() - start_time)
              LOGGER.info(
                  f'Total build time: {delta_string}')
  
              # Finished!
              post_build_complete(STATUS_CALLBACK,
                                  FEDERALIST_BUILDER_CALLBACK)
  
      except TimeoutException:
          LOGGER.info('Build has timed out')
          post_build_timeout(LOG_CALLBACK,
                             STATUS_CALLBACK,
                             FEDERALIST_BUILDER_CALLBACK)
      except UnexpectedExit as err:
          # replace any private values that might be in the error message
          err_string = replace_private_values(err.result.stderr,
                                              private_values)
  
          # log the original exception
          LOGGER.info(f'Exception raised during build:')
          LOGGER.info(err_string)
  
          # replace the message with a custom one, if it exists
          err_string = find_custom_error_message(err_string)
  
          post_build_error(LOG_CALLBACK,
                           STATUS_CALLBACK,
                           FEDERALIST_BUILDER_CALLBACK,
                           err_string)
      except Exception as err:  # pylint: disable=W0703
          err_string = str(err)
          err_string = replace_private_values(err_string,
                                              private_values)
  
          LOGGER.info('Unexpected exception raised during build:')
          LOGGER.info(err_string)
  
          err_message = ('Unexpected error. Please try again and '
                         'contact federalist-support if it persists.')
  
          post_build_error(LOG_CALLBACK,
                           STATUS_CALLBACK,
                           FEDERALIST_BUILDER_CALLBACK,
                           err_message)
code_tokens: def main ( ctx ) : start_time = datetime . now ( ) load_dotenv ( ) AWS_DEFAULT_REGION = os . environ [ 'AWS_DEFAULT_REGION' ] BUCKET = os . environ [ 'BUCKET' ] BASEURL = os . environ [ 'BASEURL' ] CACHE_CONTROL = os . environ [ 'CACHE_CONTROL' ] BRANCH = os . environ [ 'BRANCH' ] CONFIG = os . environ [ 'CONFIG' ] REPOSITORY = os . environ [ 'REPOSITORY' ] OWNER = os . environ [ 'OWNER' ] SITE_PREFIX = os . environ [ 'SITE_PREFIX' ] GENERATOR = os . environ [ 'GENERATOR' ] AWS_ACCESS_KEY_ID = os . environ [ 'AWS_ACCESS_KEY_ID' ] AWS_SECRET_ACCESS_KEY = os . environ [ 'AWS_SECRET_ACCESS_KEY' ] private_values = [ AWS_ACCESS_KEY_ID , AWS_SECRET_ACCESS_KEY ] SOURCE_REPO = os . getenv ( 'SOURCE_REPO' , '' ) SOURCE_OWNER = os . getenv ( 'SOURCE_OWNER' , '' ) GITHUB_TOKEN = os . getenv ( 'GITHUB_TOKEN' , '' ) if GITHUB_TOKEN : private_values . append ( GITHUB_TOKEN ) FEDERALIST_BUILDER_CALLBACK = os . environ [ 'FEDERALIST_BUILDER_CALLBACK' ] STATUS_CALLBACK = os . environ [ 'STATUS_CALLBACK' ] LOG_CALLBACK = os . environ [ 'LOG_CALLBACK' ] try : with Timeout ( TIMEOUT_SECONDS , swallow_exc = False ) : LOGGER . info ( f'Running build for {OWNER}/{REPOSITORY}/{BRANCH}' ) if SOURCE_OWNER and SOURCE_REPO : clone_source_flags = { '--owner' : SOURCE_OWNER , '--repository' : SOURCE_REPO , '--branch' : BRANCH , } run_task ( ctx , 'clone-repo' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = clone_source_flags , env = { 'GITHUB_TOKEN' : GITHUB_TOKEN } ) push_repo_flags = { '--owner' : OWNER , '--repository' : REPOSITORY , '--branch' : BRANCH , } run_task ( ctx , 'push-repo-remote' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = push_repo_flags , env = { 'GITHUB_TOKEN' : GITHUB_TOKEN } ) else : clone_flags = { '--owner' : OWNER , '--repository' : REPOSITORY , '--branch' : BRANCH , } run_task ( ctx , 'clone-repo' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = clone_flags , env = { 'GITHUB_TOKEN' : GITHUB_TOKEN } ) build_flags = { '--branch' : BRANCH , '--owner' : OWNER , '--repository' : REPOSITORY , '--site-prefix' : SITE_PREFIX , '--base-url' : BASEURL , } run_task ( ctx , 'run-federalist-script' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = build_flags ) if GENERATOR == 'jekyll' : build_flags [ '--config' ] = CONFIG run_task ( ctx , 'build-jekyll' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = build_flags ) elif GENERATOR == 'hugo' : run_task ( ctx , 'build-hugo' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = build_flags ) elif GENERATOR == 'static' : run_task ( ctx , 'build-static' , log_callback = LOG_CALLBACK , private_values = private_values ) else : raise ValueError ( f'Invalid GENERATOR: {GENERATOR}' ) publish_flags = { '--base-url' : BASEURL , '--site-prefix' : SITE_PREFIX , '--bucket' : BUCKET , '--cache-control' : CACHE_CONTROL , '--aws-region' : AWS_DEFAULT_REGION , } publish_env = { 'AWS_ACCESS_KEY_ID' : AWS_ACCESS_KEY_ID , 'AWS_SECRET_ACCESS_KEY' : AWS_SECRET_ACCESS_KEY , } run_task ( ctx , 'publish' , log_callback = LOG_CALLBACK , private_values = private_values , flags_dict = publish_flags , env = publish_env ) delta_string = delta_to_mins_secs ( datetime . now ( ) - start_time ) LOGGER . info ( f'Total build time: {delta_string}' ) post_build_complete ( STATUS_CALLBACK , FEDERALIST_BUILDER_CALLBACK ) except TimeoutException : LOGGER . info ( 'Build has timed out' ) post_build_timeout ( LOG_CALLBACK , STATUS_CALLBACK , FEDERALIST_BUILDER_CALLBACK ) except UnexpectedExit as err : err_string = replace_private_values ( err . result . stderr , private_values ) LOGGER . info ( f'Exception raised during build:' ) LOGGER . info ( err_string ) err_string = find_custom_error_message ( err_string ) post_build_error ( LOG_CALLBACK , STATUS_CALLBACK , FEDERALIST_BUILDER_CALLBACK , err_string ) except Exception as err : err_string = str ( err ) err_string = replace_private_values ( err_string , private_values ) LOGGER . info ( 'Unexpected exception raised during build:' ) LOGGER . info ( err_string ) err_message = ( 'Unexpected error. Please try again and ' 'contact federalist-support if it persists.' ) post_build_error ( LOG_CALLBACK , STATUS_CALLBACK , FEDERALIST_BUILDER_CALLBACK , err_message )
short_docstring: Main task to run a full site build process.
short_docstring_tokens: Main task to run a full site build process .
comment: '''
      Main task to run a full site build process.
  
      All values needed for the build are loaded from
      environment variables.
      '''$SEP$# (variable naming)$SEP$# pylint: disable=C0103$SEP$# keep track of total time$SEP$# Load from .env for development$SEP$# These environment variables will be set into the environment$SEP$# by federalist-builder.$SEP$# During development, we can use a `.env` file (loaded above)$SEP$# to make it easier to specify variables.$SEP$# List of private strings to be removed from any posted logs$SEP$# Optional environment variables$SEP$# GITHUB_TOKEN can be empty if a non-Federalist user$SEP$# makes a commit to a repo and thus initiates a build for a$SEP$# Federalist site$SEP$# only include it in list of values to strip from log output$SEP$# if it exists$SEP$# Ex: https://federalist-builder.fr.cloud.gov/builds/<token>/callback$SEP$# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/status/<token>$SEP$# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/log/<token>$SEP$# throw a timeout exception after TIMEOUT_SECONDS$SEP$# Unfortunately, pyinvoke doesn't have a great way to call tasks$SEP$# from within other tasks. If you call them directly,$SEP$# their pre- and post-dependencies are not executed.$SEP$#$SEP$# Here's the GitHub issue about this:$SEP$# https://github.com/pyinvoke/invoke/issues/170$SEP$#$SEP$# So rather than calling the task functions through python, we'll$SEP$# call them instead using `ctx.run('invoke the_task ...')` via the$SEP$# helper `run_task` method.$SEP$##$SEP$# CLONE$SEP$#$SEP$# First clone the source (ie, template) repository$SEP$# Then push the cloned source repo up to the destination repo.$SEP$# Note that the dest repo must already exist but be empty.$SEP$# The Federalist web app takes care of that operation.$SEP$# Just clone the given repository$SEP$##$SEP$# BUILD$SEP$#$SEP$# Run the npm `federalist` task (if it is defined)$SEP$# Run the appropriate build engine based on GENERATOR$SEP$# extra: --hugo-version (not yet used)$SEP$# no build arguments are needed$SEP$##$SEP$# PUBLISH$SEP$#$SEP$# Finished!$SEP$# replace any private values that might be in the error message$SEP$# log the original exception$SEP$# replace the message with a custom one, if it exists$SEP$# pylint: disable=W0703