def create_lambda_zip(  self,
                            prefix='lambda_package',
                            handler_file=None,
                            slim_handler=False,
                            minify=True,
                            exclude=None,
                            use_precompiled_packages=True,
                            include=None,
                            venv=None,
                            output=None,
                            disable_progress=False,
                            archive_format='zip'
                        ):
        """
        Create a Lambda-ready zip file of the current virtualenvironment and working directory.

        Returns path to that file.

        """
        # Validate archive_format
        if archive_format not in ['zip', 'tarball']:
            raise KeyError("The archive format to create a lambda package must be zip or tarball")

        # Pip is a weird package.
        # Calling this function in some environments without this can cause.. funkiness.
        import pip

        if not venv:
            venv = self.get_current_venv()

        build_time = str(int(time.time()))
        cwd = os.getcwd()
        if not output:
            if archive_format == 'zip':
                archive_fname = prefix + '-' + build_time + '.zip'
            elif archive_format == 'tarball':
                archive_fname = prefix + '-' + build_time + '.tar.gz'
        else:
            archive_fname = output
        archive_path = os.path.join(cwd, archive_fname)

        # Files that should be excluded from the zip
        if exclude is None:
            exclude = list()

        # Exclude the zip itself
        exclude.append(archive_path)

        # Make sure that 'concurrent' is always forbidden.
        # https://github.com/Miserlou/Zappa/issues/827
        if not 'concurrent' in exclude:
            exclude.append('concurrent')

        def splitpath(path):
            parts = []
            (path, tail) = os.path.split(path)
            while path and tail:
                parts.append(tail)
                (path, tail) = os.path.split(path)
            parts.append(os.path.join(path, tail))
            return list(map(os.path.normpath, parts))[::-1]
        split_venv = splitpath(venv)
        split_cwd = splitpath(cwd)

        # Ideally this should be avoided automatically,
        # but this serves as an okay stop-gap measure.
        if split_venv[-1] == split_cwd[-1]:  # pragma: no cover
            print(
                "Warning! Your project and virtualenv have the same name! You may want "
                "to re-create your venv with a new name, or explicitly define a "
                "'project_name', as this may cause errors."
            )

        # First, do the project..
        temp_project_path = tempfile.mkdtemp(prefix='zappa-project')

        if not slim_handler:
            # Slim handler does not take the project files.
            if minify:
                # Related: https://github.com/Miserlou/Zappa/issues/744
                excludes = ZIP_EXCLUDES + exclude + [split_venv[-1]]
                copytree(cwd, temp_project_path, metadata=False, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(cwd, temp_project_path, metadata=False, symlinks=False)

        # If a handler_file is supplied, copy that to the root of the package,
        # because that's where AWS Lambda looks for it. It can't be inside a package.
        if handler_file:
            filename = handler_file.split(os.sep)[-1]
            shutil.copy(handler_file, os.path.join(temp_project_path, filename))

        # Create and populate package ID file and write to temp project path
        package_info = {}
        package_info['uuid'] = str(uuid.uuid4())
        package_info['build_time'] = build_time
        package_info['build_platform'] = os.sys.platform
        package_info['build_user'] = getpass.getuser()
        # TODO: Add git head and info?

        # Ex, from @scoates:
        # def _get_git_branch():
        #     chdir(DIR)
        #     out = check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).strip()
        #     lambci_branch = environ.get('LAMBCI_BRANCH', None)
        #     if out == "HEAD" and lambci_branch:
        #         out += " lambci:{}".format(lambci_branch)
        #     return out

        # def _get_git_hash():
        #     chdir(DIR)
        #     return check_output(['git', 'rev-parse', 'HEAD']).strip()

        # def _get_uname():
        #     return check_output(['uname', '-a']).strip()

        # def _get_user():
        #     return check_output(['whoami']).strip()

        # def set_id_info(zappa_cli):
        #     build_info = {
        #         'branch': _get_git_branch(),
        #         'hash': _get_git_hash(),
        #         'build_uname': _get_uname(),
        #         'build_user': _get_user(),
        #         'build_time': datetime.datetime.utcnow().isoformat(),
        #     }
        #     with open(path.join(DIR, 'id_info.json'), 'w') as f:
        #         json.dump(build_info, f)
        #     return True

        package_id_file = open(os.path.join(temp_project_path, 'package_info.json'), 'w')
        dumped = json.dumps(package_info, indent=4)
        try:
            package_id_file.write(dumped)
        except TypeError: # This is a Python 2/3 issue. TODO: Make pretty!
            package_id_file.write(unicode(dumped))
        package_id_file.close()

        # Then, do site site-packages..
        egg_links = []
        temp_package_path = tempfile.mkdtemp(prefix='zappa-packages')
        if os.sys.platform == 'win32':
            site_packages = os.path.join(venv, 'Lib', 'site-packages')
        else:
            site_packages = os.path.join(venv, 'lib', get_venv_from_python_version(), 'site-packages')
        egg_links.extend(glob.glob(os.path.join(site_packages, '*.egg-link')))

        if minify:
            excludes = ZIP_EXCLUDES + exclude
            copytree(site_packages, temp_package_path, metadata=False, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
        else:
            copytree(site_packages, temp_package_path, metadata=False, symlinks=False)

        # We may have 64-bin specific packages too.
        site_packages_64 = os.path.join(venv, 'lib64', get_venv_from_python_version(), 'site-packages')
        if os.path.exists(site_packages_64):
            egg_links.extend(glob.glob(os.path.join(site_packages_64, '*.egg-link')))
            if minify:
                excludes = ZIP_EXCLUDES + exclude
                copytree(site_packages_64, temp_package_path, metadata = False, symlinks=False, ignore=shutil.ignore_patterns(*excludes))
            else:
                copytree(site_packages_64, temp_package_path, metadata = False, symlinks=False)

        if egg_links:
            self.copy_editable_packages(egg_links, temp_package_path)

        copy_tree(temp_package_path, temp_project_path, update=True)

        # Then the pre-compiled packages..
        if use_precompiled_packages:
            print("Downloading and installing dependencies..")
            installed_packages = self.get_installed_packages(site_packages, site_packages_64)

            try:
                for installed_package_name, installed_package_version in installed_packages.items():
                    if self.have_correct_lambda_package_version(installed_package_name, installed_package_version):
                        print(" - %s==%s: Using precompiled lambda package " % (installed_package_name, installed_package_version,))
                        self.extract_lambda_package(installed_package_name, temp_project_path)
                    else:
                        cached_wheel_path = self.get_cached_manylinux_wheel(installed_package_name, installed_package_version, disable_progress)
                        if cached_wheel_path:
                            # Otherwise try to use manylinux packages from PyPi..
                            # Related: https://github.com/Miserlou/Zappa/issues/398
                            shutil.rmtree(os.path.join(temp_project_path, installed_package_name), ignore_errors=True)
                            with zipfile.ZipFile(cached_wheel_path) as zfile:
                                zfile.extractall(temp_project_path)

                        elif self.have_any_lambda_package_version(installed_package_name):
                            # Finally see if we may have at least one version of the package in lambda packages
                            # Related: https://github.com/Miserlou/Zappa/issues/855
                            lambda_version = lambda_packages[installed_package_name][self.runtime]['version']
                            print(" - %s==%s: Warning! Using precompiled lambda package version %s instead!" % (installed_package_name, installed_package_version, lambda_version, ))
                            self.extract_lambda_package(installed_package_name, temp_project_path)

                # This is a special case!
                # SQLite3 is part of the _system_ Python, not a package. Still, it lives in `lambda-packages`.
                # Everybody on Python3 gets it!
                if self.runtime in ("python3.6", "python3.7"):
                    print(" - sqlite==python3: Using precompiled lambda package")
                    self.extract_lambda_package('sqlite3', temp_project_path)

            except Exception as e:
                print(e)
                # XXX - What should we do here?

        # Then archive it all up..
        if archive_format == 'zip':
            print("Packaging project as zip.")

            try:
                compression_method = zipfile.ZIP_DEFLATED
            except ImportError:  # pragma: no cover
                compression_method = zipfile.ZIP_STORED
            archivef = zipfile.ZipFile(archive_path, 'w', compression_method)

        elif archive_format == 'tarball':
            print("Packaging project as gzipped tarball.")
            archivef = tarfile.open(archive_path, 'w|gz')

        for root, dirs, files in os.walk(temp_project_path):

            for filename in files:

                # Skip .pyc files for Django migrations
                # https://github.com/Miserlou/Zappa/issues/436
                # https://github.com/Miserlou/Zappa/issues/464
                if filename[-4:] == '.pyc' and root[-10:] == 'migrations':
                    continue

                # If there is a .pyc file in this package,
                # we can skip the python source code as we'll just
                # use the compiled bytecode anyway..
                if filename[-3:] == '.py' and root[-10:] != 'migrations':
                    abs_filname = os.path.join(root, filename)
                    abs_pyc_filename = abs_filname + 'c'
                    if os.path.isfile(abs_pyc_filename):

                        # but only if the pyc is older than the py,
                        # otherwise we'll deploy outdated code!
                        py_time = os.stat(abs_filname).st_mtime
                        pyc_time = os.stat(abs_pyc_filename).st_mtime

                        if pyc_time > py_time:
                            continue

                # Make sure that the files are all correctly chmodded
                # Related: https://github.com/Miserlou/Zappa/issues/484
                # Related: https://github.com/Miserlou/Zappa/issues/682
                os.chmod(os.path.join(root, filename),  0o755)

                if archive_format == 'zip':
                    # Actually put the file into the proper place in the zip
                    # Related: https://github.com/Miserlou/Zappa/pull/716
                    zipi = zipfile.ZipInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
                    zipi.create_system = 3
                    zipi.external_attr = 0o755 << int(16) # Is this P2/P3 functional?
                    with open(os.path.join(root, filename), 'rb') as f:
                        archivef.writestr(zipi, f.read(), compression_method)
                elif archive_format == 'tarball':
                    tarinfo = tarfile.TarInfo(os.path.join(root.replace(temp_project_path, '').lstrip(os.sep), filename))
                    tarinfo.mode = 0o755

                    stat = os.stat(os.path.join(root, filename))
                    tarinfo.mtime = stat.st_mtime
                    tarinfo.size = stat.st_size
                    with open(os.path.join(root, filename), 'rb') as f:
                        archivef.addfile(tarinfo, f)

            # Create python init file if it does not exist
            # Only do that if there are sub folders or python files and does not conflict with a neighbouring module
            # Related: https://github.com/Miserlou/Zappa/issues/766
            if not contains_python_files_or_subdirs(root):
                # if the directory does not contain any .py file at any level, we can skip the rest
                dirs[:] = [d for d in dirs if d != root]
            else:
                if '__init__.py' not in files and not conflicts_with_a_neighbouring_module(root):
                    tmp_init = os.path.join(temp_project_path, '__init__.py')
                    open(tmp_init, 'a').close()
                    os.chmod(tmp_init,  0o755)

                    arcname = os.path.join(root.replace(temp_project_path, ''),
                                           os.path.join(root.replace(temp_project_path, ''), '__init__.py'))
                    if archive_format == 'zip':
                        archivef.write(tmp_init, arcname)
                    elif archive_format == 'tarball':
                        archivef.add(tmp_init, arcname)

        # And, we're done!
        archivef.close()

        # Trash the temp directory
        shutil.rmtree(temp_project_path)
        shutil.rmtree(temp_package_path)
        if os.path.isdir(venv) and slim_handler:
            # Remove the temporary handler venv folder
            shutil.rmtree(venv)

        return archive_fname
def create_lambda_zip ( self , prefix = 'lambda_package' , handler_file = None , slim_handler = False , minify = True , exclude = None , use_precompiled_packages = True , include = None , venv = None , output = None , disable_progress = False , archive_format = 'zip' ) : if archive_format not in [ 'zip' , 'tarball' ] : raise KeyError ( "The archive format to create a lambda package must be zip or tarball" ) import pip if not venv : venv = self . get_current_venv ( ) build_time = str ( int ( time . time ( ) ) ) cwd = os . getcwd ( ) if not output : if archive_format == 'zip' : archive_fname = prefix + '-' + build_time + '.zip' elif archive_format == 'tarball' : archive_fname = prefix + '-' + build_time + '.tar.gz' else : archive_fname = output archive_path = os . path . join ( cwd , archive_fname ) if exclude is None : exclude = list ( ) exclude . append ( archive_path ) if not 'concurrent' in exclude : exclude . append ( 'concurrent' ) def splitpath ( path ) : parts = [ ] ( path , tail ) = os . path . split ( path ) while path and tail : parts . append ( tail ) ( path , tail ) = os . path . split ( path ) parts . append ( os . path . join ( path , tail ) ) return list ( map ( os . path . normpath , parts ) ) [ : : - 1 ] split_venv = splitpath ( venv ) split_cwd = splitpath ( cwd ) if split_venv [ - 1 ] == split_cwd [ - 1 ] : print ( "Warning! Your project and virtualenv have the same name! You may want " "to re-create your venv with a new name, or explicitly define a " "'project_name', as this may cause errors." ) temp_project_path = tempfile . mkdtemp ( prefix = 'zappa-project' ) if not slim_handler : if minify : excludes = ZIP_EXCLUDES + exclude + [ split_venv [ - 1 ] ] copytree ( cwd , temp_project_path , metadata = False , symlinks = False , ignore = shutil . ignore_patterns ( * excludes ) ) else : copytree ( cwd , temp_project_path , metadata = False , symlinks = False ) if handler_file : filename = handler_file . split ( os . sep ) [ - 1 ] shutil . copy ( handler_file , os . path . join ( temp_project_path , filename ) ) package_info = { } package_info [ 'uuid' ] = str ( uuid . uuid4 ( ) ) package_info [ 'build_time' ] = build_time package_info [ 'build_platform' ] = os . sys . platform package_info [ 'build_user' ] = getpass . getuser ( ) package_id_file = open ( os . path . join ( temp_project_path , 'package_info.json' ) , 'w' ) dumped = json . dumps ( package_info , indent = 4 ) try : package_id_file . write ( dumped ) except TypeError : package_id_file . write ( unicode ( dumped ) ) package_id_file . close ( ) egg_links = [ ] temp_package_path = tempfile . mkdtemp ( prefix = 'zappa-packages' ) if os . sys . platform == 'win32' : site_packages = os . path . join ( venv , 'Lib' , 'site-packages' ) else : site_packages = os . path . join ( venv , 'lib' , get_venv_from_python_version ( ) , 'site-packages' ) egg_links . extend ( glob . glob ( os . path . join ( site_packages , '*.egg-link' ) ) ) if minify : excludes = ZIP_EXCLUDES + exclude copytree ( site_packages , temp_package_path , metadata = False , symlinks = False , ignore = shutil . ignore_patterns ( * excludes ) ) else : copytree ( site_packages , temp_package_path , metadata = False , symlinks = False ) site_packages_64 = os . path . join ( venv , 'lib64' , get_venv_from_python_version ( ) , 'site-packages' ) if os . path . exists ( site_packages_64 ) : egg_links . extend ( glob . glob ( os . path . join ( site_packages_64 , '*.egg-link' ) ) ) if minify : excludes = ZIP_EXCLUDES + exclude copytree ( site_packages_64 , temp_package_path , metadata = False , symlinks = False , ignore = shutil . ignore_patterns ( * excludes ) ) else : copytree ( site_packages_64 , temp_package_path , metadata = False , symlinks = False ) if egg_links : self . copy_editable_packages ( egg_links , temp_package_path ) copy_tree ( temp_package_path , temp_project_path , update = True ) if use_precompiled_packages : print ( "Downloading and installing dependencies.." ) installed_packages = self . get_installed_packages ( site_packages , site_packages_64 ) try : for installed_package_name , installed_package_version in installed_packages . items ( ) : if self . have_correct_lambda_package_version ( installed_package_name , installed_package_version ) : print ( " - %s==%s: Using precompiled lambda package " % ( installed_package_name , installed_package_version , ) ) self . extract_lambda_package ( installed_package_name , temp_project_path ) else : cached_wheel_path = self . get_cached_manylinux_wheel ( installed_package_name , installed_package_version , disable_progress ) if cached_wheel_path : shutil . rmtree ( os . path . join ( temp_project_path , installed_package_name ) , ignore_errors = True ) with zipfile . ZipFile ( cached_wheel_path ) as zfile : zfile . extractall ( temp_project_path ) elif self . have_any_lambda_package_version ( installed_package_name ) : lambda_version = lambda_packages [ installed_package_name ] [ self . runtime ] [ 'version' ] print ( " - %s==%s: Warning! Using precompiled lambda package version %s instead!" % ( installed_package_name , installed_package_version , lambda_version , ) ) self . extract_lambda_package ( installed_package_name , temp_project_path ) if self . runtime in ( "python3.6" , "python3.7" ) : print ( " - sqlite==python3: Using precompiled lambda package" ) self . extract_lambda_package ( 'sqlite3' , temp_project_path ) except Exception as e : print ( e ) if archive_format == 'zip' : print ( "Packaging project as zip." ) try : compression_method = zipfile . ZIP_DEFLATED except ImportError : compression_method = zipfile . ZIP_STORED archivef = zipfile . ZipFile ( archive_path , 'w' , compression_method ) elif archive_format == 'tarball' : print ( "Packaging project as gzipped tarball." ) archivef = tarfile . open ( archive_path , 'w|gz' ) for root , dirs , files in os . walk ( temp_project_path ) : for filename in files : if filename [ - 4 : ] == '.pyc' and root [ - 10 : ] == 'migrations' : continue if filename [ - 3 : ] == '.py' and root [ - 10 : ] != 'migrations' : abs_filname = os . path . join ( root , filename ) abs_pyc_filename = abs_filname + 'c' if os . path . isfile ( abs_pyc_filename ) : py_time = os . stat ( abs_filname ) . st_mtime pyc_time = os . stat ( abs_pyc_filename ) . st_mtime if pyc_time > py_time : continue os . chmod ( os . path . join ( root , filename ) , 0o755 ) if archive_format == 'zip' : zipi = zipfile . ZipInfo ( os . path . join ( root . replace ( temp_project_path , '' ) . lstrip ( os . sep ) , filename ) ) zipi . create_system = 3 zipi . external_attr = 0o755 << int ( 16 ) with open ( os . path . join ( root , filename ) , 'rb' ) as f : archivef . writestr ( zipi , f . read ( ) , compression_method ) elif archive_format == 'tarball' : tarinfo = tarfile . TarInfo ( os . path . join ( root . replace ( temp_project_path , '' ) . lstrip ( os . sep ) , filename ) ) tarinfo . mode = 0o755 stat = os . stat ( os . path . join ( root , filename ) ) tarinfo . mtime = stat . st_mtime tarinfo . size = stat . st_size with open ( os . path . join ( root , filename ) , 'rb' ) as f : archivef . addfile ( tarinfo , f ) if not contains_python_files_or_subdirs ( root ) : dirs [ : ] = [ d for d in dirs if d != root ] else : if '__init__.py' not in files and not conflicts_with_a_neighbouring_module ( root ) : tmp_init = os . path . join ( temp_project_path , '__init__.py' ) open ( tmp_init , 'a' ) . close ( ) os . chmod ( tmp_init , 0o755 ) arcname = os . path . join ( root . replace ( temp_project_path , '' ) , os . path . join ( root . replace ( temp_project_path , '' ) , '__init__.py' ) ) if archive_format == 'zip' : archivef . write ( tmp_init , arcname ) elif archive_format == 'tarball' : archivef . add ( tmp_init , arcname ) archivef . close ( ) shutil . rmtree ( temp_project_path ) shutil . rmtree ( temp_package_path ) if os . path . isdir ( venv ) and slim_handler : shutil . rmtree ( venv ) return archive_fname
Create a Lambda-ready zip file of the current virtualenvironment and working directory.

        Returns path to that file.