def put_tarfile(
        self,
        hdfs_path,
        local_path,
        compression='gzip',
        verbose=None,
        overwrite=False,
    ):
        """
        Write contents of tar archive to HDFS directly without having to
        decompress it locally first

        Parameters
        ----------
        hdfs_path : string
        local_path : string
        compression : {'gzip', 'bz2', None}
        overwrite : boolean, default False
        verbose : boolean, default None (global default)
        """
        import tarfile

        modes = {None: 'r', 'gzip': 'r:gz', 'bz2': 'r:bz2'}

        if compression not in modes:
            raise ValueError(
                'Invalid compression type {0}'.format(compression)
            )
        mode = modes[compression]

        tf = tarfile.open(local_path, mode=mode)
        for info in tf:
            if not info.isfile():
                continue

            buf = tf.extractfile(info)
            abspath = posixpath.join(hdfs_path, info.path)
            self.put(abspath, buf, verbose=verbose, overwrite=overwrite)
def put_tarfile ( self , hdfs_path , local_path , compression = 'gzip' , verbose = None , overwrite = False , ) : import tarfile modes = { None : 'r' , 'gzip' : 'r:gz' , 'bz2' : 'r:bz2' } if compression not in modes : raise ValueError ( 'Invalid compression type {0}' . format ( compression ) ) mode = modes [ compression ] tf = tarfile . open ( local_path , mode = mode ) for info in tf : if not info . isfile ( ) : continue buf = tf . extractfile ( info ) abspath = posixpath . join ( hdfs_path , info . path ) self . put ( abspath , buf , verbose = verbose , overwrite = overwrite )
Write contents of tar archive to HDFS directly without having to
        decompress it locally first

        Parameters
        ----------
        hdfs_path : string
        local_path : string
        compression : {'gzip', 'bz2', None}
        overwrite : boolean, default False
        verbose : boolean, default None (global default)