def download(self):
        """ Make sure data file is downloaded and stored properly """
        if not os.path.exists(self.data_path):
            # Create if it doesn't exist
            pathlib.Path(self.data_path).mkdir(parents=True, exist_ok=True)

        if not os.path.exists(self.text_path):
            http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where())

            with open(self.text_path, 'wt') as fp:
                request = http.request('GET', self.url)
                content = request.data.decode('utf8')
                fp.write(content)

        if not os.path.exists(self.processed_path):
            with open(self.text_path, 'rt') as fp:
                content = fp.read()

            alphabet = sorted(set(content))

            index_to_character = {idx: c for idx, c in enumerate(alphabet, 1)}
            character_to_index = {c: idx for idx, c in enumerate(alphabet, 1)}

            content_encoded = np.array([character_to_index[c] for c in content], dtype=np.uint8)

            data_dict = {
                'alphabet': alphabet,
                'index_to_character': index_to_character,
                'character_to_index': character_to_index,
                'content_encoded': content_encoded
            }

            with open(self.processed_path, 'wb') as fp:
                torch.save(data_dict, fp)
        else:
            with open(self.processed_path, 'rb') as fp:
                data_dict = torch.load(fp)

        return data_dict
def download ( self ) : if not os . path . exists ( self . data_path ) : pathlib . Path ( self . data_path ) . mkdir ( parents = True , exist_ok = True ) if not os . path . exists ( self . text_path ) : http = urllib3 . PoolManager ( cert_reqs = 'CERT_REQUIRED' , ca_certs = certifi . where ( ) ) with open ( self . text_path , 'wt' ) as fp : request = http . request ( 'GET' , self . url ) content = request . data . decode ( 'utf8' ) fp . write ( content ) if not os . path . exists ( self . processed_path ) : with open ( self . text_path , 'rt' ) as fp : content = fp . read ( ) alphabet = sorted ( set ( content ) ) index_to_character = { idx : c for idx , c in enumerate ( alphabet , 1 ) } character_to_index = { c : idx for idx , c in enumerate ( alphabet , 1 ) } content_encoded = np . array ( [ character_to_index [ c ] for c in content ] , dtype = np . uint8 ) data_dict = { 'alphabet' : alphabet , 'index_to_character' : index_to_character , 'character_to_index' : character_to_index , 'content_encoded' : content_encoded } with open ( self . processed_path , 'wb' ) as fp : torch . save ( data_dict , fp ) else : with open ( self . processed_path , 'rb' ) as fp : data_dict = torch . load ( fp ) return data_dict
Make sure data file is downloaded and stored properly