def __repr__(self):
        typestr = (' type=%s,' % self.type
                   if self.type != self.tok else '')

        propvals = ', '.join(
            '%s=%s' % (p, getattr(self, p))
            for p in self._properties
            if getattr(self, p)
        )

        return '%s(%s,%s %s)' % (self.__class__.__name__,
                                 self.tok,
                                 typestr,
                                 propvals)
A string representation of the token that can reproduce it
dataset/ETHPy150Open reefeed/wanish/wanish/tokenizers.py/PunktToken.__repr__