def MOD(self, a, b):
        """Modulo remainder operation"""
        try:
            result = Operators.ITEBV(256, b == 0, 0, a % b)
        except ZeroDivisionError:
            result = 0
        return result

def MOD ( self , a , b ) : try : result = Operators . ITEBV ( 256 , b == 0 , 0 , a % b ) except ZeroDivisionError : result = 0 return result


def SMOD(self, a, b):
        """Signed modulo remainder operation"""
        s0, s1 = to_signed(a), to_signed(b)
        sign = Operators.ITEBV(256, s0 < 0, -1, 1)
        try:
            result = (Operators.ABS(s0) % Operators.ABS(s1)) * sign
        except ZeroDivisionError:
            result = 0

        return Operators.ITEBV(256, s1 == 0, 0, result)

def SMOD ( self , a , b ) : s0 , s1 = to_signed ( a ) , to_signed ( b ) sign = Operators . ITEBV ( 256 , s0 < 0 , - 1 , 1 ) try : result = ( Operators . ABS ( s0 ) % Operators . ABS ( s1 ) ) * sign except ZeroDivisionError : result = 0 return Operators . ITEBV ( 256 , s1 == 0 , 0 , result )


def ADDMOD(self, a, b, c):
        """Modulo addition operation"""
        try:
            result = Operators.ITEBV(256, c == 0, 0, (a + b) % c)
        except ZeroDivisionError:
            result = 0
        return result

def ADDMOD ( self , a , b , c ) : try : result = Operators . ITEBV ( 256 , c == 0 , 0 , ( a + b ) % c ) except ZeroDivisionError : result = 0 return result


def EXP_gas(self, base, exponent):
        """Calculate extra gas fee"""
        EXP_SUPPLEMENTAL_GAS = 10   # cost of EXP exponent per byte

        def nbytes(e):
            result = 0
            for i in range(32):
                result = Operators.ITEBV(512, Operators.EXTRACT(e, i * 8, 8) != 0, i + 1, result)
            return result
        return EXP_SUPPLEMENTAL_GAS * nbytes(exponent)

def
EXP_gas
(
self
,
base
,
exponent
)
:
EXP_SUPPLEMENTAL_GAS
=
10
# cost of EXP exponent per byte
def
nbytes
(
e
)
:
result
=
0
for
i
in
range
(
32
)
:
result
=
Operators
.
ITEBV
(
512
,
Operators
.
EXTRACT
(
e
,
i
*
8
,
8
)
!=
0
,
i
+
1
,
result
)
return
result
return
EXP_SUPPLEMENTAL_GAS
*
nbytes
(
exponent
)


def SIGNEXTEND(self, size, value):
        """Extend length of two's complement signed integer"""
        # FIXME maybe use Operators.SEXTEND
        testbit = Operators.ITEBV(256, size <= 31, size * 8 + 7, 257)
        result1 = (value | (TT256 - (1 << testbit)))
        result2 = (value & ((1 << testbit) - 1))
        result = Operators.ITEBV(256, (value & (1 << testbit)) != 0, result1, result2)
        return Operators.ITEBV(256, size <= 31, result, value)

def
SIGNEXTEND
(
self
,
size
,
value
)
:
# FIXME maybe use Operators.SEXTEND
testbit
=
Operators
.
ITEBV
(
256
,
size
<=
31
,
size
*
8
+
7
,
257
)
result1
=
(
value
|
(
TT256
(
1
<<
testbit
)
)
)
result2
=
(
value
&
(
(
1
<<
testbit
)
1
)
)
result
=
Operators
.
ITEBV
(
256
,
(
value
&
(
1
<<
testbit
)
)
!=
0
,
result1
,
result2
)
return
Operators
.
ITEBV
(
256
,
size
<=
31
,
result
,
value
)

Extend length of two s complement signed integer


def LT(self, a, b):
        """Less-than comparison"""
        return Operators.ITEBV(256, Operators.ULT(a, b), 1, 0)

def LT ( self , a , b ) : return Operators . ITEBV ( 256 , Operators . ULT ( a , b ) , 1 , 0 )


def GT(self, a, b):
        """Greater-than comparison"""
        return Operators.ITEBV(256, Operators.UGT(a, b), 1, 0)

def GT ( self , a , b ) : return Operators . ITEBV ( 256 , Operators . UGT ( a , b ) , 1 , 0 )


def SGT(self, a, b):
        """Signed greater-than comparison"""
        # http://gavwood.com/paper.pdf
        s0, s1 = to_signed(a), to_signed(b)
        return Operators.ITEBV(256, s0 > s1, 1, 0)

def
SGT
(
self
,
a
,
b
)
:
# http://gavwood.com/paper.pdf
s0
,
s1
=
to_signed
(
a
)
,
to_signed
(
b
)
return
Operators
.
ITEBV
(
256
,
s0
>
s1
,
1
,
0
)


def BYTE(self, offset, value):
        """Retrieve single byte from word"""
        offset = Operators.ITEBV(256, offset < 32, (31 - offset) * 8, 256)
        return Operators.ZEXTEND(Operators.EXTRACT(value, offset, 8), 256)

def BYTE ( self , offset , value ) : offset = Operators . ITEBV ( 256 , offset < 32 , ( 31 - offset ) * 8 , 256 ) return Operators . ZEXTEND ( Operators . EXTRACT ( value , offset , 8 ) , 256 )


def SHA3(self, start, size):
        """Compute Keccak-256 hash"""
        # read memory from start to end
        # http://gavwood.com/paper.pdf
        data = self.try_simplify_to_constant(self.read_buffer(start, size))

        if issymbolic(data):
            known_sha3 = {}
            # Broadcast the signal
            self._publish('on_symbolic_sha3', data, known_sha3)  # This updates the local copy of sha3 with the pairs we need to explore

            value = 0  # never used
            known_hashes_cond = False
            for key, hsh in known_sha3.items():
                assert not issymbolic(key), "Saved sha3 data,hash pairs should be concrete"
                cond = key == data
                known_hashes_cond = Operators.OR(cond, known_hashes_cond)
                value = Operators.ITEBV(256, cond, hsh, value)
            return value

        value = sha3.keccak_256(data).hexdigest()
        value = int(value, 16)
        self._publish('on_concrete_sha3', data, value)
        logger.info("Found a concrete SHA3 example %r -> %x", data, value)
        return value


def CALLDATALOAD(self, offset):
        """Get input data of current environment"""

        if issymbolic(offset):
            if solver.can_be_true(self._constraints, offset == self._used_calldata_size):
                self.constraints.add(offset == self._used_calldata_size)
            raise ConcretizeArgument(1, policy='SAMPLED')

        self._use_calldata(offset, 32)

        data_length = len(self.data)

        bytes = []
        for i in range(32):
            try:
                c = Operators.ITEBV(8, offset + i < data_length, self.data[offset + i], 0)
            except IndexError:
                # offset + i is concrete and outside data
                c = 0

            bytes.append(c)
        return Operators.CONCAT(256, *bytes)


def CALLDATACOPY(self, mem_offset, data_offset, size):
        """Copy input data in current environment to memory"""

        if issymbolic(size):
            if solver.can_be_true(self._constraints, size <= len(self.data) + 32):
                self.constraints.add(size <= len(self.data) + 32)
            raise ConcretizeArgument(3, policy='SAMPLED')

        if issymbolic(data_offset):
            if solver.can_be_true(self._constraints, data_offset == self._used_calldata_size):
                self.constraints.add(data_offset == self._used_calldata_size)
            raise ConcretizeArgument(2, policy='SAMPLED')

        #account for calldata usage
        self._use_calldata(data_offset, size)
        self._allocate(mem_offset, size)
        for i in range(size):
            try:
                c = Operators.ITEBV(8, data_offset + i < len(self.data), Operators.ORD(self.data[data_offset + i]), 0)
            except IndexError:
                # data_offset + i is concrete and outside data
                c = 0
            self._store(mem_offset + i, c)


def CODECOPY(self, mem_offset, code_offset, size):
        """Copy code running in current environment to memory"""

        self._allocate(mem_offset, size)
        GCOPY = 3             # cost to copy one 32 byte word
        copyfee = self.safe_mul(GCOPY, Operators.UDIV(self.safe_add(size, 31), 32))
        self._consume(copyfee)

        if issymbolic(size):
            max_size = solver.max(self.constraints, size)
        else:
            max_size = size

        for i in range(max_size):
            if issymbolic(i < size):
                default = Operators.ITEBV(8, i < size, 0, self._load(mem_offset + i, 1))  # Fixme. unnecessary memory read
            else:
                if i < size:
                    default = 0
                else:
                    default = self._load(mem_offset + i, 1)

            if issymbolic(code_offset):
                value = Operators.ITEBV(8, code_offset + i >= len(self.bytecode), default, self.bytecode[code_offset + i])
            else:
                if code_offset + i >= len(self.bytecode):
                    value = default
                else:
                    value = self.bytecode[code_offset + i]
            self._store(mem_offset + i, value)
        self._publish('did_evm_read_code', code_offset, size)

def
CODECOPY
(
self
,
mem_offset
,
code_offset
,
size
)
:
self
.
_allocate
(
mem_offset
,
size
)
GCOPY
=
3
# cost to copy one 32 byte word
copyfee
=
self
.
safe_mul
(
GCOPY
,
Operators
.
UDIV
(
self
.
safe_add
(
size
,
31
)
,
32
)
)
self
.
_consume
(
copyfee
)
if
issymbolic
(
size
)
:
max_size
=
solver
.
max
(
self
.
constraints
,
size
)
else
:
max_size
=
size
for
i
in
range
(
max_size
)
:
if
issymbolic
(
i
<
size
)
:
default
=
Operators
.
ITEBV
(
8
,
i
<
size
,
0
,
self
.
_load
(
mem_offset
+
i
,
1
)
)
# Fixme. unnecessary memory read
else
:
if
i
<
size
:
default
=
0
else
:
default
=
self
.
_load
(
mem_offset
+
i
,
1
)
if
issymbolic
(
code_offset
)
:
value
=
Operators
.
ITEBV
(
8
,
code_offset
+
i
>=
len
(
self
.
bytecode
)
,
default
,
self
.
bytecode
[
code_offset
+
i
]
)
else
:
if
code_offset
+
i
>=
len
(
self
.
bytecode
)
:
value
=
default
else
:
value
=
self
.
bytecode
[
code_offset
+
i
]
self
.
_store
(
mem_offset
+
i
,
value
)
self
.
_publish
(
'did_evm_read_code'
,
code_offset
,
size
)


def EXTCODECOPY(self, account, address, offset, size):
        """Copy an account's code to memory"""
        extbytecode = self.world.get_code(account)
        self._allocate(address + size)

        for i in range(size):
            if offset + i < len(extbytecode):
                self._store(address + i, extbytecode[offset + i])
            else:
                self._store(address + i, 0)

def EXTCODECOPY ( self , account , address , offset , size ) : extbytecode = self . world . get_code ( account ) self . _allocate ( address + size ) for i in range ( size ) : if offset + i < len ( extbytecode ) : self . _store ( address + i , extbytecode [ offset + i ] ) else : self . _store ( address + i , 0 )


def MLOAD(self, address):
        """Load word from memory"""
        self._allocate(address, 32)
        value = self._load(address, 32)
        return value

def MLOAD ( self , address ) : self . _allocate ( address , 32 ) value = self . _load ( address , 32 ) return value


def MSTORE(self, address, value):
        """Save word to memory"""
        if istainted(self.pc):
            for taint in get_taints(self.pc):
                value = taint_with(value, taint)
        self._allocate(address, 32)
        self._store(address, value, 32)

def MSTORE ( self , address , value ) : if istainted ( self . pc ) : for taint in get_taints ( self . pc ) : value = taint_with ( value , taint ) self . _allocate ( address , 32 ) self . _store ( address , value , 32 )


def MSTORE8(self, address, value):
        """Save byte to memory"""
        if istainted(self.pc):
            for taint in get_taints(self.pc):
                value = taint_with(value, taint)
        self._allocate(address, 1)
        self._store(address, Operators.EXTRACT(value, 0, 8), 1)

def MSTORE8 ( self , address , value ) : if istainted ( self . pc ) : for taint in get_taints ( self . pc ) : value = taint_with ( value , taint ) self . _allocate ( address , 1 ) self . _store ( address , Operators . EXTRACT ( value , 0 , 8 ) , 1 )


def SLOAD(self, offset):
        """Load word from storage"""
        storage_address = self.address
        self._publish('will_evm_read_storage', storage_address, offset)
        value = self.world.get_storage_data(storage_address, offset)
        self._publish('did_evm_read_storage', storage_address, offset, value)
        return value

def SLOAD ( self , offset ) : storage_address = self . address self . _publish ( 'will_evm_read_storage' , storage_address , offset ) value = self . world . get_storage_data ( storage_address , offset ) self . _publish ( 'did_evm_read_storage' , storage_address , offset , value ) return value


def SSTORE(self, offset, value):
        """Save word to storage"""
        storage_address = self.address
        self._publish('will_evm_write_storage', storage_address, offset, value)
        #refund = Operators.ITEBV(256,
        #                         previous_value != 0,
        #                         Operators.ITEBV(256, value != 0, 0, GSTORAGEREFUND),
        #                         0)

        if istainted(self.pc):
            for taint in get_taints(self.pc):
                value = taint_with(value, taint)
        self.world.set_storage_data(storage_address, offset, value)
        self._publish('did_evm_write_storage', storage_address, offset, value)

#                         previous_value != 0,


def JUMPI(self, dest, cond):
        """Conditionally alter the program counter"""
        self.pc = Operators.ITEBV(256, cond != 0, dest, self.pc + self.instruction.size)
        #This set ups a check for JMPDEST in the next instruction if cond != 0
        self._set_check_jmpdest(cond != 0)


def SWAP(self, *operands):
        """Exchange 1st and 2nd stack items"""
        a = operands[0]
        b = operands[-1]
        return (b,) + operands[1:-1] + (a,)

def SWAP ( self , * operands ) : a = operands [ 0 ] b = operands [ - 1 ] return ( b , ) + operands [ 1 : - 1 ] + ( a , )


def CALLCODE(self, gas, _ignored_, value, in_offset, in_size, out_offset, out_size):
        """Message-call into this account with alternative account's code"""
        self.world.start_transaction('CALLCODE',
                                     address=self.address,
                                     data=self.read_buffer(in_offset, in_size),
                                     caller=self.address,
                                     value=value,
                                     gas=gas)
        raise StartTx()

def CALLCODE ( self , gas , _ignored_ , value , in_offset , in_size , out_offset , out_size ) : self . world . start_transaction ( 'CALLCODE' , address = self . address , data = self . read_buffer ( in_offset , in_size ) , caller = self . address , value = value , gas = gas ) raise StartTx ( )

Message - call into this account with alternative account s code


def RETURN(self, offset, size):
        """Halt execution returning output data"""
        data = self.read_buffer(offset, size)
        raise EndTx('RETURN', data)

def RETURN ( self , offset , size ) : data = self . read_buffer ( offset , size ) raise EndTx ( 'RETURN' , data )


def SELFDESTRUCT(self, recipient):
        """Halt execution and register account for later deletion"""
        #This may create a user account
        recipient = Operators.EXTRACT(recipient, 0, 160)
        address = self.address
        #FIXME for on the known addresses
        if issymbolic(recipient):
            logger.info("Symbolic recipient on self destruct")
            recipient = solver.get_value(self.constraints, recipient)

        if recipient not in self.world:
            self.world.create_account(address=recipient)

        self.world.send_funds(address, recipient, self.world.get_balance(address))
        self.world.delete_account(address)

        raise EndTx('SELFDESTRUCT')

def
SELFDESTRUCT
(
self
,
recipient
)
:
#This may create a user account
recipient
=
Operators
.
EXTRACT
(
recipient
,
0
,
160
)
address
=
self
.
address
#FIXME for on the known addresses
if
issymbolic
(
recipient
)
:
logger
.
info
(
"Symbolic recipient on self destruct"
)
recipient
=
solver
.
get_value
(
self
.
constraints
,
recipient
)
if
recipient
not
in
self
.
world
:
self
.
world
.
create_account
(
address
=
recipient
)
self
.
world
.
send_funds
(
address
,
recipient
,
self
.
world
.
get_balance
(
address
)
)
self
.
world
.
delete_account
(
address
)
raise
EndTx
(
'SELFDESTRUCT'
)


def human_transactions(self):
        """Completed human transaction"""
        txs = []
        for tx in self.transactions:
            if tx.depth == 0:
                txs.append(tx)
        return tuple(txs)

def human_transactions ( self ) : txs = [ ] for tx in self . transactions : if tx . depth == 0 : txs . append ( tx ) return tuple ( txs )


def current_human_transaction(self):
        """Current ongoing human transaction"""
        try:
            tx, _, _, _, _ = self._callstack[0]
            if tx.result is not None:
                #That tx finished. No current tx.
                return None
            assert tx.depth == 0
            return tx
        except IndexError:
            return None

def
current_human_transaction
(
self
)
:
try
:
tx
,
_
,
_
,
_
,
_
=
self
.
_callstack
[
0
]
if
tx
.
result
is
not
:
#That tx finished. No current tx.
return
assert
tx
.
depth
==
0
return
tx
except
IndexError
:
return


def get_storage_data(self, storage_address, offset):
        """
        Read a value from a storage slot on the specified account

        :param storage_address: an account address
        :param offset: the storage slot to use.
        :type offset: int or BitVec
        :return: the value
        :rtype: int or BitVec
        """
        value = self._world_state[storage_address]['storage'].get(offset, 0)
        return simplify(value)

def get_storage_data ( self , storage_address , offset ) : value = self . _world_state [ storage_address ] [ 'storage' ] . get ( offset , 0 ) return simplify ( value )


def set_storage_data(self, storage_address, offset, value):
        """
        Writes a value to a storage slot in specified account

        :param storage_address: an account address
        :param offset: the storage slot to use.
        :type offset: int or BitVec
        :param value: the value to write
        :type value: int or BitVec
        """
        self._world_state[storage_address]['storage'][offset] = value

def set_storage_data ( self , storage_address , offset , value ) : self . _world_state [ storage_address ] [ 'storage' ] [ offset ] = value


def get_storage_items(self, address):
        """
        Gets all items in an account storage

        :param address: account address
        :return: all items in account storage. items are tuple of (index, value). value can be symbolic
        :rtype: list[(storage_index, storage_value)]
        """
        storage = self._world_state[address]['storage']
        items = []
        array = storage.array
        while not isinstance(array, ArrayVariable):
            items.append((array.index, array.value))
            array = array.array
        return items

def get_storage_items ( self , address ) : storage = self . _world_state [ address ] [ 'storage' ] items = [ ] array = storage . array while not isinstance ( array , ArrayVariable ) : items . append ( ( array . index , array . value ) ) array = array . array return items


def has_storage(self, address):
        """
        True if something has been written to the storage.
        Note that if a slot has been erased from the storage this function may
        lose any meaning.
        """
        storage = self._world_state[address]['storage']
        array = storage.array
        while not isinstance(array, ArrayVariable):
            if isinstance(array, ArrayStore):
                return True
            array = array.array
        return False

def has_storage ( self , address ) : storage = self . _world_state [ address ] [ 'storage' ] array = storage . array while not isinstance ( array , ArrayVariable ) : if isinstance ( array , ArrayStore ) : return True array = array . array return False

True if something has been written to the storage . Note that if a slot has been erased from the storage this function may lose any meaning .


def new_address(self, sender=None, nonce=None):
        """Create a fresh 160bit address"""
        if sender is not None and nonce is None:
            nonce = self.get_nonce(sender)

        new_address = self.calculate_new_address(sender, nonce)
        if sender is None and new_address in self:
            return self.new_address(sender, nonce)
        return new_address

def new_address ( self , sender = None , nonce = None ) : if sender is not None and nonce is None : nonce = self . get_nonce ( sender ) new_address = self . calculate_new_address ( sender , nonce ) if sender is None and new_address in self : return self . new_address ( sender , nonce ) return new_address


def create_contract(self, price=0, address=None, caller=None, balance=0, init=None, gas=None):
        """
        Create a contract account. Sends a transaction to initialize the contract

        :param address: the address of the new account, if known. If omitted, a new address will be generated as closely to the Yellow Paper as possible.
        :param balance: the initial balance of the account in Wei
        :param init: the initialization code of the contract

        The way that the Solidity compiler expects the constructor arguments to
        be passed is by appending the arguments to the byte code produced by the
        Solidity compiler. The arguments are formatted as defined in the Ethereum
        ABI2. The arguments are then copied from the init byte array to the EVM
        memory through the CODECOPY opcode with appropriate values on the stack.
        This is done when the byte code in the init byte array is actually run
        on the network.
        """
        expected_address = self.create_account(self.new_address(sender=caller))
        if address is None:
            address = expected_address
        elif caller is not None and address != expected_address:
            raise EthereumError(f"Error: contract created from address {hex(caller)} with nonce {self.get_nonce(caller)} was expected to be at address {hex(expected_address)}, but create_contract was called with address={hex(address)}")
        self.start_transaction('CREATE', address, price, init, caller, balance, gas=gas)
        self._process_pending_transaction()
        return address

Create a contract account . Sends a transaction to initialize the contract


def _swap_mode(self):
        """Toggle between ARM and Thumb mode"""
        assert self.mode in (cs.CS_MODE_ARM, cs.CS_MODE_THUMB)
        if self.mode == cs.CS_MODE_ARM:
            self.mode = cs.CS_MODE_THUMB
        else:
            self.mode = cs.CS_MODE_ARM

def _swap_mode ( self ) : assert self . mode in ( cs . CS_MODE_ARM , cs . CS_MODE_THUMB ) if self . mode == cs . CS_MODE_ARM : self . mode = cs . CS_MODE_THUMB else : self . mode = cs . CS_MODE_ARM


def MRC(cpu, coprocessor, opcode1, dest, coprocessor_reg_n, coprocessor_reg_m, opcode2):
        """
        MRC moves to ARM register from coprocessor.

        :param Armv7Operand coprocessor: The name of the coprocessor; immediate
        :param Armv7Operand opcode1: coprocessor specific opcode; 3-bit immediate
        :param Armv7Operand dest: the destination operand: register
        :param Armv7Operand coprocessor_reg_n: the coprocessor register; immediate
        :param Armv7Operand coprocessor_reg_m: the coprocessor register; immediate
        :param Armv7Operand opcode2: coprocessor specific opcode; 3-bit immediate
        """
        assert coprocessor.type == 'coprocessor'
        assert opcode1.type == 'immediate'
        assert opcode2.type == 'immediate'
        assert dest.type == 'register'
        imm_coprocessor = coprocessor.read()
        imm_opcode1 = opcode1.read()
        imm_opcode2 = opcode2.read()
        coprocessor_n_name = coprocessor_reg_n.read()
        coprocessor_m_name = coprocessor_reg_m.read()

        if 15 == imm_coprocessor:  # MMU
            if 0 == imm_opcode1:
                if 13 == coprocessor_n_name:
                    if 3 == imm_opcode2:
                        dest.write(cpu.regfile.read('P15_C13'))
                        return
        raise NotImplementedError("MRC: unimplemented combination of coprocessor, opcode, and coprocessor register")

MRC moves to ARM register from coprocessor .


def LDRD(cpu, dest1, dest2, src, offset=None):
        """Loads double width data from memory."""
        assert dest1.type == 'register'
        assert dest2.type == 'register'
        assert src.type == 'memory'
        mem1 = cpu.read_int(src.address(), 32)
        mem2 = cpu.read_int(src.address() + 4, 32)
        writeback = cpu._compute_writeback(src, offset)
        dest1.write(mem1)
        dest2.write(mem2)
        cpu._cs_hack_ldr_str_writeback(src, offset, writeback)

def LDRD ( cpu , dest1 , dest2 , src , offset = None ) : assert dest1 . type == 'register' assert dest2 . type == 'register' assert src . type == 'memory' mem1 = cpu . read_int ( src . address ( ) , 32 ) mem2 = cpu . read_int ( src . address ( ) + 4 , 32 ) writeback = cpu . _compute_writeback ( src , offset ) dest1 . write ( mem1 ) dest2 . write ( mem2 ) cpu . _cs_hack_ldr_str_writeback ( src , offset , writeback )


def STRD(cpu, src1, src2, dest, offset=None):
        """Writes the contents of two registers to memory."""
        assert src1.type == 'register'
        assert src2.type == 'register'
        assert dest.type == 'memory'
        val1 = src1.read()
        val2 = src2.read()
        writeback = cpu._compute_writeback(dest, offset)
        cpu.write_int(dest.address(), val1, 32)
        cpu.write_int(dest.address() + 4, val2, 32)
        cpu._cs_hack_ldr_str_writeback(dest, offset, writeback)

def STRD ( cpu , src1 , src2 , dest , offset = None ) : assert src1 . type == 'register' assert src2 . type == 'register' assert dest . type == 'memory' val1 = src1 . read ( ) val2 = src2 . read ( ) writeback = cpu . _compute_writeback ( dest , offset ) cpu . write_int ( dest . address ( ) , val1 , 32 ) cpu . write_int ( dest . address ( ) + 4 , val2 , 32 ) cpu . _cs_hack_ldr_str_writeback ( dest , offset , writeback )

Writes the contents of two registers to memory .


def ADR(cpu, dest, src):
        """
        Address to Register adds an immediate value to the PC value, and writes the result to the destination register.

        :param ARMv7Operand dest: Specifies the destination register.
        :param ARMv7Operand src:
            Specifies the label of an instruction or literal data item whose address is to be loaded into
            <Rd>. The assembler calculates the required value of the offset from the Align(PC,4)
            value of the ADR instruction to this label.
        """
        aligned_pc = (cpu.instruction.address + 4) & 0xfffffffc
        dest.write(aligned_pc + src.read())

def ADR ( cpu , dest , src ) : aligned_pc = ( cpu . instruction . address + 4 ) & 0xfffffffc dest . write ( aligned_pc + src . read ( ) )

Address to Register adds an immediate value to the PC value and writes the result to the destination register .


def ADDW(cpu, dest, src, add):
        """
        This instruction adds an immediate value to a register value, and writes the result to the destination register.
        It doesn't update the condition flags.

        :param ARMv7Operand dest: Specifies the destination register. If omitted, this register is the same as src.
        :param ARMv7Operand src:
            Specifies the register that contains the first operand. If the SP is specified for dest, see ADD (SP plus
            immediate). If the PC is specified for dest, see ADR.
        :param ARMv7Operand add:
            Specifies the immediate value to be added to the value obtained from src. The range of allowed values is
            0-4095.
        """
        aligned_pc = (cpu.instruction.address + 4) & 0xfffffffc
        if src.type == 'register' and src.reg in ('PC', 'R15'):
            src = aligned_pc
        else:
            src = src.read()
        dest.write(src + add.read())

def ADDW ( cpu , dest , src , add ) : aligned_pc = ( cpu . instruction . address + 4 ) & 0xfffffffc if src . type == 'register' and src . reg in ( 'PC' , 'R15' ) : src = aligned_pc else : src = src . read ( ) dest . write ( src + add . read ( ) )

This instruction adds an immediate value to a register value and writes the result to the destination register . It doesn t update the condition flags .


def CBZ(cpu, op, dest):
        """
        Compare and Branch on Zero compares the value in a register with zero, and conditionally branches forward
        a constant value. It does not affect the condition flags.

        :param ARMv7Operand op: Specifies the register that contains the first operand.
        :param ARMv7Operand dest:
            Specifies the label of the instruction that is to be branched to. The assembler calculates the
            required value of the offset from the PC value of the CBZ instruction to this label, then
            selects an encoding that will set imm32 to that offset. Allowed offsets are even numbers in
            the range 0 to 126.
        """
        cpu.PC = Operators.ITEBV(cpu.address_bit_size,
                                 op.read(), cpu.PC, dest.read())

def CBZ ( cpu , op , dest ) : cpu . PC = Operators . ITEBV ( cpu . address_bit_size , op . read ( ) , cpu . PC , dest . read ( ) )

Compare and Branch on Zero compares the value in a register with zero and conditionally branches forward a constant value . It does not affect the condition flags .


def TBH(cpu, dest):
        """
        Table Branch Halfword causes a PC-relative forward branch using a table of single halfword offsets. A base
        register provides a pointer to the table, and a second register supplies an index into the table. The branch
        length is twice the value of the halfword returned from the table.

        :param ARMv7Operand dest: see below; register
        """
        # Capstone merges the two registers values into one operand, so we need to extract them back

        # Specifies the base register. This contains the address of the table of branch lengths. This
        # register is allowed to be the PC. If it is, the table immediately follows this instruction.
        base_addr = dest.get_mem_base_addr()
        if dest.mem.base in ('PC', 'R15'):
            base_addr = cpu.PC

        # Specifies the index register. This contains an integer pointing to a halfword within the table.
        # The offset within the table is twice the value of the index.
        offset = cpu.read_int(base_addr + dest.get_mem_offset(), 16)
        offset = Operators.ZEXTEND(offset, cpu.address_bit_size)

        cpu.PC += (offset << 1)

Table Branch Halfword causes a PC - relative forward branch using a table of single halfword offsets . A base register provides a pointer to the table and a second register supplies an index into the table . The branch length is twice the value of the halfword returned from the table .


def _dict_diff(d1, d2):
    """
    Produce a dict that includes all the keys in d2 that represent different values in d1, as well as values that
    aren't in d1.

    :param dict d1: First dict
    :param dict d2: Dict to compare with
    :rtype: dict
    """
    d = {}
    for key in set(d1).intersection(set(d2)):
        if d2[key] != d1[key]:
            d[key] = d2[key]
    for key in set(d2).difference(set(d1)):
        d[key] = d2[key]
    return d

def _dict_diff ( d1 , d2 ) : d = { } for key in set ( d1 ) . intersection ( set ( d2 ) ) : if d2 [ key ] != d1 [ key ] : d [ key ] = d2 [ key ] for key in set ( d2 ) . difference ( set ( d1 ) ) : d [ key ] = d2 [ key ] return d

Produce a dict that includes all the keys in d2 that represent different values in d1 as well as values that aren t in d1 .


def disassemble_instruction(self, code, pc):
        """Get next instruction using the Capstone disassembler

        :param str code: binary blob to be disassembled
        :param long pc: program counter
        """
        return next(self.disasm.disasm(code, pc))

def disassemble_instruction ( self , code , pc ) : return next ( self . disasm . disasm ( code , pc ) )


def add(self, constraint, check=False):
        """
        Add a constraint to the set

        :param constraint: The constraint to add to the set.
        :param check: Currently unused.
        :return:
        """
        if isinstance(constraint, bool):
            constraint = BoolConstant(constraint)
        assert isinstance(constraint, Bool)
        constraint = simplify(constraint)
        # If self._child is not None this constraint set has been forked and a
        # a derived constraintset may be using this. So we can't add any more
        # constraints to this one. After the child constraintSet is deleted
        # we regain the ability to add constraints.
        if self._child is not None:
            raise Exception('ConstraintSet is frozen')

        if isinstance(constraint, BoolConstant):
            if not constraint.value:
                logger.info("Adding an impossible constant constraint")
                self._constraints = [constraint]
            else:
                return

        self._constraints.append(constraint)

        if check:
            from ...core.smtlib import solver
            if not solver.check(self):
                raise ValueError("Added an impossible constraint")


def _declare(self, var):
        """ Declare the variable `var` """
        if var.name in self._declarations:
            raise ValueError('Variable already declared')
        self._declarations[var.name] = var
        return var

def
_declare
(
self
,
var
)
:
if
var
.
name
in
self
.
_declarations
:
raise
ValueError
(
'Variable already declared'
)
self
.
_declarations
[
var
.
name
]
=
var
return
var


def declarations(self):
        """ Returns the variable expressions of this constraint set """
        declarations = GetDeclarations()
        for a in self.constraints:
            try:
                declarations.visit(a)
            except RuntimeError:
                # TODO: (defunct) move recursion management out of PickleSerializer
                if sys.getrecursionlimit() >= PickleSerializer.MAX_RECURSION:
                    raise Exception(f'declarations recursion limit surpassed {PickleSerializer.MAX_RECURSION}, aborting')
                new_limit = sys.getrecursionlimit() + PickleSerializer.DEFAULT_RECURSION
                if new_limit <= PickleSerializer.DEFAULT_RECURSION:
                    sys.setrecursionlimit(new_limit)
                    return self.declarations
        return declarations.result


def is_declared(self, expression_var):
        """ True if expression_var is declared in this constraint set """
        if not isinstance(expression_var, Variable):
            raise ValueError(f'Expression must be a Variable (not a {type(expression_var)})')
        return any(expression_var is x for x in self.get_declared_variables())


def migrate(self, expression, name_migration_map=None):
        """ Migrate an expression created for a different constraint set to self.
            Returns an expression that can be used with this constraintSet

            All the foreign variables used in the expression are replaced by
            variables of this constraint set. If the variable was replaced before
            the replacement is taken from the provided migration map.

            The migration mapping is updated with new replacements.

            :param expression: the potentially foreign expression
            :param name_migration_map: mapping of already migrated variables. maps from string name of foreign variable to its currently existing migrated string name. this is updated during this migration.
            :return: a migrated expression where all the variables are local. name_migration_map is updated

        """
        if name_migration_map is None:
            name_migration_map = {}

        #  name_migration_map -> object_migration_map
        #  Based on the name mapping in name_migration_map build an object to
        #  object mapping to be used in the replacing of variables
        #  inv: object_migration_map's keys should ALWAYS be external/foreign
        #  expressions, and its values should ALWAYS be internal/local expressions
        object_migration_map = {}

        #List of foreign vars used in expression
        foreign_vars = itertools.filterfalse(self.is_declared, get_variables(expression))
        for foreign_var in foreign_vars:
            # If a variable with the same name was previously migrated
            if foreign_var.name in name_migration_map:
                migrated_name = name_migration_map[foreign_var.name]
                native_var = self.get_variable(migrated_name)
                assert native_var is not None, "name_migration_map contains a variable that does not exist in this ConstraintSet"
                object_migration_map[foreign_var] = native_var
            else:
                # foreign_var was not found in the local declared variables nor
                # any variable with the same name was previously migrated
                # let's make a new unique internal name for it
                migrated_name = foreign_var.name
                if migrated_name in self._declarations:
                    migrated_name = self._make_unique_name(f'{foreign_var.name}_migrated')
                # Create and declare a new variable of given type
                if isinstance(foreign_var, Bool):
                    new_var = self.new_bool(name=migrated_name)
                elif isinstance(foreign_var, BitVec):
                    new_var = self.new_bitvec(foreign_var.size, name=migrated_name)
                elif isinstance(foreign_var, Array):
                    # Note that we are discarding the ArrayProxy encapsulation
                    new_var = self.new_array(index_max=foreign_var.index_max, index_bits=foreign_var.index_bits, value_bits=foreign_var.value_bits, name=migrated_name).array
                else:
                    raise NotImplemented(f"Unknown expression type {type(var)} encountered during expression migration")
                # Update the var to var mapping
                object_migration_map[foreign_var] = new_var
                # Update the name to name mapping
                name_migration_map[foreign_var.name] = new_var.name

        #  Actually replace each appearance of migrated variables by the new ones
        migrated_expression = replace(expression, object_migration_map)
        return migrated_expression

Migrate an expression created for a different constraint set to self . Returns an expression that can be used with this constraintSet


def inverse_transform(self, X_in):
        """
        Perform the inverse transformation to encoded data. Will attempt best case reconstruction, which means
        it will return nan for handle_missing and handle_unknown settings that break the bijection. We issue
        warnings when some of those cases occur.

        Parameters
        ----------
        X_in : array-like, shape = [n_samples, n_features]

        Returns
        -------
        p: array, the same size of X_in

        """
        X = X_in.copy(deep=True)

        # first check the type
        X = util.convert_input(X)

        if self._dim is None:
            raise ValueError(
                'Must train encoder before it can be used to inverse_transform data')

        # then make sure that it is the right size
        if X.shape[1] != self._dim:
            if self.drop_invariant:
                raise ValueError("Unexpected input dimension %d, the attribute drop_invariant should "
                                 "set as False when transform data" % (X.shape[1],))
            else:
                raise ValueError('Unexpected input dimension %d, expected %d' % (X.shape[1], self._dim,))

        if not self.cols:
            return X if self.return_df else X.values

        if self.handle_unknown == 'value':
            for col in self.cols:
                if any(X[col] == -1):
                    warnings.warn("inverse_transform is not supported because transform impute "
                                  "the unknown category -1 when encode %s" % (col,))

        if self.handle_unknown == 'return_nan' and self.handle_missing == 'return_nan':
            for col in self.cols:
                if X[col].isnull().any():
                    warnings.warn("inverse_transform is not supported because transform impute "
                                  "the unknown category nan when encode %s" % (col,))

        for switch in self.mapping:
            column_mapping = switch.get('mapping')
            inverse = pd.Series(data=column_mapping.index, index=column_mapping.get_values())
            X[switch.get('col')] = X[switch.get('col')].map(inverse).astype(switch.get('data_type'))

        return X if self.return_df else X.values

Perform the inverse transformation to encoded data . Will attempt best case reconstruction which means it will return nan for handle_missing and handle_unknown settings that break the bijection . We issue warnings when some of those cases occur .


def ordinal_encoding(X_in, mapping=None, cols=None, handle_unknown='value', handle_missing='value'):
        """
        Ordinal encoding uses a single column of integers to represent the classes. An optional mapping dict can be passed
        in, in this case we use the knowledge that there is some true order to the classes themselves. Otherwise, the classes
        are assumed to have no true order and integers are selected at random.
        """

        return_nan_series = pd.Series(data=[np.nan], index=[-2])

        X = X_in.copy(deep=True)

        if cols is None:
            cols = X.columns.values

        if mapping is not None:
            mapping_out = mapping
            for switch in mapping:
                column = switch.get('col')
                X[column] = X[column].map(switch['mapping'])

                try:
                    X[column] = X[column].astype(int)
                except ValueError as e:
                    X[column] = X[column].astype(float)

                if handle_unknown == 'value':
                    X[column].fillna(-1, inplace=True)
                elif handle_unknown == 'error':
                    missing = X[column].isnull()
                    if any(missing):
                        raise ValueError('Unexpected categories found in column %s' % column)

                if handle_missing == 'return_nan':
                    X[column] = X[column].map(return_nan_series).where(X[column] == -2, X[column])

        else:
            mapping_out = []
            for col in cols:

                nan_identity = np.nan

                if util.is_category(X[col].dtype):
                    categories = X[col].cat.categories
                else:
                    categories = X[col].unique()

                index = pd.Series(categories).fillna(nan_identity).unique()

                data = pd.Series(index=index, data=range(1, len(index) + 1))

                if handle_missing == 'value' and ~data.index.isnull().any():
                    data.loc[nan_identity] = -2
                elif handle_missing == 'return_nan':
                    data.loc[nan_identity] = -2

                mapping_out.append({'col': col, 'mapping': data, 'data_type': X[col].dtype}, )

        return X, mapping_out

Ordinal encoding uses a single column of integers to represent the classes . An optional mapping dict can be passed in in this case we use the knowledge that there is some true order to the classes themselves . Otherwise the classes are assumed to have no true order and integers are selected at random .


def reverse_dummies(self, X, mapping):
        """
        Convert dummy variable into numerical variables

        Parameters
        ----------
        X : DataFrame
        mapping: list-like
              Contains mappings of column to be transformed to it's new columns and value represented

        Returns
        -------
        numerical: DataFrame

        """
        out_cols = X.columns.values.tolist()
        mapped_columns = []
        for switch in mapping:
            col = switch.get('col')
            mod = switch.get('mapping')
            insert_at = out_cols.index(mod.columns[0])

            X.insert(insert_at, col, 0)
            positive_indexes = mod.index[mod.index > 0]
            for i in range(positive_indexes.shape[0]):
                existing_col = mod.columns[i]
                val = positive_indexes[i]
                X.loc[X[existing_col] == 1, col] = val
                mapped_columns.append(existing_col)
            X.drop(mod.columns, axis=1, inplace=True)
            out_cols = X.columns.values.tolist()

        return X

def reverse_dummies ( self , X , mapping ) : out_cols = X . columns . values . tolist ( ) mapped_columns = [ ] for switch in mapping : col = switch . get ( 'col' ) mod = switch . get ( 'mapping' ) insert_at = out_cols . index ( mod . columns [ 0 ] ) X . insert ( insert_at , col , 0 ) positive_indexes = mod . index [ mod . index > 0 ] for i in range ( positive_indexes . shape [ 0 ] ) : existing_col = mod . columns [ i ] val = positive_indexes [ i ] X . loc [ X [ existing_col ] == 1 , col ] = val mapped_columns . append ( existing_col ) X . drop ( mod . columns , axis = 1 , inplace = True ) out_cols = X . columns . values . tolist ( ) return X


def get_cars_data():
    """
    Load the cars dataset, split it into X and y, and then call the label encoder to get an integer y column.

    :return:
    """

    df = pd.read_csv('source_data/cars/car.data.txt')
    X = df.reindex(columns=[x for x in df.columns.values if x != 'class'])
    y = df.reindex(columns=['class'])
    y = preprocessing.LabelEncoder().fit_transform(y.values.reshape(-1, ))

    mapping = [
        {'col': 'buying', 'mapping': [('vhigh', 0), ('high', 1), ('med', 2), ('low', 3)]},
        {'col': 'maint', 'mapping': [('vhigh', 0), ('high', 1), ('med', 2), ('low', 3)]},
        {'col': 'doors', 'mapping': [('2', 0), ('3', 1), ('4', 2), ('5more', 3)]},
        {'col': 'persons', 'mapping': [('2', 0), ('4', 1), ('more', 2)]},
        {'col': 'lug_boot', 'mapping': [('small', 0), ('med', 1), ('big', 2)]},
        {'col': 'safety', 'mapping': [('high', 0), ('med', 1), ('low', 2)]},
    ]

    return X, y, mapping

def get_cars_data ( ) : df = pd . read_csv ( 'source_data/cars/car.data.txt' ) X = df . reindex ( columns = [ x for x in df . columns . values if x != 'class' ] ) y = df . reindex ( columns = [ 'class' ] ) y = preprocessing . LabelEncoder ( ) . fit_transform ( y . values . reshape ( - 1 , ) ) mapping = [ { 'col' : 'buying' , 'mapping' : [ ( 'vhigh' , 0 ) , ( 'high' , 1 ) , ( 'med' , 2 ) , ( 'low' , 3 ) ] } , { 'col' : 'maint' , 'mapping' : [ ( 'vhigh' , 0 ) , ( 'high' , 1 ) , ( 'med' , 2 ) , ( 'low' , 3 ) ] } , { 'col' : 'doors' , 'mapping' : [ ( '2' , 0 ) , ( '3' , 1 ) , ( '4' , 2 ) , ( '5more' , 3 ) ] } , { 'col' : 'persons' , 'mapping' : [ ( '2' , 0 ) , ( '4' , 1 ) , ( 'more' , 2 ) ] } , { 'col' : 'lug_boot' , 'mapping' : [ ( 'small' , 0 ) , ( 'med' , 1 ) , ( 'big' , 2 ) ] } , { 'col' : 'safety' , 'mapping' : [ ( 'high' , 0 ) , ( 'med' , 1 ) , ( 'low' , 2 ) ] } , ] return X , y , mapping

Load the cars dataset split it into X and y and then call the label encoder to get an integer y column .


def get_splice_data():
    """
    Load the mushroom dataset, split it into X and y, and then call the label encoder to get an integer y column.

    :return:
    """

    df = pd.read_csv('source_data/splice/splice.csv')
    X = df.reindex(columns=[x for x in df.columns.values if x != 'class'])
    X['dna'] = X['dna'].map(lambda x: list(str(x).strip()))
    for idx in range(60):
        X['dna_%d' % (idx, )] = X['dna'].map(lambda x: x[idx])
    del X['dna']

    y = df.reindex(columns=['class'])
    y = preprocessing.LabelEncoder().fit_transform(y.values.reshape(-1, ))

    # this data is truly categorical, with no known concept of ordering
    mapping = None

    return X, y, mapping

Load the mushroom dataset split it into X and y and then call the label encoder to get an integer y column .


def basen_to_integer(self, X, cols, base):
        """
        Convert basen code as integers.

        Parameters
        ----------
        X : DataFrame
            encoded data
        cols : list-like
            Column names in the DataFrame that be encoded
        base : int
            The base of transform

        Returns
        -------
        numerical: DataFrame

        """
        out_cols = X.columns.values.tolist()

        for col in cols:
            col_list = [col0 for col0 in out_cols if str(col0).startswith(str(col))]
            insert_at = out_cols.index(col_list[0])

            if base == 1:
                value_array = np.array([int(col0.split('_')[-1]) for col0 in col_list])
            else:
                len0 = len(col_list)
                value_array = np.array([base ** (len0 - 1 - i) for i in range(len0)])
            X.insert(insert_at, col, np.dot(X[col_list].values, value_array.T))
            X.drop(col_list, axis=1, inplace=True)
            out_cols = X.columns.values.tolist()

        return X

def basen_to_integer ( self , X , cols , base ) : out_cols = X . columns . values . tolist ( ) for col in cols : col_list = [ col0 for col0 in out_cols if str ( col0 ) . startswith ( str ( col ) ) ] insert_at = out_cols . index ( col_list [ 0 ] ) if base == 1 : value_array = np . array ( [ int ( col0 . split ( '_' ) [ - 1 ] ) for col0 in col_list ] ) else : len0 = len ( col_list ) value_array = np . array ( [ base ** ( len0 - 1 - i ) for i in range ( len0 ) ] ) X . insert ( insert_at , col , np . dot ( X [ col_list ] . values , value_array . T ) ) X . drop ( col_list , axis = 1 , inplace = True ) out_cols = X . columns . values . tolist ( ) return X


def col_transform(self, col, digits):
        """
        The lambda body to transform the column values
        """

        if col is None or float(col) < 0.0:
            return None
        else:
            col = self.number_to_base(int(col), self.base, digits)
            if len(col) == digits:
                return col
            else:
                return [0 for _ in range(digits - len(col))] + col

def col_transform ( self , col , digits ) : if col is None or float ( col ) < 0.0 : return None else : col = self . number_to_base ( int ( col ) , self . base , digits ) if len ( col ) == digits : return col else : return [ 0 for _ in range ( digits - len ( col ) ) ] + col


def get_obj_cols(df):
    """
    Returns names of 'object' columns in the DataFrame.
    """
    obj_cols = []
    for idx, dt in enumerate(df.dtypes):
        if dt == 'object' or is_category(dt):
            obj_cols.append(df.columns.values[idx])

    return obj_cols

def get_obj_cols ( df ) : obj_cols = [ ] for idx , dt in enumerate ( df . dtypes ) : if dt == 'object' or is_category ( dt ) : obj_cols . append ( df . columns . values [ idx ] ) return obj_cols

Returns names of object columns in the DataFrame .


def convert_input(X):
    """
    Unite data into a DataFrame.
    """
    if not isinstance(X, pd.DataFrame):
        if isinstance(X, list):
            X = pd.DataFrame(X)
        elif isinstance(X, (np.generic, np.ndarray)):
            X = pd.DataFrame(X)
        elif isinstance(X, csr_matrix):
            X = pd.DataFrame(X.todense())
        elif isinstance(X, pd.Series):
            X = pd.DataFrame(X)
        else:
            raise ValueError('Unexpected input type: %s' % (str(type(X))))

        X = X.apply(lambda x: pd.to_numeric(x, errors='ignore'))

    return X

def
convert_input
(
X
)
:
if
not
isinstance
(
X
,
pd
.
DataFrame
)
:
if
isinstance
(
X
,
list
)
:
X
=
pd
.
DataFrame
(
X
)
elif
isinstance
(
X
,
(
np
.
generic
,
np
.
ndarray
)
)
:
X
=
pd
.
DataFrame
(
X
)
elif
isinstance
(
X
,
csr_matrix
)
:
X
=
pd
.
DataFrame
(
X
.
todense
(
)
)
elif
isinstance
(
X
,
pd
.
Series
)
:
X
=
pd
.
DataFrame
(
X
)
else
:
raise
ValueError
(
'Unexpected input type: %s'
%
(
str
(
type
(
X
)
)
)
)
X
=
X
.
apply
(
lambda
x
:
pd
.
to_numeric
(
x
,
errors
=
'ignore'
)
)
return
X


def convert_input_vector(y, index):
    """
    Unite target data type into a Series.
    If the target is a Series or a DataFrame, we preserve its index.
    But if the target does not contain index attribute, we use the index from the argument.
    """
    if y is None:
        return None
    if isinstance(y, pd.Series):
        return y
    elif isinstance(y, np.ndarray):
        if len(np.shape(y))==1:  # vector
            return pd.Series(y, name='target', index=index)
        elif len(np.shape(y))==2 and np.shape(y)[0]==1:  # single row in a matrix
            return pd.Series(y[0, :], name='target', index=index)
        elif len(np.shape(y))==2 and np.shape(y)[1]==1:  # single column in a matrix
            return pd.Series(y[:, 0], name='target', index=index)
        else:
            raise ValueError('Unexpected input shape: %s' % (str(np.shape(y))))
    elif np.isscalar(y):
        return pd.Series([y], name='target', index=index)
    elif isinstance(y, list):
        if len(y)==0 or (len(y)>0 and not isinstance(y[0], list)): # empty list or a vector
            return pd.Series(y, name='target', index=index)
        elif len(y)>0 and isinstance(y[0], list) and len(y[0])==1: # single row in a matrix
            flatten = lambda y: [item for sublist in y for item in sublist]
            return pd.Series(flatten(y), name='target', index=index)
        elif len(y)==1 and isinstance(y[0], list): # single column in a matrix
            return pd.Series(y[0], name='target', index=index)
        else:
            raise ValueError('Unexpected input shape')
    elif isinstance(y, pd.DataFrame):
        if len(list(y))==0: # empty DataFrame
            return pd.Series(y, name='target')
        if len(list(y))==1: # a single column
            return y.iloc[:, 0]
        else:
            raise ValueError('Unexpected input shape: %s' % (str(y.shape)))
    else:
        return pd.Series(y, name='target', index=index)

def
convert_input_vector
(
y
,
index
)
:
if
y
is
:
return
if
isinstance
(
y
,
pd
.
Series
)
:
return
y
elif
isinstance
(
y
,
np
.
ndarray
)
:
if
len
(
np
.
shape
(
y
)
)
==
1
:
# vector
return
pd
.
Series
(
y
,
name
=
'target'
,
index
=
index
)
elif
len
(
np
.
shape
(
y
)
)
==
2
and
np
.
shape
(
y
)
[
0
]
==
1
:
# single row in a matrix
return
pd
.
Series
(
y
[
0
,
:
]
,
name
=
'target'
,
index
=
index
)
elif
len
(
np
.
shape
(
y
)
)
==
2
and
np
.
shape
(
y
)
[
1
]
==
1
:
# single column in a matrix
return
pd
.
Series
(
y
[
:
,
0
]
,
name
=
'target'
,
index
=
index
)
else
:
raise
ValueError
(
'Unexpected input shape: %s'
%
(
str
(
np
.
shape
(
y
)
)
)
)
elif
np
.
isscalar
(
y
)
:
return
pd
.
Series
(
[
y
]
,
name
=
'target'
,
index
=
index
)
elif
isinstance
(
y
,
list
)
:
if
len
(
y
)
==
0
or
(
len
(
y
)
>
0
and
not
isinstance
(
y
[
0
]
,
list
)
)
:
# empty list or a vector
return
pd
.
Series
(
y
,
name
=
'target'
,
index
=
index
)
elif
len
(
y
)
>
0
and
isinstance
(
y
[
0
]
,
list
)
and
len
(
y
[
0
]
)
==
1
:
# single row in a matrix
flatten
=
lambda
y
:
[
item
for
sublist
in
y
for
item
in
sublist
]
return
pd
.
Series
(
flatten
(
y
)
,
name
=
'target'
,
index
=
index
)
elif
len
(
y
)
==
1
and
isinstance
(
y
[
0
]
,
list
)
:
# single column in a matrix
return
pd
.
Series
(
y
[
0
]
,
name
=
'target'
,
index
=
index
)
else
:
raise
ValueError
(
'Unexpected input shape'
)
elif
isinstance
(
y
,
pd
.
DataFrame
)
:
if
len
(
list
(
y
)
)
==
0
:
# empty DataFrame
return
pd
.
Series
(
y
,
name
=
'target'
)
if
len
(
list
(
y
)
)
==
1
:
# a single column
return
y
.
iloc
[
:
,
0
]
else
:
raise
ValueError
(
'Unexpected input shape: %s'
%
(
str
(
y
.
shape
)
)
)
else
:
return
pd
.
Series
(
y
,
name
=
'target'
,
index
=
index
)

Unite target data type into a Series . If the target is a Series or a DataFrame we preserve its index . But if the target does not contain index attribute we use the index from the argument .


def score_models(clf, X, y, encoder, runs=1):
    """
    Takes in a classifier that supports multiclass classification, and X and a y, and returns a cross validation score.

    """

    scores = []

    X_test = None
    for _ in range(runs):
        X_test = encoder().fit_transform(X, y)

        # Some models, like logistic regression, like normalized features otherwise they underperform and/or take a long time to converge.
        # To be rigorous, we should have trained the normalization on each fold individually via pipelines.
        # See grid_search_example to learn how to do it.
        X_test = StandardScaler().fit_transform(X_test)

        scores.append(cross_validate(clf, X_test, y, n_jobs=1, cv=5)['test_score'])
        gc.collect()

    scores = [y for z in [x for x in scores] for y in z]

    return float(np.mean(scores)), float(np.std(scores)), scores, X_test.shape[1]

Takes in a classifier that supports multiclass classification and X and a y and returns a cross validation score .


def main(loader, name):
    """
    Here we iterate through the datasets and score them with a classifier using different encodings.

    """

    scores = []
    raw_scores_ds = {}

    # first get the dataset
    X, y, mapping = loader()

    clf = linear_model.LogisticRegression(solver='lbfgs', multi_class='auto', max_iter=200, random_state=0)

    # try each encoding method available, which works on multiclass problems
    encoders = (set(category_encoders.__all__) - {'WOEEncoder'})  # WoE is currently only for binary targets

    for encoder_name in encoders:
        encoder = getattr(category_encoders, encoder_name)
        start_time = time.time()
        score, stds, raw_scores, dim = score_models(clf, X, y, encoder)
        scores.append([encoder_name, name, dim, score, stds, time.time() - start_time])
        raw_scores_ds[encoder_name] = raw_scores
        gc.collect()

    results = pd.DataFrame(scores, columns=['Encoding', 'Dataset', 'Dimensionality', 'Avg. Score', 'Score StDev', 'Elapsed Time'])

    raw = pd.DataFrame.from_dict(raw_scores_ds)
    ax = raw.plot(kind='box', return_type='axes')
    plt.title('Scores for Encodings on %s Dataset' % (name,))
    plt.ylabel('Score (higher is better)')
    for tick in ax.get_xticklabels():
        tick.set_rotation(90)
    plt.grid()
    plt.tight_layout()
    plt.show()

    return results, raw

Here we iterate through the datasets and score them with a classifier using different encodings .


def secho(message, **kwargs):
    """A wrapper around click.secho that disables any coloring being used
    if colors have been disabled.
    """
    # If colors are disabled, remove any color or other style data
    # from keyword arguments.
    if not settings.color:
        for key in ('fg', 'bg', 'bold', 'blink'):
            kwargs.pop(key, None)

    # Okay, now call click.secho normally.
    return click.secho(message, **kwargs)

A wrapper around click . secho that disables any coloring being used if colors have been disabled .


def associate_notification_template(self, job_template,
                                        notification_template, status):
        """Associate a notification template from this job template.

        =====API DOCS=====
        Associate a notification template from this job template.

        :param job_template: The job template to associate to.
        :type job_template: str
        :param notification_template: The notification template to be associated.
        :type notification_template: str
        :param status: type of notification this notification template should be associated to.
        :type status: str
        :returns: Dictionary of only one key "changed", which indicates whether the association succeeded.
        :rtype: dict

        =====API DOCS=====
        """
        return self._assoc('notification_templates_%s' % status,
                           job_template, notification_template)

def associate_notification_template ( self , job_template , notification_template , status ) : return self . _assoc ( 'notification_templates_%s' % status , job_template , notification_template )

Associate a notification template from this job template .


def disassociate_notification_template(self, job_template,
                                           notification_template, status):
        """Disassociate a notification template from this job template.

        =====API DOCS=====
        Disassociate a notification template from this job template.

        :param job_template: The job template to disassociate from.
        :type job_template: str
        :param notification_template: The notification template to be disassociated.
        :type notification_template: str
        :param status: type of notification this notification template should be disassociated from.
        :type status: str
        :returns: Dictionary of only one key "changed", which indicates whether the disassociation succeeded.
        :rtype: dict

        =====API DOCS=====
        """
        return self._disassoc('notification_templates_%s' % status,
                              job_template, notification_template)

def disassociate_notification_template ( self , job_template , notification_template , status ) : return self . _disassoc ( 'notification_templates_%s' % status , job_template , notification_template )

Disassociate a notification template from this job template .


def callback(self, pk=None, host_config_key='', extra_vars=None):
        """Contact Tower and request a configuration update using this job template.

        =====API DOCS=====
        Contact Tower and request a provisioning callback using this job template.

        :param pk: Primary key of the job template to run provisioning callback against.
        :type pk: int
        :param host_config_key: Key string used to authenticate the callback host.
        :type host_config_key: str
        :param extra_vars: Extra variables that are passed to provisioning callback.
        :type extra_vars: array of str
        :returns: A dictionary of a single key "changed", which indicates whether the provisioning callback
                  is successful.
        :rtype: dict

        =====API DOCS=====
        """
        url = self.endpoint + '%s/callback/' % pk
        if not host_config_key:
            host_config_key = client.get(url).json()['host_config_key']
        post_data = {'host_config_key': host_config_key}
        if extra_vars:
            post_data['extra_vars'] = parser.process_extra_vars(list(extra_vars), force_json=True)
        r = client.post(url, data=post_data, auth=None)
        if r.status_code == 201:
            return {'changed': True}

def callback ( self , pk = None , host_config_key = '' , extra_vars = None ) : url = self . endpoint + '%s/callback/' % pk if not host_config_key : host_config_key = client . get ( url ) . json ( ) [ 'host_config_key' ] post_data = { 'host_config_key' : host_config_key } if extra_vars : post_data [ 'extra_vars' ] = parser . process_extra_vars ( list ( extra_vars ) , force_json = True ) r = client . post ( url , data = post_data , auth = None ) if r . status_code == 201 : return { 'changed' : True }

Contact Tower and request a configuration update using this job template .


def jt_aggregate(func, is_create=False, has_pk=False):
    """Decorator to aggregate unified_jt-related fields.

    Args:
        func: The CURD method to be decorated.
        is_create: Boolean flag showing whether this method is create.
        has_pk: Boolean flag showing whether this method uses pk as argument.

    Returns:
        A function with necessary click-related attributes whose keyworded
        arguments are aggregated.

    Raises:
        exc.UsageError: Either more than one unified jt fields are
            provided, or none is provided when is_create flag is set.
    """
    def helper(kwargs, obj):
        """The helper function preceding actual function that aggregates
        unified jt fields.
        """
        unified_job_template = None
        for item in UNIFIED_JT:
            if kwargs.get(item, None) is not None:
                jt_id = kwargs.pop(item)
                if unified_job_template is None:
                    unified_job_template = (item, jt_id)
                else:
                    raise exc.UsageError(
                        'More than one unified job template fields provided, '
                        'please tighten your criteria.'
                    )
        if unified_job_template is not None:
            kwargs['unified_job_template'] = unified_job_template[1]
            obj.identity = tuple(list(obj.identity) + ['unified_job_template'])
            return '/'.join([UNIFIED_JT[unified_job_template[0]],
                             str(unified_job_template[1]), 'schedules/'])
        elif is_create:
            raise exc.UsageError('You must provide exactly one unified job'
                                 ' template field during creation.')

    def decorator_without_pk(obj, *args, **kwargs):
        old_endpoint = obj.endpoint
        new_endpoint = helper(kwargs, obj)
        if is_create:
            obj.endpoint = new_endpoint
        result = func(obj, *args, **kwargs)
        obj.endpoint = old_endpoint
        return result

    def decorator_with_pk(obj, pk=None, *args, **kwargs):
        old_endpoint = obj.endpoint
        new_endpoint = helper(kwargs, obj)
        if is_create:
            obj.endpoint = new_endpoint
        result = func(obj, pk=pk, *args, **kwargs)
        obj.endpoint = old_endpoint
        return result

    decorator = decorator_with_pk if has_pk else decorator_without_pk
    for item in CLICK_ATTRS:
        setattr(decorator, item, getattr(func, item, []))
    decorator.__doc__ = func.__doc__

    return decorator

Decorator to aggregate unified_jt - related fields .


def lookup_stdout(self, pk=None, start_line=None, end_line=None,
                      full=True):
        """
        Internal method that lies to our `monitor` method by returning
        a scorecard for the workflow job where the standard out
        would have been expected.
        """
        uj_res = get_resource('unified_job')
        # Filters
        #  - limit search to jobs spawned as part of this workflow job
        #  - order in the order in which they should add to the list
        #  - only include final job states
        query_params = (('unified_job_node__workflow_job', pk),
                        ('order_by', 'finished'),
                        ('status__in', 'successful,failed,error'))
        jobs_list = uj_res.list(all_pages=True, query=query_params)
        if jobs_list['count'] == 0:
            return ''

        return_content = ResSubcommand(uj_res)._format_human(jobs_list)
        lines = return_content.split('\n')
        if not full:
            lines = lines[:-1]

        N = len(lines)
        start_range = start_line
        if start_line is None:
            start_range = 0
        elif start_line > N:
            start_range = N

        end_range = end_line
        if end_line is None or end_line > N:
            end_range = N

        lines = lines[start_range:end_range]
        return_content = '\n'.join(lines)
        if len(lines) > 0:
            return_content += '\n'

        return return_content

Internal method that lies to our monitor method by returning a scorecard for the workflow job where the standard out would have been expected .


def launch(self, workflow_job_template=None, monitor=False, wait=False,
               timeout=None, extra_vars=None, **kwargs):
        """Launch a new workflow job based on a workflow job template.

        Creates a new workflow job in Ansible Tower, starts it, and
        returns back an ID in order for its status to be monitored.

        =====API DOCS=====
        Launch a new workflow job based on a workflow job template.

        :param workflow_job_template: Primary key or name of the workflow job template to launch new job.
        :type workflow_job_template: str
        :param monitor: Flag that if set, immediately calls ``monitor`` on the newly launched workflow job rather
                        than exiting with a success.
        :type monitor: bool
        :param wait: Flag that if set, monitor the status of the workflow job, but do not print while job is
                     in progress.
        :type wait: bool
        :param timeout: If provided with ``monitor`` flag set, this attempt will time out after the given number
                        of seconds.
        :type timeout: int
        :param extra_vars: yaml formatted texts that contains extra variables to pass on.
        :type extra_vars: array of strings
        :param `**kwargs`: Fields needed to create and launch a workflow job.
        :returns: Result of subsequent ``monitor`` call if ``monitor`` flag is on; Result of subsequent ``wait``
                  call if ``wait`` flag is on; loaded JSON output of the job launch if none of the two flags are on.
        :rtype: dict

        =====API DOCS=====
        """
        if extra_vars is not None and len(extra_vars) > 0:
            kwargs['extra_vars'] = parser.process_extra_vars(extra_vars)

        debug.log('Launching the workflow job.', header='details')
        self._pop_none(kwargs)
        post_response = client.post('workflow_job_templates/{0}/launch/'.format(
            workflow_job_template), data=kwargs).json()

        workflow_job_id = post_response['id']
        post_response['changed'] = True

        if monitor:
            return self.monitor(workflow_job_id, timeout=timeout)
        elif wait:
            return self.wait(workflow_job_id, timeout=timeout)

        return post_response

def
launch
(
self
,
workflow_job_template
=
,
monitor
=
False
,
wait
=
False
,
timeout
=
,
extra_vars
=
,
*
*
kwargs
)
:
if
extra_vars
is
not
and
len
(
extra_vars
)
>
0
:
kwargs
[
'extra_vars'
]
=
parser
.
process_extra_vars
(
extra_vars
)
debug
.
log
(
'Launching the workflow job.'
,
header
=
'details'
)
self
.
_pop_none
(
kwargs
)
post_response
=
client
.
post
(
'workflow_job_templates/{0}/launch/'
.
format
(
workflow_job_template
)
,
data
=
kwargs
)
.
json
(
)
workflow_job_id
=
post_response
[
'id'
]
post_response
[
'changed'
]
=
True
if
monitor
:
return
self
.
monitor
(
workflow_job_id
,
timeout
=
timeout
)
elif
wait
:
return
self
.
wait
(
workflow_job_id
,
timeout
=
timeout
)
return
post_response

Launch a new workflow job based on a workflow job template .


def parse_args(self, ctx, args):
        """Parse arguments sent to this command.

        The code for this method is taken from MultiCommand:
        https://github.com/mitsuhiko/click/blob/master/click/core.py

        It is Copyright (c) 2014 by Armin Ronacher.
        See the license:
        https://github.com/mitsuhiko/click/blob/master/LICENSE
        """
        if not args and self.no_args_is_help and not ctx.resilient_parsing:
            click.echo(ctx.get_help())
            ctx.exit()
        return super(ActionSubcommand, self).parse_args(ctx, args)

def parse_args ( self , ctx , args ) : if not args and self . no_args_is_help and not ctx . resilient_parsing : click . echo ( ctx . get_help ( ) ) ctx . exit ( ) return super ( ActionSubcommand , self ) . parse_args ( ctx , args )


def format_options(self, ctx, formatter):
        """Monkey-patch click's format_options method to support option categorization.
        """
        field_opts = []
        global_opts = []
        local_opts = []
        other_opts = []
        for param in self.params:
            if param.name in SETTINGS_PARMS:
                opts = global_opts
            elif getattr(param, 'help', None) and param.help.startswith('[FIELD]'):
                opts = field_opts
                param.help = param.help[len('[FIELD]'):]
            else:
                opts = local_opts
            rv = param.get_help_record(ctx)
            if rv is None:
                continue
            else:
                opts.append(rv)

        if self.add_help_option:
            help_options = self.get_help_option_names(ctx)
            if help_options:
                other_opts.append([join_options(help_options)[0], 'Show this message and exit.'])

        if field_opts:
            with formatter.section('Field Options'):
                formatter.write_dl(field_opts)
        if local_opts:
            with formatter.section('Local Options'):
                formatter.write_dl(local_opts)
        if global_opts:
            with formatter.section('Global Options'):
                formatter.write_dl(global_opts)
        if other_opts:
            with formatter.section('Other Options'):
                formatter.write_dl(other_opts)

def
format_options
(
self
,
ctx
,
formatter
)
:
field_opts
=
[
]
global_opts
=
[
]
local_opts
=
[
]
other_opts
=
[
]
for
param
in
self
.
params
:
if
param
.
name
in
SETTINGS_PARMS
:
opts
=
global_opts
elif
getattr
(
param
,
'help'
,
)
and
param
.
help
.
startswith
(
'[FIELD]'
)
:
opts
=
field_opts
param
.
help
=
param
.
help
[
len
(
'[FIELD]'
)
:
]
else
:
opts
=
local_opts
rv
=
param
.
get_help_record
(
ctx
)
if
rv
is
:
continue
else
:
opts
.
append
(
rv
)
if
self
.
add_help_option
:
help_options
=
self
.
get_help_option_names
(
ctx
)
if
help_options
:
other_opts
.
append
(
[
join_options
(
help_options
)
[
0
]
,
'Show this message and exit.'
]
)
if
field_opts
:
with
formatter
.
section
(
'Field Options'
)
:
formatter
.
write_dl
(
field_opts
)
if
local_opts
:
with
formatter
.
section
(
'Local Options'
)
:
formatter
.
write_dl
(
local_opts
)
if
global_opts
:
with
formatter
.
section
(
'Global Options'
)
:
formatter
.
write_dl
(
global_opts
)
if
other_opts
:
with
formatter
.
section
(
'Other Options'
)
:
formatter
.
write_dl
(
other_opts
)

Monkey - patch click s format_options method to support option categorization .


def get(self, pk):
        """Return one and exactly one object

        =====API DOCS=====
        Return one and exactly one Tower setting.

        :param pk: Primary key of the Tower setting to retrieve
        :type pk: int
        :returns: loaded JSON of the retrieved Tower setting object.
        :rtype: dict
        :raises tower_cli.exceptions.NotFound: When no specified Tower setting exists.

        =====API DOCS=====
        """
        # The Tower API doesn't provide a mechanism for retrieving a single
        # setting value at a time, so fetch them all and filter
        try:
            return next(s for s in self.list()['results'] if s['id'] == pk)
        except StopIteration:
            raise exc.NotFound('The requested object could not be found.')


def _convert_pagenum(self, kwargs):
        """
        Convert next and previous from URLs to integers
        """
        for key in ('next', 'previous'):
            if not kwargs.get(key):
                continue
            match = re.search(r'page=(?P<num>[\d]+)', kwargs[key])
            if match is None and key == 'previous':
                kwargs[key] = 1
                continue
            kwargs[key] = int(match.groupdict()['num'])

def _convert_pagenum ( self , kwargs ) : for key in ( 'next' , 'previous' ) : if not kwargs . get ( key ) : continue match = re . search ( r'page=(?P<num>[\d]+)' , kwargs [ key ] ) if match is None and key == 'previous' : kwargs [ key ] = 1 continue kwargs [ key ] = int ( match . groupdict ( ) [ 'num' ] )


def delete(self, pk=None, fail_on_missing=False, **kwargs):
        """Remove the given object.

        If `fail_on_missing` is True, then the object's not being found is considered a failure; otherwise,
        a success with no change is reported.

        =====API DOCS=====
        Remove the given object.

        :param pk: Primary key of the resource to be deleted.
        :type pk: int
        :param fail_on_missing: Flag that if set, the object's not being found is considered a failure; otherwise,
                                a success with no change is reported.
        :type fail_on_missing: bool
        :param `**kwargs`: Keyword arguments used to look up resource object to delete if ``pk`` is not provided.
        :returns: dictionary of only one field "changed", which is a flag indicating whether the specified resource
                  is successfully deleted.
        :rtype: dict

        =====API DOCS=====
        """
        # If we weren't given a primary key, determine which record we're deleting.
        if not pk:
            existing_data = self._lookup(fail_on_missing=fail_on_missing, **kwargs)
            if not existing_data:
                return {'changed': False}
            pk = existing_data['id']

        # Attempt to delete the record. If it turns out the record doesn't exist, handle the 404 appropriately
        # (this is an okay response if `fail_on_missing` is False).
        url = '%s%s/' % (self.endpoint, pk)
        debug.log('DELETE %s' % url, fg='blue', bold=True)
        try:
            client.delete(url)
            return {'changed': True}
        except exc.NotFound:
            if fail_on_missing:
                raise
            return {'changed': False}


def get(self, pk=None, **kwargs):
        """Return one and exactly one object.

        Lookups may be through a primary key, specified as a positional argument, and/or through filters specified
        through keyword arguments.

        If the number of results does not equal one, raise an exception.

        =====API DOCS=====
        Retrieve one and exactly one object.

        :param pk: Primary key of the resource to be read. Tower CLI will only attempt to read *that* object
                   if ``pk`` is provided (not ``None``).
        :type pk: int
        :param `**kwargs`: Keyword arguments used to look up resource object to retrieve if ``pk`` is not provided.
        :returns: loaded JSON of the retrieved resource object.
        :rtype: dict

        =====API DOCS=====
        """
        if kwargs.pop('include_debug_header', True):
            debug.log('Getting the record.', header='details')
        response = self.read(pk=pk, fail_on_no_results=True, fail_on_multiple_results=True, **kwargs)
        return response['results'][0]

def
get
(
self
,
pk
=
,
*
*
kwargs
)
:
if
kwargs
.
pop
(
'include_debug_header'
,
True
)
:
debug
.
log
(
'Getting the record.'
,
header
=
'details'
)
response
=
self
.
read
(
pk
=
pk
,
fail_on_no_results
=
True
,
fail_on_multiple_results
=
True
,
*
*
kwargs
)
return
response
[
'results'
]
[
0
]


def _disassoc(self, url_fragment, me, other):
        """Disassociate the `other` record from the `me` record."""

        # Get the endpoint for foreign records within this object.
        url = self.endpoint + '%d/%s/' % (me, url_fragment)

        # Attempt to determine whether the other record already is absent, for the "changed" moniker.
        r = client.get(url, params={'id': other}).json()
        if r['count'] == 0:
            return {'changed': False}

        # Send a request removing the foreign record from this one.
        r = client.post(url, data={'disassociate': True, 'id': other})
        return {'changed': True}

Disassociate the other record from the me record .


def copy(self, pk=None, new_name=None, **kwargs):
        """Copy an object.

        Only the ID is used for the lookup. All provided fields are used to override the old data from the
        copied resource.

        =====API DOCS=====
        Copy an object.

        :param pk: Primary key of the resource object to be copied
        :param new_name: The new name to give the resource if deep copying via the API
        :type pk: int
        :param `**kwargs`: Keyword arguments of fields whose given value will override the original value.
        :returns: loaded JSON of the copied new resource object.
        :rtype: dict

        =====API DOCS=====
        """
        orig = self.read(pk, fail_on_no_results=True, fail_on_multiple_results=True)
        orig = orig['results'][0]
        # Remove default values (anything where the value is None).
        self._pop_none(kwargs)

        newresource = copy(orig)
        newresource.pop('id')
        basename = newresource['name'].split('@', 1)[0].strip()

        # Modify data to fit the call pattern of the tower-cli method
        for field in self.fields:
            if field.multiple and field.name in newresource:
                newresource[field.name] = (newresource.get(field.name),)

        if new_name is None:
            # copy client-side, the old mechanism
            newresource['name'] = "%s @ %s" % (basename, time.strftime('%X'))
            newresource.update(kwargs)

            return self.write(create_on_missing=True, fail_on_found=True,
                              **newresource)
        else:
            # copy server-side, the new mechanism
            if kwargs:
                raise exc.TowerCLIError('Cannot override {} and also use --new-name.'.format(kwargs.keys()))
            copy_endpoint = '{}/{}/copy/'.format(self.endpoint.strip('/'), pk)
            return client.post(copy_endpoint, data={'name': new_name}).json()


def last_job_data(self, pk=None, **kwargs):
        """
        Internal utility function for Unified Job Templates. Returns data about the last job run off of that UJT
        """
        ujt = self.get(pk, include_debug_header=True, **kwargs)

        # Determine the appropriate inventory source update.
        if 'current_update' in ujt['related']:
            debug.log('A current job; retrieving it.', header='details')
            return client.get(ujt['related']['current_update'][7:]).json()
        elif ujt['related'].get('last_update', None):
            debug.log('No current job or update exists; retrieving the most recent.', header='details')
            return client.get(ujt['related']['last_update'][7:]).json()
        else:
            raise exc.NotFound('No related jobs or updates exist.')

Internal utility function for Unified Job Templates . Returns data about the last job run off of that UJT


def lookup_stdout(self, pk=None, start_line=None, end_line=None, full=True):
        """
        Internal utility function to return standard out. Requires the pk of a unified job.
        """
        stdout_url = '%s%s/stdout/' % (self.unified_job_type, pk)
        payload = {'format': 'json', 'content_encoding': 'base64', 'content_format': 'ansi'}
        if start_line:
            payload['start_line'] = start_line
        if end_line:
            payload['end_line'] = end_line
        debug.log('Requesting a copy of job standard output', header='details')
        resp = client.get(stdout_url, params=payload).json()
        content = b64decode(resp['content'])
        return content.decode('utf-8', 'replace')

Internal utility function to return standard out . Requires the pk of a unified job .


def stdout(self, pk, start_line=None, end_line=None, outfile=sys.stdout, **kwargs):
        """
        Print out the standard out of a unified job to the command line or output file.
        For Projects, print the standard out of most recent update.
        For Inventory Sources, print standard out of most recent sync.
        For Jobs, print the job's standard out.
        For Workflow Jobs, print a status table of its jobs.

        =====API DOCS=====
        Print out the standard out of a unified job to the command line or output file.
        For Projects, print the standard out of most recent update.
        For Inventory Sources, print standard out of most recent sync.
        For Jobs, print the job's standard out.
        For Workflow Jobs, print a status table of its jobs.

        :param pk: Primary key of the job resource object to be monitored.
        :type pk: int
        :param start_line: Line at which to start printing job output
        :param end_line: Line at which to end printing job output
        :param outfile: Alternative file than stdout to write job stdout to.
        :type outfile: file
        :param `**kwargs`: Keyword arguments used to look up job resource object to monitor if ``pk`` is
                           not provided.
        :returns: A dictionary containing changed=False
        :rtype: dict

        =====API DOCS=====
        """

        # resource is Unified Job Template
        if self.unified_job_type != self.endpoint:
            unified_job = self.last_job_data(pk, **kwargs)
            pk = unified_job['id']
        # resource is Unified Job, but pk not given
        elif not pk:
            unified_job = self.get(**kwargs)
            pk = unified_job['id']

        content = self.lookup_stdout(pk, start_line, end_line)
        opened = False
        if isinstance(outfile, six.string_types):
            outfile = open(outfile, 'w')
            opened = True
        if len(content) > 0:
            click.echo(content, nl=1, file=outfile)
        if opened:
            outfile.close()

        return {"changed": False}

Print out the standard out of a unified job to the command line or output file . For Projects print the standard out of most recent update . For Inventory Sources print standard out of most recent sync . For Jobs print the job s standard out . For Workflow Jobs print a status table of its jobs .


def monitor(self, pk, parent_pk=None, timeout=None, interval=0.5, outfile=sys.stdout, **kwargs):
        """
        Stream the standard output from a job, project update, or inventory udpate.

        =====API DOCS=====
        Stream the standard output from a job run to stdout.

        :param pk: Primary key of the job resource object to be monitored.
        :type pk: int
        :param parent_pk: Primary key of the unified job template resource object whose latest job run will be
                          monitored if ``pk`` is not set.
        :type parent_pk: int
        :param timeout: Number in seconds after which this method will time out.
        :type timeout: float
        :param interval: Polling interval to refresh content from Tower.
        :type interval: float
        :param outfile: Alternative file than stdout to write job stdout to.
        :type outfile: file
        :param `**kwargs`: Keyword arguments used to look up job resource object to monitor if ``pk`` is
                           not provided.
        :returns: A dictionary combining the JSON output of the finished job resource object, as well as
                  two extra fields: "changed", a flag indicating if the job resource object is finished
                  as expected; "id", an integer which is the primary key of the job resource object being
                  monitored.
        :rtype: dict
        :raises tower_cli.exceptions.Timeout: When monitor time reaches time out.
        :raises tower_cli.exceptions.JobFailure: When the job being monitored runs into failure.

        =====API DOCS=====
        """

        # If we do not have the unified job info, infer it from parent
        if pk is None:
            pk = self.last_job_data(parent_pk, **kwargs)['id']
        job_endpoint = '%s%s/' % (self.unified_job_type, pk)

        # Pause until job is in running state
        self.wait(pk, exit_on=['running', 'successful'], outfile=outfile)

        # Loop initialization
        start = time.time()
        start_line = 0
        result = client.get(job_endpoint).json()

        click.echo('\033[0;91m------Starting Standard Out Stream------\033[0m', nl=2, file=outfile)

        # Poll the Ansible Tower instance for status and content, and print standard out to the out file
        while not result['failed'] and result['status'] != 'successful':

            result = client.get(job_endpoint).json()

            # Put the process to sleep briefly.
            time.sleep(interval)

            # Make request to get standard out
            content = self.lookup_stdout(pk, start_line, full=False)

            # In the first moments of running the job, the standard out
            # may not be available yet
            if not content.startswith("Waiting for results"):
                line_count = len(content.splitlines())
                start_line += line_count
                click.echo(content, nl=0, file=outfile)

            if timeout and time.time() - start > timeout:
                raise exc.Timeout('Monitoring aborted due to timeout.')

        # Special final line for closure with workflow jobs
        if self.endpoint == '/workflow_jobs/':
            click.echo(self.lookup_stdout(pk, start_line, full=True), nl=1)

        click.echo('\033[0;91m------End of Standard Out Stream--------\033[0m', nl=2, file=outfile)

        if result['failed']:
            raise exc.JobFailure('Job failed.')

        # Return the job ID and other response data
        answer = OrderedDict((('changed', True), ('id', pk)))
        answer.update(result)
        # Make sure to return ID of resource and not update number relevant for project creation and update
        if parent_pk:
            answer['id'] = parent_pk
        else:
            answer['id'] = pk
        return answer

Stream the standard output from a job project update or inventory udpate .


def status(self, pk=None, detail=False, **kwargs):
        """Print the current job status. This is used to check a running job. You can look up the job with
        the same parameters used for a get request.

        =====API DOCS=====
        Retrieve the current job status.

        :param pk: Primary key of the resource to retrieve status from.
        :type pk: int
        :param detail: Flag that if set, return the full JSON of the job resource rather than a status summary.
        :type detail: bool
        :param `**kwargs`: Keyword arguments used to look up resource object to retrieve status from if ``pk``
                           is not provided.
        :returns: full loaded JSON of the specified unified job if ``detail`` flag is on; trimed JSON containing
                  only "elapsed", "failed" and "status" fields of the unified job if ``detail`` flag is off.
        :rtype: dict

        =====API DOCS=====
        """
        # Remove default values (anything where the value is None).
        self._pop_none(kwargs)

        # Search for the record if pk not given
        if not pk:
            job = self.get(include_debug_header=True, **kwargs)
        # Get the job from Ansible Tower if pk given
        else:
            debug.log('Asking for job status.', header='details')
            finished_endpoint = '%s%s/' % (self.endpoint, pk)
            job = client.get(finished_endpoint).json()

        # In most cases, we probably only want to know the status of the job and the amount of time elapsed.
        # However, if we were asked for verbose information, provide it.
        if detail:
            return job

        # Print just the information we need.
        return {
            'elapsed': job['elapsed'],
            'failed': job['failed'],
            'status': job['status'],
        }

Print the current job status . This is used to check a running job . You can look up the job with the same parameters used for a get request .


def cancel(self, pk=None, fail_if_not_running=False, **kwargs):
        """Cancel a currently running job.

        Fails with a non-zero exit status if the job cannot be canceled.
        You must provide either a pk or parameters in the job's identity.

        =====API DOCS=====
        Cancel a currently running job.

        :param pk: Primary key of the job resource to restart.
        :type pk: int
        :param fail_if_not_running: Flag that if set, raise exception if the job resource cannot be canceled.
        :type fail_if_not_running: bool
        :param `**kwargs`: Keyword arguments used to look up job resource object to restart if ``pk`` is not
                           provided.
        :returns: A dictionary of two keys: "status", which is "canceled", and "changed", which indicates if
                  the job resource has been successfully canceled.
        :rtype: dict
        :raises tower_cli.exceptions.TowerCLIError: When the job resource cannot be canceled and
                                                    ``fail_if_not_running`` flag is on.
        =====API DOCS=====
        """
        # Search for the record if pk not given
        if not pk:
            existing_data = self.get(**kwargs)
            pk = existing_data['id']

        cancel_endpoint = '%s%s/cancel/' % (self.endpoint, pk)
        # Attempt to cancel the job.
        try:
            client.post(cancel_endpoint)
            changed = True
        except exc.MethodNotAllowed:
            changed = False
            if fail_if_not_running:
                raise exc.TowerCLIError('Job not running.')

        # Return a success.
        return {'status': 'canceled', 'changed': changed}

def
cancel
(
self
,
pk
=
,
fail_if_not_running
=
False
,
*
*
kwargs
)
:
# Search for the record if pk not given
if
not
pk
:
existing_data
=
self
.
get
(
*
*
kwargs
)
pk
=
existing_data
[
'id'
]
cancel_endpoint
=
'%s%s/cancel/'
%
(
self
.
endpoint
,
pk
)
# Attempt to cancel the job.
try
:
client
.
post
(
cancel_endpoint
)
changed
=
True
except
exc
.
MethodNotAllowed
:
changed
=
False
if
fail_if_not_running
:
raise
exc
.
TowerCLIError
(
'Job not running.'
)
# Return a success.
return
{
'status'
:
'canceled'
,
'changed'
:
changed
}


def relaunch(self, pk=None, **kwargs):
        """Relaunch a stopped job.

        Fails with a non-zero exit status if the job cannot be relaunched.
        You must provide either a pk or parameters in the job's identity.

        =====API DOCS=====
        Relaunch a stopped job resource.

        :param pk: Primary key of the job resource to relaunch.
        :type pk: int
        :param `**kwargs`: Keyword arguments used to look up job resource object to relaunch if ``pk`` is not
                           provided.
        :returns: A dictionary combining the JSON output of the relaunched job resource object, as well
                  as an extra field "changed", a flag indicating if the job resource object is status-changed
                  as expected.
        :rtype: dict

        =====API DOCS=====
        """
        # Search for the record if pk not given
        if not pk:
            existing_data = self.get(**kwargs)
            pk = existing_data['id']

        relaunch_endpoint = '%s%s/relaunch/' % (self.endpoint, pk)
        data = {}
        # Attempt to relaunch the job.
        answer = {}
        try:
            result = client.post(relaunch_endpoint, data=data).json()
            if 'id' in result:
                answer.update(result)
            answer['changed'] = True
        except exc.MethodNotAllowed:
            answer['changed'] = False

        # Return the answer.
        return answer

def
relaunch
(
self
,
pk
=
,
*
*
kwargs
)
:
# Search for the record if pk not given
if
not
pk
:
existing_data
=
self
.
get
(
*
*
kwargs
)
pk
=
existing_data
[
'id'
]
relaunch_endpoint
=
'%s%s/relaunch/'
%
(
self
.
endpoint
,
pk
)
data
=
{
}
# Attempt to relaunch the job.
answer
=
{
}
try
:
result
=
client
.
post
(
relaunch_endpoint
,
data
=
data
)
.
json
(
)
if
'id'
in
result
:
answer
.
update
(
result
)
answer
[
'changed'
]
=
True
except
exc
.
MethodNotAllowed
:
answer
[
'changed'
]
=
False
# Return the answer.
return
answer


def batch_update(self, pk=None, **kwargs):
        """Update all related inventory sources of the given inventory.

        Note global option --format is not available here, as the output would always be JSON-formatted.

        =====API DOCS=====
        Update all related inventory sources of the given inventory.

        :param pk: Primary key of the given inventory.
        :type pk: int
        :param `**kwargs`: Keyword arguments list of available fields used for searching resource objects.
        :returns: A JSON object of update status of the given inventory.
        :rtype: dict
        =====API DOCS=====
        """
        res = self.get(pk=pk, **kwargs)
        url = self.endpoint + '%d/%s/' % (res['id'], 'update_inventory_sources')
        return client.post(url, data={}).json()

def batch_update ( self , pk = None , * * kwargs ) : res = self . get ( pk = pk , * * kwargs ) url = self . endpoint + '%d/%s/' % ( res [ 'id' ] , 'update_inventory_sources' ) return client . post ( url , data = { } ) . json ( )

Update all related inventory sources of the given inventory .


def read(self, *args, **kwargs):
        '''
        Do extra processing so we can display the actor field as
        a top-level field
        '''
        if 'actor' in kwargs:
            kwargs['actor'] = kwargs.pop('actor')
        r = super(Resource, self).read(*args, **kwargs)
        if 'results' in r:
            for d in r['results']:
                self._promote_actor(d)
        else:
            self._promote_actor(d)
        return r

def read ( self , * args , * * kwargs ) : if 'actor' in kwargs : kwargs [ 'actor' ] = kwargs . pop ( 'actor' ) r = super ( Resource , self ) . read ( * args , * * kwargs ) if 'results' in r : for d in r [ 'results' ] : self . _promote_actor ( d ) else : self . _promote_actor ( d ) return r

Do extra processing so we can display the actor field as a top - level field


def log(s, header='', file=sys.stderr, nl=1, **kwargs):
    """Log the given output to stderr if and only if we are in
    verbose mode.

    If we are not in verbose mode, this is a no-op.
    """
    # Sanity check: If we are not in verbose mode, this is a no-op.
    if not settings.verbose:
        return

    # Construct multi-line string to stderr if header is provided.
    if header:
        word_arr = s.split(' ')
        multi = []
        word_arr.insert(0, '%s:' % header.upper())
        i = 0
        while i < len(word_arr):
            to_add = ['***']
            count = 3
            while count <= 79:
                count += len(word_arr[i]) + 1
                if count <= 79:
                    to_add.append(word_arr[i])
                    i += 1
                    if i == len(word_arr):
                        break
            # Handle corner case of extra-long word longer than 75 characters.
            if len(to_add) == 1:
                to_add.append(word_arr[i])
                i += 1
            if i != len(word_arr):
                count -= len(word_arr[i]) + 1
            to_add.append('*' * (78 - count))
            multi.append(' '.join(to_add))
        s = '\n'.join(multi)
        lines = len(multi)
    else:
        lines = 1

    # If `nl` is an int greater than the number of rows of a message,
    # add the appropriate newlines to the output.
    if isinstance(nl, int) and nl > lines:
        s += '\n' * (nl - lines)

    # Output to stderr.
    return secho(s, file=file, **kwargs)

Log the given output to stderr if and only if we are in verbose mode .


def configure_model(self, attrs, field_name):
        '''
        Hook for ResourceMeta class to call when initializing model class.
        Saves fields obtained from resource class backlinks
        '''
        self.relationship = field_name
        self._set_method_names(relationship=field_name)
        if self.res_name is None:
            self.res_name = grammar.singularize(attrs.get('endpoint', 'unknown').strip('/'))

def configure_model ( self , attrs , field_name ) : self . relationship = field_name self . _set_method_names ( relationship = field_name ) if self . res_name is None : self . res_name = grammar . singularize ( attrs . get ( 'endpoint' , 'unknown' ) . strip ( '/' ) )

Hook for ResourceMeta class to call when initializing model class . Saves fields obtained from resource class backlinks