def userselect_increment(block):
    """Let user interactively select byte increment."""
    print("Selected block:")
    print('\n    ' + ('\n    '.join(block['lines'])))
    print()

    increment = None
    while increment is None:
        increment = input("Choose store pointer increment (number of bytes): ")
        try:
            increment = int(increment)
        except ValueError:
            increment = None

    block['pointer_increment'] = increment
    return increment

Let user interactively select byte increment .


def userselect_block(blocks, default=None, debug=False):
    """Let user interactively select block."""
    print("Blocks found in assembly file:")
    print("      block     | OPs | pck. | AVX || Registers |    ZMM   |    YMM   |    XMM   |    GP   ||ptr.inc|\n"
          "----------------+-----+------+-----++-----------+----------+----------+----------+---------++-------|")
    for idx, b in blocks:
        print('{:>2} {b[labels]!r:>12} | {b[ops]:>3} | {b[packed_instr]:>4} | {b[avx_instr]:>3} |'
              '| {b[regs][0]:>3} ({b[regs][1]:>3}) | {b[ZMM][0]:>3} ({b[ZMM][1]:>2}) | '
              '{b[YMM][0]:>3} ({b[YMM][1]:>2}) | '
              '{b[XMM][0]:>3} ({b[XMM][1]:>2}) | {b[GP][0]:>2} ({b[GP][1]:>2}) || '
              '{b[pointer_increment]!s:>5} |'.format(idx, b=b))

        if debug:
            ln = b['first_line']
            print(' '*4 + 'Code:')
            for l in b['lines']:
                print(' '*8 + '{:>5} | {}'.format(ln, l))
                ln += 1
            print(' '*4 + 'Metadata:')
            print(textwrap.indent(
                pformat({k: v for k,v in b.items() if k not in ['lines']}),
                ' '*8))

    # Let user select block:
    block_idx = -1
    while not (0 <= block_idx < len(blocks)):
        block_idx = input("Choose block to be marked [" + str(default) + "]: ") or default
        try:
            block_idx = int(block_idx)
        except ValueError:
            block_idx = -1
    # block = blocks[block_idx][1]

    return block_idx


def insert_markers(asm_lines, start_line, end_line):
    """Insert IACA marker into list of ASM instructions at given indices."""
    asm_lines = (asm_lines[:start_line] + START_MARKER +
                 asm_lines[start_line:end_line + 1] + END_MARKER +
                 asm_lines[end_line + 1:])
    return asm_lines

Insert IACA marker into list of ASM instructions at given indices .


def iaca_instrumentation(input_file, output_file,
                         block_selection='auto',
                         pointer_increment='auto_with_manual_fallback',
                         debug=False):
    """
    Add IACA markers to an assembly file.

    If instrumentation fails because loop increment could not determined automatically, a ValueError
    is raised.

    :param input_file: file-like object to read from
    :param output_file: file-like object to write to
    :param block_selection: index of the assembly block to instrument, or 'auto' for automatically
                            using block with the
                            most vector instructions, or 'manual' to read index to prompt user
    :param pointer_increment: number of bytes the pointer is incremented after the loop or
                              - 'auto': automatic detection, otherwise RuntimeError is raised
                              - 'auto_with_manual_fallback': like auto with fallback to manual input
                              - 'manual': prompt user
    :param debug: output additional internal analysis information. Only works with manual selection.
    :return: the instrumented assembly block
    """
    assembly_orig = input_file.readlines()

    # If input and output files are the same, overwrite with output
    if input_file is output_file:
        output_file.seek(0)
        output_file.truncate()

    if debug:
        block_selection = 'manual'

    assembly = strip_and_uncomment(copy(assembly_orig))
    assembly = strip_unreferenced_labels(assembly)
    blocks = find_asm_blocks(assembly)
    if block_selection == 'auto':
        block_idx = select_best_block(blocks)
    elif block_selection == 'manual':
        block_idx = userselect_block(blocks, default=select_best_block(blocks), debug=debug)
    elif isinstance(block_selection, int):
        block_idx = block_selection
    else:
        raise ValueError("block_selection has to be an integer, 'auto' or 'manual' ")

    block = blocks[block_idx][1]

    if pointer_increment == 'auto':
        if block['pointer_increment'] is None:
            raise RuntimeError("pointer_increment could not be detected automatically. Use "
                               "--pointer-increment to set manually to byte offset of store "
                               "pointer address between consecutive assembly block iterations.")
    elif pointer_increment == 'auto_with_manual_fallback':
        if block['pointer_increment'] is None:
            block['pointer_increment'] = userselect_increment(block)
    elif pointer_increment == 'manual':
        block['pointer_increment'] = userselect_increment(block)
    elif isinstance(pointer_increment, int):
        block['pointer_increment'] = pointer_increment
    else:
        raise ValueError("pointer_increment has to be an integer, 'auto', 'manual' or  "
                         "'auto_with_manual_fallback' ")

    instrumented_asm = insert_markers(assembly_orig, block['first_line'], block['last_line'])
    output_file.writelines(instrumented_asm)

    return block


def main():
    """Execute command line interface."""
    parser = argparse.ArgumentParser(
        description='Find and analyze basic loop blocks and mark for IACA.',
        epilog='For help, examples, documentation and bug reports go to:\nhttps://github.com'
               '/RRZE-HPC/kerncraft\nLicense: AGPLv3')
    parser.add_argument('--version', action='version', version='%(prog)s {}'.format(__version__))
    parser.add_argument('source', type=argparse.FileType(), nargs='?', default=sys.stdin,
                        help='assembly file to analyze (default: stdin)')
    parser.add_argument('--outfile', '-o', type=argparse.FileType('w'), nargs='?',
                        default=sys.stdout, help='output file location (default: stdout)')
    parser.add_argument('--debug', action='store_true',
                        help='Output nternal analysis information for debugging.')
    args = parser.parse_args()

    # pointer_increment is given, since it makes no difference on the command lien and requires
    # less user input
    iaca_instrumentation(input_file=args.source, output_file=args.outfile,
                         block_selection='manual', pointer_increment=1, debug=args.debug)


def simulate(kernel, model, define_dict, blocking_constant, blocking_length):
    """Setup and execute model with given blocking length"""
    kernel.clear_state()

    # Add constants from define arguments
    for k, v in define_dict.items():
        kernel.set_constant(k, v)

    kernel.set_constant(blocking_constant, blocking_length)

    model.analyze()
    return sum([cy for dscr, cy in model.results['cycles']])

def
simulate
(
kernel
,
model
,
define_dict
,
blocking_constant
,
blocking_length
)
:
kernel
.
clear_state
(
)
# Add constants from define arguments
for
k
,
v
in
define_dict
.
items
(
)
:
kernel
.
set_constant
(
k
,
v
)
kernel
.
set_constant
(
blocking_constant
,
blocking_length
)
model
.
analyze
(
)
return
sum
(
[
cy
for
dscr
,
cy
in
model
.
results
[
'cycles'
]
]
)


def space(start, stop, num, endpoint=True, log=False, base=10):
    """
    Return list of evenly spaced integers over an interval.

    Numbers can either be evenly distributed in a linear space (if *log* is False) or in a log
    space (if *log* is True). If *log* is True, base is used to define the log space basis.

    If *endpoint* is True, *stop* will be the last retruned value, as long as *num* >= 2.
    """
    assert type(start) is int and type(stop) is int and type(num) is int, \
        "start, stop and num need to be intergers"
    assert num >= 2, "num has to be atleast 2"

    if log:
        start = math.log(start, base)
        stop = math.log(stop, base)

    if endpoint:
        step_length = float((stop - start)) / float(num - 1)
    else:
        step_length = float((stop - start)) / float(num)

    i = 0
    while i < num:
        if log:
            yield int(round(base ** (start + i * step_length)))
        else:
            yield int(round(start + i * step_length))
        i += 1

Return list of evenly spaced integers over an interval .


def get_last_modified_datetime(dir_path=os.path.dirname(__file__)):
    """Return datetime object of latest change in kerncraft module directory."""
    max_mtime = 0
    for root, dirs, files in os.walk(dir_path):
        for f in files:
            p = os.path.join(root, f)
            try:
                max_mtime = max(max_mtime, os.stat(p).st_mtime)
            except FileNotFoundError:
                pass
    return datetime.utcfromtimestamp(max_mtime)

def get_last_modified_datetime ( dir_path = os . path . dirname ( __file__ ) ) : max_mtime = 0 for root , dirs , files in os . walk ( dir_path ) : for f in files : p = os . path . join ( root , f ) try : max_mtime = max ( max_mtime , os . stat ( p ) . st_mtime ) except FileNotFoundError : pass return datetime . utcfromtimestamp ( max_mtime )

Return datetime object of latest change in kerncraft module directory .


def check_arguments(args, parser):
    """Check arguments passed by user that are not checked by argparse itself."""
    if args.asm_block not in ['auto', 'manual']:
        try:
            args.asm_block = int(args.asm_block)
        except ValueError:
            parser.error('--asm-block can only be "auto", "manual" or an integer')

    # Set default unit depending on performance model requested
    if not args.unit:
        if 'Roofline' in args.pmodel or 'RooflineIACA' in args.pmodel:
            args.unit = 'FLOP/s'
        else:
            args.unit = 'cy/CL'

Check arguments passed by user that are not checked by argparse itself .


def main():
    """Initialize and run command line interface."""
    # Create and populate parser
    parser = create_parser()

    # Parse given arguments
    args = parser.parse_args()

    # Checking arguments
    check_arguments(args, parser)

    # BUSINESS LOGIC IS FOLLOWING
    run(parser, args)

def
main
(
)
:
# Create and populate parser
parser
=
create_parser
(
)
# Parse given arguments
args
=
parser
.
parse_args
(
)
# Checking arguments
check_arguments
(
args
,
parser
)
# BUSINESS LOGIC IS FOLLOWING
run
(
parser
,
args
)

Initialize and run command line interface .


def main():
    """Comand line interface of picklemerge."""
    parser = argparse.ArgumentParser(
        description='Recursively merges two or more pickle files. Only supports pickles consisting '
        'of a single dictionary object.')
    parser.add_argument('destination', type=argparse.FileType('r+b'),
                        help='File to write to and include in resulting pickle. (WILL BE CHANGED)')
    parser.add_argument('source', type=argparse.FileType('rb'), nargs='+',
                        help='File to include in resulting pickle.')

    args = parser.parse_args()

    result = pickle.load(args.destination)
    assert isinstance(result, collections.Mapping), "only Mapping types can be handled."

    for s in args.source:
        data = pickle.load(s)
        assert isinstance(data, collections.Mapping), "only Mapping types can be handled."

        update(result, data)

    args.destination.seek(0)
    args.destination.truncate()
    pickle.dump(result, args.destination)


def symbol_pos_int(*args, **kwargs):
    """Create a sympy.Symbol with positive and integer assumptions."""
    kwargs.update({'positive': True,
                   'integer': True})
    return sympy.Symbol(*args, **kwargs)

def symbol_pos_int ( * args , * * kwargs ) : kwargs . update ( { 'positive' : True , 'integer' : True } ) return sympy . Symbol ( * args , * * kwargs )

Create a sympy . Symbol with positive and integer assumptions .


def transform_multidim_to_1d_decl(decl):
    """
    Transform ast of multidimensional declaration to a single dimension declaration.

    In-place operation!

    Returns name and dimensions of array (to be used with transform_multidim_to_1d_ref())
    """
    dims = []
    type_ = decl.type
    while type(type_) is c_ast.ArrayDecl:
        dims.append(type_.dim)
        type_ = type_.type

    if dims:
        # Multidimensional array
        decl.type.dim = reduce(lambda l, r: c_ast.BinaryOp('*', l, r), dims)
        decl.type.type = type_

    return decl.name, dims

def
transform_multidim_to_1d_decl
(
decl
)
:
dims
=
[
]
type_
=
decl
.
type
while
type
(
type_
)
is
c_ast
.
ArrayDecl
:
dims
.
append
(
type_
.
dim
)
type_
=
type_
.
type
if
dims
:
# Multidimensional array
decl
.
type
.
dim
=
reduce
(
lambda
l
,
r
:
c_ast
.
BinaryOp
(
'*'
,
l
,
r
)
,
dims
)
decl
.
type
.
type
=
type_
return
decl
.
name
,
dims

Transform ast of multidimensional declaration to a single dimension declaration .


def transform_multidim_to_1d_ref(aref, dimension_dict):
    """
    Transform ast of multidimensional reference to a single dimension reference.

    In-place operation!
    """
    dims = []
    name = aref
    while type(name) is c_ast.ArrayRef:
        dims.append(name.subscript)
        name = name.name

    subscript_list = []
    for i, d in enumerate(dims):
        if i == 0:
            subscript_list.append(d)
        else:
            subscript_list.append(c_ast.BinaryOp('*', d, reduce(
                lambda l, r: c_ast.BinaryOp('*', l, r),
                dimension_dict[name.name][-1:-i-1:-1])))

    aref.subscript = reduce(
        lambda l, r: c_ast.BinaryOp('+', l, r), subscript_list)
    aref.name = name

def transform_multidim_to_1d_ref ( aref , dimension_dict ) : dims = [ ] name = aref while type ( name ) is c_ast . ArrayRef : dims . append ( name . subscript ) name = name . name subscript_list = [ ] for i , d in enumerate ( dims ) : if i == 0 : subscript_list . append ( d ) else : subscript_list . append ( c_ast . BinaryOp ( '*' , d , reduce ( lambda l , r : c_ast . BinaryOp ( '*' , l , r ) , dimension_dict [ name . name ] [ - 1 : - i - 1 : - 1 ] ) ) ) aref . subscript = reduce ( lambda l , r : c_ast . BinaryOp ( '+' , l , r ) , subscript_list ) aref . name = name

Transform ast of multidimensional reference to a single dimension reference .


def find_node_type(ast, node_type):
    """Return list of array references in AST."""
    if type(ast) is node_type:
        return [ast]
    elif type(ast) is list:
        return reduce(operator.add, list(map(lambda a: find_node_type(a, node_type), ast)), [])
    elif ast is None:
        return []
    else:
        return reduce(operator.add,
                      [find_node_type(o[1], node_type) for o in ast.children()], [])

def find_node_type ( ast , node_type ) : if type ( ast ) is node_type : return [ ast ] elif type ( ast ) is list : return reduce ( operator . add , list ( map ( lambda a : find_node_type ( a , node_type ) , ast ) ) , [ ] ) elif ast is None : return [ ] else : return reduce ( operator . add , [ find_node_type ( o [ 1 ] , node_type ) for o in ast . children ( ) ] , [ ] )

Return list of array references in AST .


def force_iterable(f):
    """Will make any functions return an iterable objects by wrapping its result in a list."""
    def wrapper(*args, **kwargs):
        r = f(*args, **kwargs)
        if hasattr(r, '__iter__'):
            return r
        else:
            return [r]
    return wrapper

def force_iterable ( f ) : def wrapper ( * args , * * kwargs ) : r = f ( * args , * * kwargs ) if hasattr ( r , '__iter__' ) : return r else : return [ r ] return wrapper

Will make any functions return an iterable objects by wrapping its result in a list .


def check(self):
        """Check that information about kernel makes sens and is valid."""
        datatypes = [v[0] for v in self.variables.values()]
        assert len(set(datatypes)) <= 1, 'mixing of datatypes within a kernel is not supported.'

Check that information about kernel makes sens and is valid .


def set_constant(self, name, value):
        """
        Set constant of name to value.

        :param name: may be a str or a sympy.Symbol
        :param value: must be an int
        """
        assert isinstance(name, str) or isinstance(name, sympy.Symbol), \
            "constant name needs to be of type str, unicode or a sympy.Symbol"
        assert type(value) is int, "constant value needs to be of type int"
        if isinstance(name, sympy.Symbol):
            self.constants[name] = value
        else:
            self.constants[symbol_pos_int(name)] = value


def subs_consts(self, expr):
        """Substitute constants in expression unless it is already a number."""
        if isinstance(expr, numbers.Number):
            return expr
        else:
            return expr.subs(self.constants)

def subs_consts ( self , expr ) : if isinstance ( expr , numbers . Number ) : return expr else : return expr . subs ( self . constants )

Substitute constants in expression unless it is already a number .


def array_sizes(self, in_bytes=False, subs_consts=False):
        """
        Return a dictionary with all arrays sizes.

        :param in_bytes: If True, output will be in bytes, not element counts.
        :param subs_consts: If True, output will be numbers and not symbolic.

        Scalar variables are ignored.
        """
        var_sizes = {}

        for var_name, var_info in self.variables.items():
            var_type, var_size = var_info

            # Skiping sclars
            if var_size is None:
                continue

            var_sizes[var_name] = reduce(operator.mul, var_size, 1)

            # Multiply by bytes per element if requested
            if in_bytes:
                element_size = self.datatypes_size[var_type]
                var_sizes[var_name] *= element_size

        if subs_consts:
            return {k: self.subs_consts(v) for k, v in var_sizes.items()}
        else:
            return var_sizes

Return a dictionary with all arrays sizes .


def _calculate_relative_offset(self, name, access_dimensions):
        """
        Return the offset from the iteration center in number of elements.

        The order of indices used in access is preserved.
        """
        # TODO to be replaced with compile_global_offsets
        offset = 0
        base_dims = self.variables[name][1]

        for dim, offset_info in enumerate(access_dimensions):
            offset_type, idx_name, dim_offset = offset_info
            assert offset_type == 'rel', 'Only relative access to arrays is supported at the moment'

            if offset_type == 'rel':
                offset += self.subs_consts(
                    dim_offset*reduce(operator.mul, base_dims[dim+1:], sympy.Integer(1)))
            else:
                # should not happen
                pass

        return offset

Return the offset from the iteration center in number of elements .


def _remove_duplicate_accesses(self):
        """
        Remove duplicate source and destination accesses
        """
        self.destinations = {var_name: set(acs) for var_name, acs in self.destinations.items()}
        self.sources = {var_name: set(acs) for var_name, acs in self.sources.items()}

def _remove_duplicate_accesses ( self ) : self . destinations = { var_name : set ( acs ) for var_name , acs in self . destinations . items ( ) } self . sources = { var_name : set ( acs ) for var_name , acs in self . sources . items ( ) }


def iteration_length(self, dimension=None):
        """
        Return the number of global loop iterations that are performed.

        If dimension is not None, it is the loop dimension that is returned
        (-1 is the inner most loop and 0 the outermost)
        """
        total_length = 1

        if dimension is not None:
            loops = [self._loop_stack[dimension]]
        else:
            loops = reversed(self._loop_stack)

        for var_name, start, end, incr in loops:
            # This unspools the iterations:
            length = end-start
            total_length = total_length*length
        return self.subs_consts(total_length)

def
iteration_length
(
self
,
dimension
=
)
:
total_length
=
1
if
dimension
is
not
:
loops
=
[
self
.
_loop_stack
[
dimension
]
]
else
:
loops
=
reversed
(
self
.
_loop_stack
)
for
var_name
,
start
,
end
,
incr
in
loops
:
# This unspools the iterations:
length
=
end
start
total_length
=
total_length
*
length
return
self
.
subs_consts
(
total_length
)

Return the number of global loop iterations that are performed .


def get_loop_stack(self, subs_consts=False):
        """Yield loop stack dictionaries in order from outer to inner."""
        for l in self._loop_stack:
            if subs_consts:
                yield {'index': l[0],
                       'start': self.subs_consts(l[1]),
                       'stop': self.subs_consts(l[2]),
                       'increment': self.subs_consts(l[3])}
            else:
                yield {'index': l[0], 'start': l[1], 'stop': l[2], 'increment': l[3]}

def get_loop_stack ( self , subs_consts = False ) : for l in self . _loop_stack : if subs_consts : yield { 'index' : l [ 0 ] , 'start' : self . subs_consts ( l [ 1 ] ) , 'stop' : self . subs_consts ( l [ 2 ] ) , 'increment' : self . subs_consts ( l [ 3 ] ) } else : yield { 'index' : l [ 0 ] , 'start' : l [ 1 ] , 'stop' : l [ 2 ] , 'increment' : l [ 3 ] }

Yield loop stack dictionaries in order from outer to inner .


def index_order(self, sources=True, destinations=True):
        """
        Return the order of indices as they appear in array references.

        Use *source* and *destination* to filter output
        """
        if sources:
            arefs = chain(*self.sources.values())
        else:
            arefs = []
        if destinations:
            arefs = chain(arefs, *self.destinations.values())

        ret = []
        for a in [aref for aref in arefs if aref is not None]:
            ref = []
            for expr in a:
                ref.append(expr.free_symbols)
            ret.append(ref)

        return ret

def index_order ( self , sources = True , destinations = True ) : if sources : arefs = chain ( * self . sources . values ( ) ) else : arefs = [ ] if destinations : arefs = chain ( arefs , * self . destinations . values ( ) ) ret = [ ] for a in [ aref for aref in arefs if aref is not None ] : ref = [ ] for expr in a : ref . append ( expr . free_symbols ) ret . append ( ref ) return ret

Return the order of indices as they appear in array references .


def compile_sympy_accesses(self, sources=True, destinations=True):
        """
        Return a dictionary of lists of sympy accesses, for each variable.

        Use *source* and *destination* to filter output
        """
        sympy_accesses = defaultdict(list)
        # Compile sympy accesses
        for var_name in self.variables:
            if sources:
                for r in self.sources.get(var_name, []):
                    if r is None:
                        continue
                    sympy_accesses[var_name].append(self.access_to_sympy(var_name, r))
            if destinations:
                for w in self.destinations.get(var_name, []):
                    if w is None:
                        continue
                    sympy_accesses[var_name].append(self.access_to_sympy(var_name, w))

        return sympy_accesses

def
compile_sympy_accesses
(
self
,
sources
=
True
,
destinations
=
True
)
:
sympy_accesses
=
defaultdict
(
list
)
# Compile sympy accesses
for
var_name
in
self
.
variables
:
if
sources
:
for
r
in
self
.
sources
.
get
(
var_name
,
[
]
)
:
if
r
is
:
continue
sympy_accesses
[
var_name
]
.
append
(
self
.
access_to_sympy
(
var_name
,
r
)
)
if
destinations
:
for
w
in
self
.
destinations
.
get
(
var_name
,
[
]
)
:
if
w
is
:
continue
sympy_accesses
[
var_name
]
.
append
(
self
.
access_to_sympy
(
var_name
,
w
)
)
return
sympy_accesses

Return a dictionary of lists of sympy accesses for each variable .


def compile_relative_distances(self, sympy_accesses=None):
        """
        Return load and store distances between accesses.

        :param sympy_accesses: optionally restrict accesses, default from compile_sympy_accesses()

        e.g. if accesses are to [+N, +1, -1, -N], relative distances are [N-1, 2, N-1]

        returned is a dict of list of sympy expressions, for each variable
        """
        if sympy_accesses is None:
            sympy_accesses = self.compile_sympy_accesses()

        sympy_distances = defaultdict(list)
        for var_name, accesses in sympy_accesses.items():
            for i in range(1, len(accesses)):
                sympy_distances[var_name].append((accesses[i-1]-accesses[i]).simplify())

        return sympy_distances

def compile_relative_distances ( self , sympy_accesses = None ) : if sympy_accesses is None : sympy_accesses = self . compile_sympy_accesses ( ) sympy_distances = defaultdict ( list ) for var_name , accesses in sympy_accesses . items ( ) : for i in range ( 1 , len ( accesses ) ) : sympy_distances [ var_name ] . append ( ( accesses [ i - 1 ] - accesses [ i ] ) . simplify ( ) ) return sympy_distances

Return load and store distances between accesses .


def global_iterator_to_indices(self, git=None):
        """
        Return sympy expressions translating global_iterator to loop indices.

        If global_iterator is given, an integer is returned
        """
        # unwind global iteration count into loop counters:
        base_loop_counters = {}
        global_iterator = symbol_pos_int('global_iterator')
        idiv = implemented_function(sympy.Function(str('idiv')), lambda x, y: x//y)
        total_length = 1
        last_incr = 1
        for var_name, start, end, incr in reversed(self._loop_stack):
            loop_var = symbol_pos_int(var_name)

            # This unspools the iterations:
            length = end-start  # FIXME is incr handled correct here?
            counter = start+(idiv(global_iterator*last_incr, total_length)*incr) % length
            total_length = total_length*length
            last_incr = incr

            base_loop_counters[loop_var] = sympy.lambdify(
                global_iterator,
                self.subs_consts(counter), modules=[numpy, {'Mod': numpy.mod}])

            if git is not None:
                try:  # Try to resolve to integer if global_iterator was given
                    base_loop_counters[loop_var] = sympy.Integer(self.subs_consts(counter))
                    continue
                except (ValueError, TypeError):
                    base_loop_counters[loop_var] = base_loop_counters[loop_var](git)

        return base_loop_counters

Return sympy expressions translating global_iterator to loop indices .


def global_iterator(self):
        """
        Return global iterator sympy expression
        """
        global_iterator = sympy.Integer(0)
        total_length = sympy.Integer(1)
        for var_name, start, end, incr in reversed(self._loop_stack):
            loop_var = symbol_pos_int(var_name)
            length = end - start  # FIXME is incr handled correct here?
            global_iterator += (loop_var - start) * total_length
            total_length *= length
        return global_iterator

def
global_iterator
(
self
)
:
global_iterator
=
sympy
.
Integer
(
0
)
total_length
=
sympy
.
Integer
(
1
)
for
var_name
,
start
,
end
,
incr
in
reversed
(
self
.
_loop_stack
)
:
loop_var
=
symbol_pos_int
(
var_name
)
length
=
end
start
# FIXME is incr handled correct here?
global_iterator
+=
(
loop_var
start
)
*
total_length
total_length
*=
length
return
global_iterator


def indices_to_global_iterator(self, indices):
        """
        Transform a dictionary of indices to a global iterator integer.

        Inverse of global_iterator_to_indices().
        """
        global_iterator = self.subs_consts(self.global_iterator().subs(indices))
        return global_iterator

def indices_to_global_iterator ( self , indices ) : global_iterator = self . subs_consts ( self . global_iterator ( ) . subs ( indices ) ) return global_iterator

Transform a dictionary of indices to a global iterator integer .


def max_global_iteration(self):
        """Return global iterator with last iteration number"""
        return self.indices_to_global_iterator({
            symbol_pos_int(var_name): end-1 for var_name, start, end, incr in self._loop_stack
        })

def max_global_iteration ( self ) : return self . indices_to_global_iterator ( { symbol_pos_int ( var_name ) : end - 1 for var_name , start , end , incr in self . _loop_stack } )


def print_kernel_info(self, output_file=sys.stdout):
        """Print kernel information in human readble format."""
        table = ('     idx |        min        max       step\n' +
                 '---------+---------------------------------\n')
        for l in self._loop_stack:
            table += '{:>8} | {!r:>10} {!r:>10} {!r:>10}\n'.format(*l)
        print(prefix_indent('loop stack:        ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self.sources.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join(['{!r:}'.format(o) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data sources:      ', table), file=output_file)

        table = ('    name |  offsets   ...\n' +
                 '---------+------------...\n')
        for name, offsets in list(self.destinations.items()):
            prefix = '{:>8} | '.format(name)
            right_side = '\n'.join(['{!r:}'.format(o) for o in offsets])
            table += prefix_indent(prefix, right_side, later_prefix='         | ')
        print(prefix_indent('data destinations: ', table), file=output_file)

        table = (' op | count \n' +
                 '----+-------\n')
        for op, count in list(self._flops.items()):
            table += '{:>3} | {:>4}\n'.format(op, count)
        table += '     =======\n'
        table += '      {:>4}'.format(sum(self._flops.values()))
        print(prefix_indent('FLOPs:     ', table), file=output_file)

'     idx |        min        max       step\n'

Print kernel information in human readble format .


def print_variables_info(self, output_file=sys.stdout):
        """Print variables information in human readble format."""
        table = ('    name |   type size             \n' +
                 '---------+-------------------------\n')
        for name, var_info in list(self.variables.items()):
            table += '{:>8} | {:>6} {!s:<10}\n'.format(name, var_info[0], var_info[1])
        print(prefix_indent('variables: ', table), file=output_file)

def
print_variables_info
(
self
,
output_file
=
sys
.
stdout
)
:
table
=
(
'    name |   type size             \n'
+
'---------+-------------------------\n'
)
for
name
,
var_info
in
list
(
self
.
variables
.
items
(
)
)
:
table
+=
'{:>8} | {:>6} {!s:<10}\n'
.
format
(
name
,
var_info
[
0
]
,
var_info
[
1
]
)
print
(
prefix_indent
(
'variables: '
,
table
)
,
file
=
output_file
)

Print variables information in human readble format .


def print_constants_info(self, output_file=sys.stdout):
        """Print constants information in human readble format."""
        table = ('    name | value     \n' +
                 '---------+-----------\n')
        for name, value in list(self.constants.items()):
            table += '{!s:>8} | {:<10}\n'.format(name, value)
        print(prefix_indent('constants: ', table), file=output_file)

def
print_constants_info
(
self
,
output_file
=
sys
.
stdout
)
:
table
=
(
'    name | value     \n'
+
'---------+-----------\n'
)
for
name
,
value
in
list
(
self
.
constants
.
items
(
)
)
:
table
+=
'{!s:>8} | {:<10}\n'
.
format
(
name
,
value
)
print
(
prefix_indent
(
'constants: '
,
table
)
,
file
=
output_file
)

Print constants information in human readble format .


def print_kernel_code(self, output_file=sys.stdout):
        """Print source code of kernel."""
        print(self.kernel_code, file=output_file)

def print_kernel_code ( self , output_file = sys . stdout ) : print ( self . kernel_code , file = output_file )


def conv_ast_to_sym(self, math_ast):
        """
        Convert mathematical expressions to a sympy representation.

        May only contain paranthesis, addition, subtraction and multiplication from AST.
        """
        if type(math_ast) is c_ast.ID:
            return symbol_pos_int(math_ast.name)
        elif type(math_ast) is c_ast.Constant:
            return sympy.Integer(math_ast.value)
        else:  # elif type(dim) is c_ast.BinaryOp:
            op = {
                '*': operator.mul,
                '+': operator.add,
                '-': operator.sub
            }

            return op[math_ast.op](
                self.conv_ast_to_sym(math_ast.left),
                self.conv_ast_to_sym(math_ast.right))

def
conv_ast_to_sym
(
self
,
math_ast
)
:
if
type
(
math_ast
)
is
c_ast
.
ID
:
return
symbol_pos_int
(
math_ast
.
name
)
elif
type
(
math_ast
)
is
c_ast
.
Constant
:
return
sympy
.
Integer
(
math_ast
.
value
)
else
:
# elif type(dim) is c_ast.BinaryOp:
op
=
{
'*'
:
operator
.
mul
,
'+'
:
operator
.
add
,
'-'
:
operator
.
sub
}
return
op
[
math_ast
.
op
]
(
self
.
conv_ast_to_sym
(
math_ast
.
left
)
,
self
.
conv_ast_to_sym
(
math_ast
.
right
)
)

Convert mathematical expressions to a sympy representation .


def _get_offsets(self, aref, dim=0):
        """
        Return a tuple of offsets of an ArrayRef object in all dimensions.

        The index order is right to left (c-code order).
        e.g. c[i+1][j-2] -> (-2, +1)

        If aref is actually a c_ast.ID, None will be returned.
        """
        if isinstance(aref, c_ast.ID):
            return None

        # Check for restrictions
        assert type(aref.name) in [c_ast.ArrayRef, c_ast.ID], \
            "array references must only be used with variables or other array references"
        assert type(aref.subscript) in [c_ast.ID, c_ast.Constant, c_ast.BinaryOp], \
            'array subscript must only contain variables or binary operations'

        # Convert subscript to sympy and append
        idxs = [self.conv_ast_to_sym(aref.subscript)]

        # Check for more indices (multi-dimensional access)
        if type(aref.name) is c_ast.ArrayRef:
            idxs += self._get_offsets(aref.name, dim=dim+1)

        # Reverse to preserver order (the subscripts in the AST are traversed backwards)
        if dim == 0:
            idxs.reverse()

        return tuple(idxs)

Return a tuple of offsets of an ArrayRef object in all dimensions .


def _get_basename(cls, aref):
        """
        Return base name of ArrayRef object.

        e.g. c[i+1][j-2] -> 'c'
        """
        if isinstance(aref.name, c_ast.ArrayRef):
            return cls._get_basename(aref.name)
        elif isinstance(aref.name, str):
            return aref.name
        else:
            return aref.name.name

def _get_basename ( cls , aref ) : if isinstance ( aref . name , c_ast . ArrayRef ) : return cls . _get_basename ( aref . name ) elif isinstance ( aref . name , str ) : return aref . name else : return aref . name . name


def get_index_type(self, loop_nest=None):
        """
        Return index type used in loop nest.

        If index type between loops differ, an exception is raised.
        """
        if loop_nest is None:
            loop_nest = self.get_kernel_loop_nest()
        if type(loop_nest) is c_ast.For:
            loop_nest = [loop_nest]
        index_types = (None, None)
        for s in loop_nest:
            if type(s) is c_ast.For:
                if type(s.stmt) in [c_ast.For, c_ast.Compound]:
                    other = self.get_index_type(loop_nest=s.stmt)
                else:
                    other = None
                index_types = (s.init.decls[0].type.type.names, other)
                break
        if index_types[0] == index_types[1] or index_types[1] is None:
            return index_types[0]
        else:
            raise ValueError("Loop indices must have same type, found {}.".format(index_types))


def _build_const_declartions(self, with_init=True):
        """
        Generate constants declarations

        :return: list of declarations
        """
        decls = []

        # Use type as provided by user in loop indices
        index_type = self.get_index_type()

        i = 2  # subscript for cli input, 1 is reserved for repeat
        for k in self.constants:
            # const long long N = strtoul(argv[2])
            # with increasing N and 1
            # TODO change subscript of argv depending on constant count
            type_decl = c_ast.TypeDecl(k.name, ['const'], c_ast.IdentifierType(index_type))
            init = None
            if with_init:
                init = c_ast.FuncCall(
                    c_ast.ID('atoi'),
                    c_ast.ExprList([c_ast.ArrayRef(c_ast.ID('argv'),
                                                   c_ast.Constant('int', str(i)))]))
            i += 1
            decls.append(c_ast.Decl(
                k.name, ['const'], [], [],
                type_decl, init, None))

        return decls


def get_array_declarations(self):
        """Return array declarations."""
        return [d for d in self.kernel_ast.block_items
                if type(d) is c_ast.Decl and type(d.type) is c_ast.ArrayDecl]

def get_array_declarations ( self ) : return [ d for d in self . kernel_ast . block_items if type ( d ) is c_ast . Decl and type ( d . type ) is c_ast . ArrayDecl ]


def get_kernel_loop_nest(self):
        """Return kernel loop nest including any preceding pragmas and following swaps."""
        loop_nest = [s for s in self.kernel_ast.block_items
                     if type(s) in [c_ast.For, c_ast.Pragma, c_ast.FuncCall]]
        assert len(loop_nest) >= 1, "Found to few for statements in kernel"
        return loop_nest

def
get_kernel_loop_nest
(
self
)
:
loop_nest
=
[
s
for
s
in
self
.
kernel_ast
.
block_items
if
type
(
s
)
in
[
c_ast
.
For
,
c_ast
.
Pragma
,
c_ast
.
FuncCall
]
]
assert
len
(
loop_nest
)
>=
1
,
"Found to few for statements in kernel"
return
loop_nest

Return kernel loop nest including any preceding pragmas and following swaps .


def _build_array_declarations(self, with_init=True):
        """
        Generate declaration statements for arrays.

        Also transforming multi-dim to 1d arrays and initializing with malloc.

        :param with_init: ommit malloc initialization

        :return: list of declarations nodes, dictionary of array names and original dimensions
        """
        # copy array declarations from from kernel ast
        array_declarations = deepcopy(self.get_array_declarations())
        array_dict = []
        for d in array_declarations:
            # We need to transform
            array_dict.append(transform_multidim_to_1d_decl(d))
            transform_array_decl_to_malloc(d, with_init=with_init)
        return array_declarations, dict(array_dict)

Generate declaration statements for arrays .


def _find_inner_most_loop(self, loop_nest):
        """Return inner most for loop in loop nest"""
        r = None
        for s in loop_nest:
            if type(s) is c_ast.For:
                return self._find_inner_most_loop(s) or s
            else:
                r = r or self._find_inner_most_loop(s)
        return r

def _find_inner_most_loop ( self , loop_nest ) : r = None for s in loop_nest : if type ( s ) is c_ast . For : return self . _find_inner_most_loop ( s ) or s else : r = r or self . _find_inner_most_loop ( s ) return r


def _build_array_initializations(self, array_dimensions):
        """
        Generate initialization statements for arrays.

        :param array_dimensions: dictionary of array dimensions

        :return: list of nodes
        """
        kernel = deepcopy(deepcopy(self.get_kernel_loop_nest()))
        # traverse to the inner most for loop:
        inner_most = self._find_inner_most_loop(kernel)
        orig_inner_stmt = inner_most.stmt
        inner_most.stmt = c_ast.Compound([])

        rand_float_str = str(random.uniform(1.0, 0.1))

        # find all array references in original orig_inner_stmt
        for aref in find_node_type(orig_inner_stmt, c_ast.ArrayRef):
            # transform to 1d references
            transform_multidim_to_1d_ref(aref, array_dimensions)
            # build static assignments and inject into inner_most.stmt
            inner_most.stmt.block_items.append(c_ast.Assignment(
                '=', aref, c_ast.Constant('float', rand_float_str)))

        return kernel

Generate initialization statements for arrays .


def _build_dummy_calls(self):
        """
        Generate false if branch with dummy calls

        Requires kerncraft.h to be included, which defines dummy(...) and var_false.

        :return: dummy statement
        """
        # Make sure nothing gets removed by inserting dummy calls
        dummy_calls = []
        for d in self.kernel_ast.block_items:
            # Only consider toplevel declarations from kernel ast
            if type(d) is not c_ast.Decl: continue
            if type(d.type) is c_ast.ArrayDecl:
                dummy_calls.append(c_ast.FuncCall(
                    c_ast.ID('dummy'),
                    c_ast.ExprList([c_ast.ID(d.name)])))
            else:
                dummy_calls.append(c_ast.FuncCall(
                    c_ast.ID('dummy'),
                    c_ast.ExprList([c_ast.UnaryOp('&', c_ast.ID(d.name))])))
        dummy_stmt = c_ast.If(
            cond=c_ast.ID('var_false'),
            iftrue=c_ast.Compound(dummy_calls),
            iffalse=None)
        return dummy_stmt


def _build_kernel_function_declaration(self, name='kernel'):
        """Build and return kernel function declaration"""
        array_declarations, array_dimensions = self._build_array_declarations(with_init=False)
        scalar_declarations = self._build_scalar_declarations(with_init=False)
        const_declarations = self._build_const_declartions(with_init=False)
        return c_ast.FuncDecl(args=c_ast.ParamList(params=array_declarations + scalar_declarations +
                                                          const_declarations),
                              type=c_ast.TypeDecl(declname=name,
                                                  quals=[],
                                                  type=c_ast.IdentifierType(names=['void'])))

def _build_kernel_function_declaration ( self , name = 'kernel' ) : array_declarations , array_dimensions = self . _build_array_declarations ( with_init = False ) scalar_declarations = self . _build_scalar_declarations ( with_init = False ) const_declarations = self . _build_const_declartions ( with_init = False ) return c_ast . FuncDecl ( args = c_ast . ParamList ( params = array_declarations + scalar_declarations + const_declarations ) , type = c_ast . TypeDecl ( declname = name , quals = [ ] , type = c_ast . IdentifierType ( names = [ 'void' ] ) ) )


def _build_scalar_declarations(self, with_init=True):
        """Build and return scalar variable declarations"""
        # copy scalar declarations from from kernel ast
        scalar_declarations = [deepcopy(d) for d in self.kernel_ast.block_items
                               if type(d) is c_ast.Decl and type(d.type) is c_ast.TypeDecl]
        # add init values to declarations
        if with_init:
            random.seed(2342)  # we want reproducible random numbers
            for d in scalar_declarations:
                if d.type.type.names[0] in ['double', 'float']:
                    d.init = c_ast.Constant('float', str(random.uniform(1.0, 0.1)))
                elif d.type.type.names[0] in ['int', 'long', 'long long',
                                              'unsigned int', 'unsigned long', 'unsigned long long']:
                    d.init = c_ast.Constant('int', 2)

        return scalar_declarations


def get_kernel_code(self, openmp=False, as_filename=False, name='kernel'):
        """
        Generate and return compilable source code with kernel function from AST.

        :param openmp: if true, OpenMP code will be generated
        :param as_filename: if true, will save to file and return filename
        :param name: name of kernel function
        """
        assert self.kernel_ast is not None, "AST does not exist, this could be due to running " \
                                            "based on a kernel description rather than code."
        file_name = 'kernel'
        if openmp:
            file_name += '-omp'
        file_name += '.c'

        fp, already_available = self._get_intermediate_file(
            file_name, machine_and_compiler_dependent=False)

        # Use already cached version
        if already_available:
            code = fp.read()
        else:
            array_declarations, array_dimensions = self._build_array_declarations()

            # Prepare actual kernel loop nest
            if openmp:
                # with OpenMP code
                kernel = deepcopy(self.get_kernel_loop_nest())
                # find all array references in kernel
                for aref in find_node_type(kernel, c_ast.ArrayRef):
                    # transform to 1d references
                    transform_multidim_to_1d_ref(aref, array_dimensions)
                omp_pragmas = [p for p in find_node_type(kernel, c_ast.Pragma)
                               if 'omp' in p.string]
                # TODO if omp parallel was found, remove it (also replace "parallel for" -> "for")
                # if no omp for pragmas are present, insert suitable ones
                if not omp_pragmas:
                    kernel.insert(0, c_ast.Pragma("omp for"))
                # otherwise do not change anything
            else:
                # with original code
                kernel = deepcopy(self.get_kernel_loop_nest())
                # find all array references in kernel
                for aref in find_node_type(kernel, c_ast.ArrayRef):
                    # transform to 1d references
                    transform_multidim_to_1d_ref(aref, array_dimensions)

            function_ast = c_ast.FuncDef(decl=c_ast.Decl(
                name=name, type=self._build_kernel_function_declaration(name=name), quals=[],
                storage=[], funcspec=[], init=None, bitsize=None),
                body=c_ast.Compound(block_items=kernel),
                param_decls=None)

            # Generate code
            code = CGenerator().visit(function_ast)

            # Insert missing #includes from template to top of code
            code = '#include "kerncraft.h"\n\n' + code

            # Store to file
            fp.write(code)
        fp.close()

        if as_filename:
            return fp.name
        else:
            return code

Generate and return compilable source code with kernel function from AST .


def _build_kernel_call(self, name='kernel'):
        """Generate and return kernel call ast."""
        return c_ast.FuncCall(name=c_ast.ID(name=name), args=c_ast.ExprList(exprs=[
            c_ast.ID(name=d.name) for d in (
                    self._build_array_declarations()[0] +
                    self._build_scalar_declarations() +
                    self._build_const_declartions())]))

def _build_kernel_call ( self , name = 'kernel' ) : return c_ast . FuncCall ( name = c_ast . ID ( name = name ) , args = c_ast . ExprList ( exprs = [ c_ast . ID ( name = d . name ) for d in ( self . _build_array_declarations ( ) [ 0 ] + self . _build_scalar_declarations ( ) + self . _build_const_declartions ( ) ) ] ) )


def get_main_code(self, as_filename=False, kernel_function_name='kernel'):
        """
        Generate and return compilable source code from AST.
        """
        # TODO produce nicer code, including help text and other "comfort features".
        assert self.kernel_ast is not None, "AST does not exist, this could be due to running " \
                                            "based on a kernel description rather than code."

        fp, already_available = self._get_intermediate_file('main.c',
                                                            machine_and_compiler_dependent=False)

        # Use already cached version
        if already_available:
            code = fp.read()
        else:
            parser = CParser()
            template_code = self.CODE_TEMPLATE
            template_ast = parser.parse(clean_code(template_code,
                                                   macros=True, comments=True, pragmas=False))
            ast = deepcopy(template_ast)

            # Define and replace DECLARE_CONSTS
            replace_id(ast, "DECLARE_CONSTS", self._build_const_declartions(with_init=True))

            # Define and replace DECLARE_ARRAYS
            array_declarations, array_dimensions = self._build_array_declarations()
            replace_id(ast, "DECLARE_ARRAYS", array_declarations)

            # Define and replace DECLARE_INIT_SCALARS
            replace_id(ast, "DECLARE_INIT_SCALARS", self._build_scalar_declarations())

            # Define and replace DUMMY_CALLS
            replace_id(ast, "DUMMY_CALLS", self._build_dummy_calls())

            # Define and replace KERNEL_DECL
            ast.ext.insert(0, self._build_kernel_function_declaration(
                name=kernel_function_name))

            # Define and replace KERNEL_CALL
            replace_id(ast, "KERNEL_CALL", self._build_kernel_call())

            # Define and replace INIT_ARRAYS based on previously generated kernel
            replace_id(ast, "INIT_ARRAYS", self._build_array_initializations(array_dimensions))

            # Generate code
            code = CGenerator().visit(ast)

            # Insert missing #includes from template to top of code
            code = '\n'.join([l for l in template_code.split('\n') if l.startswith("#include")]) + \
                   '\n\n' + code

            # Store to file
            fp.write(code)
        fp.close()

        if as_filename:
            return fp.name
        else:
            return code

Generate and return compilable source code from AST .


def iaca_analysis(self, micro_architecture, asm_block='auto',
                      pointer_increment='auto_with_manual_fallback', verbose=False):
        """
        Run an IACA analysis and return its outcome.

        *asm_block* controls how the to-be-marked block is chosen. "auto" (default) results in
        the largest block, "manual" results in interactive and a number in the according block.

        *pointer_increment* is the number of bytes the pointer is incremented after the loop or
           - 'auto': automatic detection, RuntimeError is raised in case of failure
           - 'auto_with_manual_fallback': automatic detection, fallback to manual input
           - 'manual': prompt user
        """
        asm_filename = self.compile_kernel(assembly=True, verbose=verbose)
        asm_marked_filename = os.path.splitext(asm_filename)[0]+'-iaca.s'
        with open(asm_filename, 'r') as in_file, open(asm_marked_filename, 'w') as out_file:
            self.asm_block = iaca.iaca_instrumentation(
                in_file, out_file,
                block_selection=asm_block,
                pointer_increment=pointer_increment)
        obj_name = self.assemble_to_object(asm_marked_filename, verbose=verbose)
        return iaca.iaca_analyse_instrumented_binary(obj_name, micro_architecture), self.asm_block

def iaca_analysis ( self , micro_architecture , asm_block = 'auto' , pointer_increment = 'auto_with_manual_fallback' , verbose = False ) : asm_filename = self . compile_kernel ( assembly = True , verbose = verbose ) asm_marked_filename = os . path . splitext ( asm_filename ) [ 0 ] + '-iaca.s' with open ( asm_filename , 'r' ) as in_file , open ( asm_marked_filename , 'w' ) as out_file : self . asm_block = iaca . iaca_instrumentation ( in_file , out_file , block_selection = asm_block , pointer_increment = pointer_increment ) obj_name = self . assemble_to_object ( asm_marked_filename , verbose = verbose ) return iaca . iaca_analyse_instrumented_binary ( obj_name , micro_architecture ) , self . asm_block

Run an IACA analysis and return its outcome .


def build_executable(self, lflags=None, verbose=False, openmp=False):
        """Compile source to executable with likwid capabilities and return the executable name."""
        compiler, compiler_args = self._machine.get_compiler()

        kernel_obj_filename = self.compile_kernel(openmp=openmp, verbose=verbose)
        out_filename, already_exists = self._get_intermediate_file(
            os.path.splitext(os.path.basename(kernel_obj_filename))[0], binary=True, fp=False)

        if not already_exists:
            main_source_filename = self.get_main_code(as_filename=True)

            if not (('LIKWID_INCLUDE' in os.environ or 'LIKWID_INC' in os.environ) and
                    'LIKWID_LIB' in os.environ):
                print('Could not find LIKWID_INCLUDE (e.g., "-I/app/likwid/4.1.2/include") and '
                      'LIKWID_LIB (e.g., "-L/apps/likwid/4.1.2/lib") environment variables',
                      file=sys.stderr)
                sys.exit(1)

            compiler_args += [
                '-std=c99',
                '-I'+reduce_path(os.path.abspath(os.path.dirname(
                    os.path.realpath(__file__)))+'/headers/'),
                os.environ.get('LIKWID_INCLUDE', ''),
                os.environ.get('LIKWID_INC', ''),
                '-llikwid']

            # This is a special case for unittesting
            if os.environ.get('LIKWID_LIB') == '':
                compiler_args = compiler_args[:-1]

            if lflags is None:
                lflags = []
            lflags += os.environ['LIKWID_LIB'].split(' ') + ['-pthread']
            compiler_args += os.environ['LIKWID_LIB'].split(' ') + ['-pthread']

            infiles = [reduce_path(os.path.abspath(os.path.dirname(
                os.path.realpath(__file__)))+'/headers/dummy.c'),
                       kernel_obj_filename, main_source_filename]

            cmd = [compiler] + infiles + compiler_args + ['-o', out_filename]
            # remove empty arguments
            cmd = list(filter(bool, cmd))
            if verbose:
                print('Executing (build_executable): ', ' '.join(cmd))
            try:
                subprocess.check_output(cmd)
            except subprocess.CalledProcessError as e:
                print("Build failed:", e, file=sys.stderr)
                sys.exit(1)
        else:
            if verbose:
                print('Executing (build_executable): ', 'using cached', out_filename)

        return out_filename

Compile source to executable with likwid capabilities and return the executable name .


def string_to_sympy(cls, s):
        """Convert any string to a sympy object or None."""
        if isinstance(s, int):
            return sympy.Integer(s)
        elif isinstance(s, list):
            return tuple([cls.string_to_sympy(e) for e in s])
        elif s is None:
            return None
        else:
            # Step 1 build expression with the whole alphabet redefined:
            local_dict = {c: symbol_pos_int(c) for c in s if c in string.ascii_letters}
            # TODO find nicer solution for N and other pre-mapped letters
            preliminary_expr = parse_expr(s, local_dict=local_dict)
            # Replace all free symbols with positive integer versions:
            local_dict.update(
                {s.name: symbol_pos_int(s.name) for s in preliminary_expr.free_symbols})
            return parse_expr(s, local_dict=local_dict)

Convert any string to a sympy object or None .


def get_identifier(self):
        """Return identifier which is either the machine file name or sha256 checksum of data."""
        if self._path:
            return os.path.basename(self._path)
        else:
            return hashlib.sha256(hashlib.sha256(repr(self._data).encode())).hexdigest()

def get_identifier ( self ) : if self . _path : return os . path . basename ( self . _path ) else : return hashlib . sha256 ( hashlib . sha256 ( repr ( self . _data ) . encode ( ) ) ) . hexdigest ( )

Return identifier which is either the machine file name or sha256 checksum of data .


def get_last_modified_datetime(self):
        """Return datetime object of modified time of machine file. Return now if not a file."""
        if self._path:
            statbuf = os.stat(self._path)
            return datetime.utcfromtimestamp(statbuf.st_mtime)
        else:
            return datetime.now()

def get_last_modified_datetime ( self ) : if self . _path : statbuf = os . stat ( self . _path ) return datetime . utcfromtimestamp ( statbuf . st_mtime ) else : return datetime . now ( )

Return datetime object of modified time of machine file . Return now if not a file .


def get_cachesim(self, cores=1):
        """
        Return a cachesim.CacheSimulator object based on the machine description.

        :param cores: core count (default: 1)
        """
        cache_dict = {}
        for c in self['memory hierarchy']:
            # Skip main memory
            if 'cache per group' not in c:
                continue
            cache_dict[c['level']] = deepcopy(c['cache per group'])
            # Scale size of shared caches according to cores
            if c['cores per group'] > 1:
                cache_dict[c['level']]['sets'] //= cores

        cs, caches, mem = cachesim.CacheSimulator.from_dict(cache_dict)

        return cs

Return a cachesim . CacheSimulator object based on the machine description .


def get_bandwidth(self, cache_level, read_streams, write_streams, threads_per_core, cores=None):
        """
        Return best fitting bandwidth according to number of threads, read and write streams.

        :param cache_level: integer of cache (0 is L1, 1 is L2 ...)
        :param read_streams: number of read streams expected
        :param write_streams: number of write streams expected
        :param threads_per_core: number of threads that are run on each core
        :param cores: if not given, will choose maximum bandwidth for single NUMA domain
        """
        # try to find best fitting kernel (closest to read/write ratio):
        # write allocate has to be handled in kernel information (all writes are also reads)
        # TODO support for non-write-allocate architectures
        try:
            target_ratio = read_streams/write_streams
        except ZeroDivisionError:
            target_ratio = float('inf')
        measurement_kernel = 'load'
        measurement_kernel_info = self['benchmarks']['kernels'][measurement_kernel]
        measurement_kernel_ratio = float('inf')
        for kernel_name, kernel_info in sorted(self['benchmarks']['kernels'].items()):
            try:
                kernel_ratio = ((kernel_info['read streams']['streams'] +
                                 kernel_info['write streams']['streams'] -
                                 kernel_info['read+write streams']['streams']) /
                                kernel_info['write streams']['streams'])
            except ZeroDivisionError:
                kernel_ratio = float('inf')

            if abs(kernel_ratio - target_ratio) < abs(measurement_kernel_ratio - target_ratio):
                measurement_kernel = kernel_name
                measurement_kernel_info = kernel_info
                measurement_kernel_ratio = kernel_ratio

        # choose smt, and then use max/saturation bw
        bw_level = self['memory hierarchy'][cache_level]['level']
        bw_measurements = \
            self['benchmarks']['measurements'][bw_level][threads_per_core]
        assert threads_per_core == bw_measurements['threads per core'], \
            'malformed measurement dictionary in machine file.'
        if cores is not None:
            # Used by Roofline model
            run_index = bw_measurements['cores'].index(cores)
            bw = bw_measurements['results'][measurement_kernel][run_index]
        else:
            # Used by ECM model
            # Choose maximum number of cores to get bandwidth for
            max_cores = min(self['memory hierarchy'][cache_level]['cores per group'],
                            self['cores per NUMA domain'])
            bw = max(bw_measurements['results'][measurement_kernel][:max_cores])

        # Correct bandwidth due to miss-measurement of write allocation
        # TODO support non-temporal stores and non-write-allocate architectures
        if cache_level == 0:
            # L1 does not have write-allocate, so everything is measured correctly
            factor = 1.0
        else:
            factor = (float(measurement_kernel_info['read streams']['bytes']) +
                      2.0*float(measurement_kernel_info['write streams']['bytes']) -
                      float(measurement_kernel_info['read+write streams']['bytes'])) / \
                     (float(measurement_kernel_info['read streams']['bytes']) +
                      float(measurement_kernel_info['write streams']['bytes']))
        bw = bw * factor

        return bw, measurement_kernel

Return best fitting bandwidth according to number of threads read and write streams .


def get_compiler(self, compiler=None, flags=None):
        """
        Return tuple of compiler and compiler flags.

        Selects compiler and flags from machine description file, commandline arguments or call
        arguements.
        """
        if self._args:
            compiler = compiler or self._args.compiler
            flags = flags or self._args.compiler_flags
        if compiler is None:
            # Select first available compiler in machine description file's compiler dict
            for c in self['compiler'].keys():
                # Making sure compiler is available:
                if find_executable(c) is not None:
                    compiler = c
                    break
            else:
                raise RuntimeError("No compiler ({}) was found. Add different one in machine file, "
                                   "via --compiler argument or make sure it will be found in "
                                   "$PATH.".format(list(self['compiler'].keys())))
        if flags is None:
            # Select from machine description file
            flags = self['compiler'].get(compiler, '')

        return compiler, flags.split(' ')

Return tuple of compiler and compiler flags .


def parse_perfctr_event(perfctr):
        """
        Parse events in machine description to tuple representation used in Benchmark module.

        Examples:
        >>> parse_perfctr_event('PERF_EVENT:REG[0-3]')
        ('PERF_EVENT', 'REG[0-3]')
        >>> parse_perfctr_event('PERF_EVENT:REG[0-3]:STAY:FOO=23:BAR=0x23')
        ('PERF_EVENT', 'REG[0-3]', {'STAY': None, 'FOO': 23, 'BAR': 35})

        """
        split_perfctr = perfctr.split(':')
        assert len(split_perfctr) >= 2, "Atleast one colon (:) is required in the event name"
        event_tuple = split_perfctr[:2]
        parameters = {}
        for p in split_perfctr[2:]:
            if '=' in p:
                k, v = p.split('=')
                if v.startswith('0x'):
                    parameters[k] = int(v, 16)
                else:
                    parameters[k] = int(v)
            else:
                parameters[p] = None
        event_tuple.append(parameters)
        return tuple(event_tuple)

Parse events in machine description to tuple representation used in Benchmark module .


def _enforce_no_overlap(self, start_at=0):
        """Enforce that no ranges overlap in internal storage."""
        i = start_at
        while i+1 < len(self.data):
            if self.data[i][1] >= self.data[i+1][0]:
                # beginning of i+1-th range is contained in i-th range
                if self.data[i][1] < self.data[i+1][1]:
                    # i+1-th range is longer, thus enlarge i-th range
                    self.data[i][1] = self.data[i+1][1]

                # removed contained range
                del self.data[i+1]
            i += 1

Enforce that no ranges overlap in internal storage .


def get_header_path() -> str:
    """Return local folder path of header files."""
    import os
    return os.path.abspath(os.path.dirname(os.path.realpath(__file__))) + '/headers/'

def get_header_path ( ) -> str : import os return os . path . abspath ( os . path . dirname ( os . path . realpath ( __file__ ) ) ) + '/headers/'

Return local folder path of header files .


def _align_iteration_with_cl_boundary(self, iteration, subtract=True):
        """Align iteration with cacheline boundary."""
        # FIXME handle multiple datatypes
        element_size = self.kernel.datatypes_size[self.kernel.datatype]
        cacheline_size = self.machine['cacheline size']
        elements_per_cacheline = int(cacheline_size // element_size)

        # Gathering some loop information:
        inner_loop = list(self.kernel.get_loop_stack(subs_consts=True))[-1]
        inner_increment = inner_loop['increment']

        # do this by aligning either writes (preferred) or reads
        # Assumption: writes (and reads) increase linearly
        o = self.kernel.compile_global_offsets(iteration=iteration)[0]
        if len(o[1]):
            # we have a write to work with:
            first_offset = min(o[1])
        else:
            # we use reads
            first_offset = min(o[0])

        diff = first_offset - \
               (int(first_offset) >> self.csim.first_level.cl_bits << self.csim.first_level.cl_bits)
        if diff == 0:
            return iteration
        elif subtract:
            return iteration - (diff // element_size) // inner_increment
        else:
            return iteration + (elements_per_cacheline - diff // element_size) // inner_increment

Align iteration with cacheline boundary .


def get_loads(self):
        """Return a list with number of loaded cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['LOAD_count'] / self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_loads
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'LOAD_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of loaded cache lines per memory hierarchy level .


def get_hits(self):
        """Return a list with number of hit cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['HIT_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_hits
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'HIT_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of hit cache lines per memory hierarchy level .


def get_misses(self):
        """Return a list with number of missed cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['MISS_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_misses
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'MISS_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of missed cache lines per memory hierarchy level .


def get_stores(self):
        """Return a list with number of stored cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['STORE_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_stores
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'STORE_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of stored cache lines per memory hierarchy level .


def get_evicts(self):
        """Return a list with number of evicted cache lines per memory hierarchy level."""
        return [self.stats[cache_level]['EVICT_count']/self.first_dim_factor
                for cache_level in range(len(self.machine['memory hierarchy']))]

def
get_evicts
(
self
)
:
return
[
self
.
stats
[
cache_level
]
[
'EVICT_count'
]
/
self
.
first_dim_factor
for
cache_level
in
range
(
len
(
self
.
machine
[
'memory hierarchy'
]
)
)
]

Return a list with number of evicted cache lines per memory hierarchy level .


def get_infos(self):
        """Return verbose information about the predictor."""
        first_dim_factor = self.first_dim_factor
        infos = {'memory hierarchy': [], 'cache stats': self.stats,
                 'cachelines in stats': first_dim_factor}
        for cache_level, cache_info in list(enumerate(self.machine['memory hierarchy'])):
            infos['memory hierarchy'].append({
                'index': len(infos['memory hierarchy']),
                'level': '{}'.format(cache_info['level']),
                'total loads': self.stats[cache_level]['LOAD_byte']/first_dim_factor,
                'total misses': self.stats[cache_level]['MISS_byte']/first_dim_factor,
                'total hits': self.stats[cache_level]['HIT_byte']/first_dim_factor,
                'total stores': self.stats[cache_level]['STORE_byte']/first_dim_factor,
                'total evicts': self.stats[cache_level]['EVICT_byte']/first_dim_factor,
                'total lines load': self.stats[cache_level]['LOAD_count']/first_dim_factor,
                'total lines misses': self.stats[cache_level]['MISS_count']/first_dim_factor,
                'total lines hits': self.stats[cache_level]['HIT_count']/first_dim_factor,
                'total lines stores': self.stats[cache_level]['STORE_count']/first_dim_factor,
                'total lines evicts': self.stats[cache_level]['EVICT_count']/first_dim_factor,
                'cycles': None})
        return infos

def
get_infos
(
self
)
:
first_dim_factor
=
self
.
first_dim_factor
infos
=
{
'memory hierarchy'
:
[
]
,
'cache stats'
:
self
.
stats
,
'cachelines in stats'
:
first_dim_factor
}
for
cache_level
,
cache_info
in
list
(
enumerate
(
self
.
machine
[
'memory hierarchy'
]
)
)
:
infos
[
'memory hierarchy'
]
.
append
(
{
'index'
:
len
(
infos
[
'memory hierarchy'
]
)
,
'level'
:
'{}'
.
format
(
cache_info
[
'level'
]
)
,
'total loads'
:
self
.
stats
[
cache_level
]
[
'LOAD_byte'
]
/
first_dim_factor
,
'total misses'
:
self
.
stats
[
cache_level
]
[
'MISS_byte'
]
/
first_dim_factor
,
'total hits'
:
self
.
stats
[
cache_level
]
[
'HIT_byte'
]
/
first_dim_factor
,
'total stores'
:
self
.
stats
[
cache_level
]
[
'STORE_byte'
]
/
first_dim_factor
,
'total evicts'
:
self
.
stats
[
cache_level
]
[
'EVICT_byte'
]
/
first_dim_factor
,
'total lines load'
:
self
.
stats
[
cache_level
]
[
'LOAD_count'
]
/
first_dim_factor
,
'total lines misses'
:
self
.
stats
[
cache_level
]
[
'MISS_count'
]
/
first_dim_factor
,
'total lines hits'
:
self
.
stats
[
cache_level
]
[
'HIT_count'
]
/
first_dim_factor
,
'total lines stores'
:
self
.
stats
[
cache_level
]
[
'STORE_count'
]
/
first_dim_factor
,
'total lines evicts'
:
self
.
stats
[
cache_level
]
[
'EVICT_count'
]
/
first_dim_factor
,
'cycles'
:
}
)
return
infos

Return verbose information about the predictor .


def fix_env_variable(name, value):
    """Fix environment variable to a value within context. Unset if value is None."""
    orig = os.environ.get(name, None)
    if value is not None:
        # Set if value is not None
        os.environ[name] = value
    elif name in os.environ:
        # Unset if value is None
        del os.environ[name]
    try:
        yield
    finally:
        if orig is not None:
            # Restore original value
            os.environ[name] = orig
        elif name in os.environ:
            # Unset
            del os.environ[name]

def
fix_env_variable
(
name
,
value
)
:
orig
=
os
.
environ
.
get
(
name
,
)
if
value
is
not
:
# Set if value is not None
os
.
environ
[
name
]
=
value
elif
name
in
os
.
environ
:
# Unset if value is None
del
os
.
environ
[
name
]
try
:
yield
finally
:
if
orig
is
not
:
# Restore original value
os
.
environ
[
name
]
=
orig
elif
name
in
os
.
environ
:
# Unset
del
os
.
environ
[
name
]

Fix environment variable to a value within context . Unset if value is None .


def configure_arggroup(cls, parser):
        """Configure argument parser."""
        parser.add_argument(
            '--no-phenoecm', action='store_true',
            help='Disables the phenomenological ECM model building.')
        parser.add_argument(
            '--iterations', type=int, default=10,
            help='Number of outer-loop iterations (e.g. time loop) during benchmarking. '
                 'Default is 10, but actual number will be adapted to at least 0.2s runtime.')
        parser.add_argument(
            '--ignore-warnings', action='store_true',
            help='Ignore warnings about missmatched CPU model and frequency.')


def report(self, output_file=sys.stdout):
        """Report gathered analysis data in human readable form."""
        if self.verbose > 1:
            with pprint_nosort():
                pprint.pprint(self.results)

        if self.verbose > 0:
            print('Runtime (per repetition): {:.2g} s'.format(
                self.results['Runtime (per repetition) [s]']),
                file=output_file)
        if self.verbose > 0:
            print('Iterations per repetition: {!s}'.format(
                self.results['Iterations per repetition']),
                file=output_file)
        print('Runtime (per cacheline update): {:.2f} cy/CL'.format(
            self.results['Runtime (per cacheline update) [cy/CL]']),
            file=output_file)
        print('MEM volume (per repetition): {:.0f} Byte'.format(
            self.results['MEM volume (per repetition) [B]']),
            file=output_file)
        print('Performance: {:.2f} MFLOP/s'.format(self.results['Performance [MFLOP/s]']),
              file=output_file)
        print('Performance: {:.2f} MLUP/s'.format(self.results['Performance [MLUP/s]']),
              file=output_file)
        print('Performance: {:.2f} It/s'.format(self.results['Performance [MIt/s]']),
              file=output_file)
        if self.verbose > 0:
            print('MEM bandwidth: {:.2f} MByte/s'.format(self.results['MEM BW [MByte/s]']),
                  file=output_file)
        print('', file=output_file)

        if not self.no_phenoecm:
            print("Data Transfers:")
            print("{:^8} |".format("cache"), end='')
            for metrics in self.results['data transfers'].values():
                for metric_name in sorted(metrics):
                    print(" {:^14}".format(metric_name), end='')
                print()
                break
            for cache, metrics in sorted(self.results['data transfers'].items()):
                print("{!s:^8} |".format(cache), end='')
                for k, v in sorted(metrics.items()):
                    print(" {!s:^14}".format(v), end='')
                print()
            print()

            print('Phenomenological ECM model: {{ {T_OL:.1f} || {T_nOL:.1f} | {T_L1L2:.1f} | '
                  '{T_L2L3:.1f} | {T_L3MEM:.1f} }} cy/CL'.format(
                **{k: float(v) for k, v in self.results['ECM'].items()}),
                file=output_file)
            print('T_OL assumes that two loads per cycle may be retiered, which is true for '
                  '128bit SSE/half-AVX loads on SNB and IVY, and 256bit full-AVX loads on HSW, '
                  'BDW, SKL and SKX, but it also depends on AGU availability.',
                  file=output_file)

Report gathered analysis data in human readable form .


def parse_description():
    """
    Parse the description in the README file

    CommandLine:
        python -c "import setup; print(setup.parse_description())"
    """
    from os.path import dirname, join, exists
    readme_fpath = join(dirname(__file__), 'README.md')
    # print('readme_fpath = %r' % (readme_fpath,))
    # This breaks on pip install, so check that it exists.
    if exists(readme_fpath):
        # try:
        #     # convert markdown to rst for pypi
        #     import pypandoc
        #     return pypandoc.convert(readme_fpath, 'rst')
        # except Exception as ex:
            # strip out markdown to make a clean readme for pypi
            textlines = []
            with open(readme_fpath, 'r') as f:
                capture = False
                for line in f.readlines():
                    if '# Purpose' in line:
                        capture = True
                    elif line.startswith('##'):
                        break
                    elif capture:
                        textlines += [line]
            text = ''.join(textlines).strip()
            text = text.replace('\n\n', '_NLHACK_')
            text = text.replace('\n', ' ')
            text = text.replace('_NLHACK_', '\n\n')
            return text
    return ''

#     # convert markdown to rst for pypi


def schedule_retry(self, config):
    """Schedule a retry"""
    raise self.retry(countdown=config.get('SAILTHRU_RETRY_SECONDS'),
                     max_retries=config.get('SAILTHRU_RETRY_ATTEMPTS'))

def schedule_retry ( self , config ) : raise self . retry ( countdown = config . get ( 'SAILTHRU_RETRY_SECONDS' ) , max_retries = config . get ( 'SAILTHRU_RETRY_ATTEMPTS' ) )


def _build_purchase_item(course_id, course_url, cost_in_cents, mode, course_data, sku):
    """Build and return Sailthru purchase item object"""

    # build item description
    item = {
        'id': "{}-{}".format(course_id, mode),
        'url': course_url,
        'price': cost_in_cents,
        'qty': 1,
    }

    # get title from course info if we don't already have it from Sailthru
    if 'title' in course_data:
        item['title'] = course_data['title']
    else:
        # can't find, just invent title
        item['title'] = 'Course {} mode: {}'.format(course_id, mode)

    if 'tags' in course_data:
        item['tags'] = course_data['tags']

    # add vars to item
    item['vars'] = dict(course_data.get('vars', {}), mode=mode, course_run_id=course_id)

    item['vars']['purchase_sku'] = sku

    return item


def _record_purchase(sailthru_client, email, item, purchase_incomplete, message_id, options):
    """Record a purchase in Sailthru

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        item (dict): Sailthru required information about the course
        purchase_incomplete (boolean): True if adding item to shopping cart
        message_id (str): Cookie used to identify marketing campaign
        options (dict): Sailthru purchase API options (e.g. template name)

    Returns:
        False if retryable error, else True
    """
    try:
        sailthru_response = sailthru_client.purchase(email, [item],
                                                     incomplete=purchase_incomplete, message_id=message_id,
                                                     options=options)

        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to record purchase in Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

    except SailthruClientError as exc:
        logger.exception("Exception attempting to record purchase for %s in Sailthru - %s", email, text_type(exc))
        return False

    return True


def _get_course_content(course_id, course_url, sailthru_client, site_code, config):
    """Get course information using the Sailthru content api or from cache.

    If there is an error, just return with an empty response.

    Arguments:
        course_id (str): course key of the course
        course_url (str): LMS url for course info page.
        sailthru_client (object): SailthruClient
        site_code (str): site code
        config (dict): config options

    Returns:
        course information from Sailthru
    """
    # check cache first
    cache_key = "{}:{}".format(site_code, course_url)
    response = cache.get(cache_key)
    if not response:
        try:
            sailthru_response = sailthru_client.api_get("content", {"id": course_url})
            if not sailthru_response.is_ok():
                response = {}
            else:
                response = sailthru_response.json
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

        except SailthruClientError:
            response = {}

        if not response:
            logger.error('Could not get course data from Sailthru on enroll/purchase event. '
                         'Calling Ecommerce Course API to get course info for enrollment confirmation email')
            response = _get_course_content_from_ecommerce(course_id, site_code=site_code)
            if response:
                cache.set(cache_key, response, config.get('SAILTHRU_CACHE_TTL_SECONDS'))

    return response

Get course information using the Sailthru content api or from cache .


def _get_course_content_from_ecommerce(course_id, site_code=None):
    """
    Get course information using the Ecommerce course api.

    In case of error returns empty response.
    Arguments:
        course_id (str): course key of the course
        site_code (str): site code

    Returns:
        course information from Ecommerce
    """
    api = get_ecommerce_client(site_code=site_code)
    try:
        api_response = api.courses(course_id).get()
    except Exception:  # pylint: disable=broad-except
        logger.exception(
            'An error occurred while retrieving data for course run [%s] from the Catalog API.',
            course_id,
            exc_info=True
        )
        return {}

    return {
        'title': api_response.get('name'),
        'verification_deadline': api_response.get('verification_deadline')
    }

Get course information using the Ecommerce course api .


def _update_unenrolled_list(sailthru_client, email, course_url, unenroll):
    """Maintain a list of courses the user has unenrolled from in the Sailthru user record

    Arguments:
        sailthru_client (object): SailthruClient
        email (str): user's email address
        course_url (str): LMS url for course info page.
        unenroll (boolean): True if unenrolling, False if enrolling

    Returns:
        False if retryable error, else True
    """
    try:
        # get the user 'vars' values from sailthru
        sailthru_response = sailthru_client.api_get("user", {"id": email, "fields": {"vars": 1}})
        if not sailthru_response.is_ok():
            error = sailthru_response.get_error()
            logger.error("Error attempting to read user record from Sailthru: %s", error.get_message())
            return not can_retry_sailthru_request(error)

        response_json = sailthru_response.json

        unenroll_list = []
        if response_json and "vars" in response_json and response_json["vars"] \
           and "unenrolled" in response_json["vars"]:
            unenroll_list = response_json["vars"]["unenrolled"]

        changed = False
        # if unenrolling, add course to unenroll list
        if unenroll:
            if course_url not in unenroll_list:
                unenroll_list.append(course_url)
                changed = True

        # if enrolling, remove course from unenroll list
        elif course_url in unenroll_list:
            unenroll_list.remove(course_url)
            changed = True

        if changed:
            # write user record back
            sailthru_response = sailthru_client.api_post(
                'user', {'id': email, 'key': 'email', 'vars': {'unenrolled': unenroll_list}})

            if not sailthru_response.is_ok():
                error = sailthru_response.get_error()
                logger.error("Error attempting to update user record in Sailthru: %s", error.get_message())
                return not can_retry_sailthru_request(error)

        return True

    except SailthruClientError as exc:
        logger.exception("Exception attempting to update user record for %s in Sailthru - %s", email, text_type(exc))
        return False


def send_course_refund_email(self, email, refund_id, amount, course_name, order_number, order_url, site_code=None):
    """ Sends the course refund email.

    Args:
        self: Ignore.
        email (str): Recipient's email address.
        refund_id (int): ID of the refund that initiated this task.
        amount (str): Formatted amount of the refund.
        course_name (str): Name of the course for which payment was refunded.
        order_number (str): Order number of the order that was refunded.
        order_url (str): Receipt URL of the refunded order.
        site_code (str): Identifier of the site sending the email.
    """
    config = get_sailthru_configuration(site_code)

    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        # NOTE: We rely on the function to log the error for us
        return

    email_vars = {
        'amount': amount,
        'course_name': course_name,
        'order_number': order_number,
        'order_url': order_url,
    }

    try:
        response = sailthru_client.send(
            template=config['templates']['course_refund'],
            email=email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            'A client error occurred while attempting to send a course refund notification for refund [%d].',
            refund_id
        )
        return

    if response.is_ok():
        logger.info('Course refund notification sent for refund %d.', refund_id)
    else:
        error = response.get_error()
        logger.error(
            'An error occurred while attempting to send a course refund notification for refund [%d]: %d - %s',
            refund_id, error.get_error_code(), error.get_message()
        )

        if can_retry_sailthru_request(error):
            logger.info(
                'An attempt will be made again to send a course refund notification for refund [%d].',
                refund_id
            )
            schedule_retry(self, config)
        else:
            logger.warning(
                'No further attempts will be made to send a course refund notification for refund [%d].',
                refund_id
            )


def _send_offer_assignment_notification_email(config, user_email, subject, email_body, site_code, task):
    """Handles sending offer assignment notification emails and retrying failed emails when appropriate."""
    try:
        sailthru_client = get_sailthru_client(site_code)
    except SailthruError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None
    email_vars = {
        'subject': subject,
        'email_body': email_body,
    }
    try:
        response = sailthru_client.send(
            template=config['templates']['assignment_email'],
            email=user_email,
            _vars=email_vars
        )
    except SailthruClientError:
        logger.exception(
            '[Offer Assignment] A client error occurred while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(message=email_body)
        )
        return None

    if not response.is_ok():
        error = response.get_error()
        logger.error(
            '[Offer Assignment] A {token_error_code} - {token_error_message} error occurred'
            ' while attempting to send a offer assignment notification.'
            ' Message: {message}'.format(
                message=email_body,
                token_error_code=error.get_error_code(),
                token_error_message=error.get_message()
            )
        )
        if can_retry_sailthru_request(error):
            logger.info(
                '[Offer Assignment] An attempt will be made to resend the offer assignment notification.'
                ' Message: {message}'.format(message=email_body)
            )
            schedule_retry(task, config)
        else:
            logger.warning(
                '[Offer Assignment] No further attempts will be made to send the offer assignment notification.'
                ' Failed Message: {message}'.format(message=email_body)
            )

    return response

Handles sending offer assignment notification emails and retrying failed emails when appropriate .


def get_logger_config(log_dir='/var/tmp',
                      logging_env='no_env',
                      edx_filename='edx.log',
                      dev_env=False,
                      debug=False,
                      local_loglevel='INFO',
                      service_variant='ecomworker'):

    """
    Returns a dictionary containing logging configuration.

    If dev_env is True, logging will not be done via local rsyslogd.
    Instead, application logs will be dropped into log_dir. 'edx_filename'
    is ignored unless dev_env is True.
    """
    # Revert to INFO if an invalid string is passed in
    if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']:
        local_loglevel = 'INFO'

    hostname = platform.node().split('.')[0]
    syslog_format = (
        '[service_variant={service_variant}]'
        '[%(name)s][env:{logging_env}] %(levelname)s '
        '[{hostname}  %(process)d] [%(filename)s:%(lineno)d] '
        '- %(message)s'
    ).format(
        service_variant=service_variant,
        logging_env=logging_env, hostname=hostname
    )

    if debug:
        handlers = ['console']
    else:
        handlers = ['local']

    logger_config = {
        'version': 1,
        'disable_existing_loggers': False,
        'formatters': {
            'standard': {
                'format': '%(asctime)s %(levelname)s %(process)d '
                          '[%(name)s] %(filename)s:%(lineno)d - %(message)s',
            },
            'syslog_format': {'format': syslog_format},
            'raw': {'format': '%(message)s'},
        },
        'handlers': {
            'console': {
                'level': 'DEBUG' if debug else 'INFO',
                'class': 'logging.StreamHandler',
                'formatter': 'standard',
                'stream': sys.stdout,
            },
        },
        'loggers': {
            'requests': {
                'handlers': handlers,
                'level': 'WARNING',
                'propagate': True
            },
            '': {
                'handlers': handlers,
                'level': 'DEBUG',
                'propagate': False
            },
        }
    }

    if dev_env:
        edx_file_loc = os.path.join(log_dir, edx_filename)
        logger_config['handlers'].update({
            'local': {
                'class': 'logging.handlers.RotatingFileHandler',
                'level': local_loglevel,
                'formatter': 'standard',
                'filename': edx_file_loc,
                'maxBytes': 1024 * 1024 * 2,
                'backupCount': 5,
            },
        })
    else:
        logger_config['handlers'].update({
            'local': {
                'level': local_loglevel,
                'class': 'logging.handlers.SysLogHandler',
                # Use a different address for Mac OS X
                'address': '/var/run/syslog' if sys.platform == 'darwin' else '/dev/log',
                'formatter': 'syslog_format',
                'facility': SysLogHandler.LOG_LOCAL0,
            },
        })

    return logger_config

Returns a dictionary containing logging configuration .


def _retry_order(self, exception, max_fulfillment_retries, order_number):
    """
    Retry with exponential backoff until fulfillment
    succeeds or the retry limit is reached. If the retry limit is exceeded,
    the exception is re-raised.
    """
    retries = self.request.retries
    if retries == max_fulfillment_retries:
        logger.exception('Fulfillment of order [%s] failed. Giving up.', order_number)
    else:
        logger.warning('Fulfillment of order [%s] failed. Retrying.', order_number)

    countdown = 2 ** retries
    raise self.retry(exc=exception, countdown=countdown, max_retries=max_fulfillment_retries)

Retry with exponential backoff until fulfillment succeeds or the retry limit is reached . If the retry limit is exceeded the exception is re - raised .


def fulfill_order(self, order_number, site_code=None, email_opt_in=False):
    """Fulfills an order.

    Arguments:
        order_number (str): Order number indicating which order to fulfill.

    Returns:
        None
    """
    max_fulfillment_retries = get_configuration('MAX_FULFILLMENT_RETRIES', site_code=site_code)
    api = get_ecommerce_client(site_code=site_code)
    try:
        logger.info('Requesting fulfillment of order [%s].', order_number)
        api.orders(order_number).fulfill.put(email_opt_in=email_opt_in)
    except exceptions.HttpClientError as exc:
        status_code = exc.response.status_code  # pylint: disable=no-member
        if status_code == 406:
            # The order is not fulfillable. Therefore, it must be complete.
            logger.info('Order [%s] has already been fulfilled. Ignoring.', order_number)
            raise Ignore()
        else:
            # Unknown client error. Let's retry to resolve it.
            logger.warning(
                'Fulfillment of order [%s] failed because of HttpClientError. Retrying',
                order_number,
                exc_info=True
            )
            _retry_order(self, exc, max_fulfillment_retries, order_number)

    except (exceptions.HttpServerError, exceptions.Timeout, SSLError) as exc:
        # Fulfillment failed, retry
        _retry_order(self, exc, max_fulfillment_retries, order_number)


def get_sailthru_client(site_code):
    """
    Returns a Sailthru client for the specified site.

    Args:
        site_code (str): Site for which the client should be configured.

    Returns:
        SailthruClient

    Raises:
        SailthruNotEnabled: If Sailthru is not enabled for the specified site.
        ConfigurationError: If either the Sailthru API key or secret are not set for the site.
    """
    # Get configuration
    config = get_sailthru_configuration(site_code)

    # Return if Sailthru integration disabled
    if not config.get('SAILTHRU_ENABLE'):
        msg = 'Sailthru is not enabled for site {}'.format(site_code)
        log.debug(msg)
        raise SailthruNotEnabled(msg)

    # Make sure key and secret configured
    key = config.get('SAILTHRU_KEY')
    secret = config.get('SAILTHRU_SECRET')

    if not (key and secret):
        msg = 'Both key and secret are required for site {}'.format(site_code)
        log.error(msg)
        raise ConfigurationError(msg)

    return SailthruClient(key, secret)

Returns a Sailthru client for the specified site .


def get(self, key):
        """Get an object from the cache

        Arguments:
            key (str): Cache key

        Returns:
            Cached object
        """
        lock.acquire()
        try:
            if key not in self:
                return None

            current_time = time.time()
            if self[key].expire > current_time:
                return self[key].value

            # expired key, clean out all expired keys
            deletes = []
            for k, val in self.items():
                if val.expire <= current_time:
                    deletes.append(k)
            for k in deletes:
                del self[k]

            return None
        finally:
            lock.release()


def set(self, key, value, duration):
        """Save an object in the cache

        Arguments:
            key (str): Cache key
            value (object): object to cache
            duration (int): time in seconds to keep object in cache

        """
        lock.acquire()
        try:
            self[key] = CacheObject(value, duration)
        finally:
            lock.release()

def set ( self , key , value , duration ) : lock . acquire ( ) try : self [ key ] = CacheObject ( value , duration ) finally : lock . release ( )


def get_configuration(variable, site_code=None):
    """
    Get a value from configuration.

    Retrieves the value corresponding to the given variable from the configuration module
    currently in use by the app.  Specify a site_code value to check for a site-specific override.

    Arguments:
        variable (str): The name of a variable from the configuration module.

    Keyword Arguments:
        site_code (str): The SITE_OVERRIDES key to inspect for site-specific values

    Returns:
        The value corresponding to the variable, or None if the variable is not found.
    """
    name = os.environ.get(CONFIGURATION_MODULE)

    # __import__ performs a full import, but only returns the top-level
    # package, not the targeted module. sys.modules is a dictionary
    # mapping module names to loaded modules.
    __import__(name)
    module = sys.modules[name]

    # Locate the setting in the specified module, then attempt to apply a site-specific override
    setting_value = getattr(module, variable, None)
    site_overrides = getattr(module, 'SITE_OVERRIDES', None)
    if site_overrides and site_code is not None:
        site_specific_overrides = site_overrides.get(site_code)
        if site_specific_overrides:
            override_value = site_specific_overrides.get(variable)
            if override_value:
                setting_value = override_value

    if setting_value is None:
        raise RuntimeError('Worker is improperly configured: {} is unset in {}.'.format(variable, module))
    return setting_value


def get_overrides_filename(variable):
    """
    Get the name of the file containing configuration overrides
    from the provided environment variable.
    """
    filename = os.environ.get(variable)

    if filename is None:
        msg = 'Please set the {} environment variable.'.format(variable)
        raise EnvironmentError(msg)

    return filename

Get the name of the file containing configuration overrides from the provided environment variable .


def get_value_by_version(d):
    """
    Finds the value depending in current eplus version.

    Parameters
    ----------
    d: dict
        {(0, 0): value, (x, x): value, ...}
        for current version (cv), current value is the value of version v such as v <= cv < v+1
    """
    from oplus import CONF  # touchy import

    cv = CONF.eplus_version[:2]
    for v, value in sorted(d.items(), reverse=True):
        if cv >= v:
            return value

def
get_value_by_version
(
d
)
:
from
oplus
import
CONF
# touchy import
cv
=
CONF
.
eplus_version
[
:
2
]
for
v
,
value
in
sorted
(
d
.
items
(
)
,
reverse
=
True
)
:
if
cv
>=
v
:
return
value

Finds the value depending in current eplus version .