def walk(self, *types):
        '''
        Iterator which visits all suites and suite files,
        yielding test cases and keywords
        '''
        requested = types if len(types) > 0 else [SuiteFile, ResourceFile, SuiteFolder, Testcase, Keyword]

        for thing in self.robot_files:
            if thing.__class__ in requested:
                yield thing
            if isinstance(thing, SuiteFolder):
                for child in thing.walk():
                    if child.__class__ in requested:
                        yield child
            else:
                for child in thing.walk(*types):
                    yield child

def walk ( self , * types ) : requested = types if len ( types ) > 0 else [ SuiteFile , ResourceFile , SuiteFolder , Testcase , Keyword ] for thing in self . robot_files : if thing . __class__ in requested : yield thing if isinstance ( thing , SuiteFolder ) : for child in thing . walk ( ) : if child . __class__ in requested : yield child else : for child in thing . walk ( * types ) : yield child

Iterator which visits all suites and suite files yielding test cases and keywords


def _load(self, path):

        '''
        The general idea is to do a quick parse, creating a list of
        tables. Each table is nothing more than a list of rows, with
        each row being a list of cells. Additional parsing such as
        combining rows into statements is done on demand. This first
        pass is solely to read in the plain text and organize it by table.
        '''

        self.tables = []
        current_table = DefaultTable(self)

        with Utf8Reader(path) as f:
            # N.B. the caller should be catching errors
            self.raw_text = f.read()

            f._file.seek(0) # bleh; wish this wasn't a private property
            matcher = Matcher(re.IGNORECASE)
            for linenumber, raw_text in enumerate(f.readlines()):
                linenumber += 1; # start counting at 1 rather than zero

                # this mimics what the robot TSV reader does --
                # it replaces non-breaking spaces with regular spaces,
                # and then strips trailing whitespace
                raw_text = raw_text.replace(u'\xA0', ' ')
                raw_text = raw_text.rstrip()

                # FIXME: I'm keeping line numbers but throwing away
                # where each cell starts. I should be preserving that
                # (though to be fair, robot is throwing that away so
                # I'll have to write my own splitter if I want to save
                # the character position)
                cells = TxtReader.split_row(raw_text)
                _heading_regex = r'^\s*\*+\s*(.*?)[ *]*$'

                if matcher(_heading_regex, cells[0]):
                    # we've found the start of a new table
                    table_name = matcher.group(1)
                    current_table = tableFactory(self, linenumber, table_name, raw_text)
                    self.tables.append(current_table)
                else:
                    current_table.append(Row(linenumber, raw_text, cells))

The general idea is to do a quick parse creating a list of tables . Each table is nothing more than a list of rows with each row being a list of cells . Additional parsing such as combining rows into statements is done on demand . This first pass is solely to read in the plain text and organize it by table .


def type(self):
        '''Return 'suite' or 'resource' or None

        This will return 'suite' if a testcase table is found;
        It will return 'resource' if at least one robot table
        is found. If no tables are found it will return None
        '''

        robot_tables = [table for table in self.tables if not isinstance(table, UnknownTable)]
        if len(robot_tables) == 0:
            return None

        for table in self.tables:
            if isinstance(table, TestcaseTable):
                return "suite"

        return "resource"

def type ( self ) : robot_tables = [ table for table in self . tables if not isinstance ( table , UnknownTable ) ] if len ( robot_tables ) == 0 : return None for table in self . tables : if isinstance ( table , TestcaseTable ) : return "suite" return "resource"


def keywords(self):
        '''Generator which returns all keywords in the suite'''
        for table in self.tables:
            if isinstance(table, KeywordTable):
                for keyword in table.keywords:
                    yield keyword

def keywords ( self ) : for table in self . tables : if isinstance ( table , KeywordTable ) : for keyword in table . keywords : yield keyword


def dump(self):
        '''Regurgitate the tables and rows'''
        for table in self.tables:
            print("*** %s ***" % table.name)
            table.dump()

def
dump
(
self
)
:
for
table
in
self
.
tables
:
print
(
"*** %s ***"
%
table
.
name
)
table
.
dump
(
)


def settings(self):
        '''Generator which returns all of the statements in all of the settings tables'''
        for table in self.tables:
            if isinstance(table, SettingTable):
                for statement in table.statements:
                    yield statement

def settings ( self ) : for table in self . tables : if isinstance ( table , SettingTable ) : for statement in table . statements : yield statement


def variables(self):
        '''Generator which returns all of the statements in all of the variables tables'''
        for table in self.tables:
            if isinstance(table, VariableTable):
                # FIXME: settings have statements, variables have rows WTF? :-(
                for statement in table.rows:
                    if statement[0] != "":
                        yield statement


def statements(self):
        '''Return a list of statements

        This is done by joining together any rows that
        have continuations
        '''
        # FIXME: no need to do this every time; we should cache the
        # result
        if len(self.rows) == 0:
            return []

        current_statement = Statement(self.rows[0])
        current_statement.startline = self.rows[0].linenumber
        current_statement.endline = self.rows[0].linenumber
        statements = []
        for row in self.rows[1:]:
            if len(row) > 0 and row[0] == "...":
                # we found a continuation
                current_statement += row[1:]
                current_statement.endline = row.linenumber
            else:
                if len(current_statement) > 0:
                    # append current statement to the list of statements...
                    statements.append(current_statement)
                # start a new statement
                current_statement = Statement(row)
                current_statement.startline = row.linenumber
                current_statement.endline = row.linenumber

        if len(current_statement) > 0:
            statements.append(current_statement)

        # trim trailing blank statements
        while (len(statements[-1]) == 0 or
               ((len(statements[-1]) == 1) and len(statements[-1][0]) == 0)):
            statements.pop()
        return statements


def append(self, row):
        '''
        The idea is, we recognize when we have a new testcase by
        checking the first cell. If it's not empty and not a comment,
        we have a new test case.

        '''
        if len(row) == 0:
            # blank line. Should we throw it away, or append a BlankLine object?
            return

        if (row[0] != "" and
            (not row[0].lstrip().startswith("#"))):
            # we have a new child table
            self._children.append(self._childClass(self.parent, row.linenumber, row[0]))
            if len(row.cells) > 1:
                # It appears the first row -- which contains the test case or
                # keyword name -- also has the first logical row of cells.
                # We'll create a Row, but we'll make the first cell empty instead
                # of leaving the name in it, since other code always assumes the
                # first cell is empty.
                #
                # To be honest, I'm not sure this is the Right Thing To Do, but
                # I'm too lazy to audit the code to see if it matters if we keep
                # the first cell intact. Sorry if this ends up causing you grief
                # some day...
                row[0] = ""
                self._children[-1].append(row.linenumber, row.raw_text, row.cells)

        elif len(self._children) == 0:
            # something before the first test case
            # For now, append it to self.comments; eventually we should flag
            # an error if it's NOT a comment
            self.comments.append(row)

        else:
            # another row for the testcase
            if len(row.cells) > 0:
                self._children[-1].append(row.linenumber, row.raw_text, row.cells)

The idea is we recognize when we have a new testcase by checking the first cell . If it s not empty and not a comment we have a new test case .


def report(self, obj, message, linenum, char_offset=0):
        """Report an error or warning"""
        self.controller.report(linenumber=linenum, filename=obj.path,
                               severity=self.severity, message=message,
                               rulename = self.__class__.__name__,
                               char=char_offset)

def report ( self , obj , message , linenum , char_offset = 0 ) : self . controller . report ( linenumber = linenum , filename = obj . path , severity = self . severity , message = message , rulename = self . __class__ . __name__ , char = char_offset )


def run(self, args):
        """Parse command line arguments, and run rflint"""

        self.args = self.parse_and_process_args(args)

        if self.args.version:
            print(__version__)
            return 0

        if self.args.rulefile:
            for filename in self.args.rulefile:
                self._load_rule_file(filename)

        if self.args.list:
            self.list_rules()
            return 0

        if self.args.describe:
            self._describe_rules(self.args.args)
            return 0

        self.counts = { ERROR: 0, WARNING: 0, "other": 0}

        for filename in self.args.args:
            if not (os.path.exists(filename)):
                sys.stderr.write("rflint: %s: No such file or directory\n" % filename)
                continue
            if os.path.isdir(filename):
                self._process_folder(filename)
            else:
                self._process_file(filename)

        if self.counts[ERROR] > 0:
            return self.counts[ERROR] if self.counts[ERROR] < 254 else 255

        return 0

Parse command line arguments and run rflint


def list_rules(self):
        """Print a list of all rules"""
        for rule in sorted(self.all_rules, key=lambda rule: rule.name):
            print(rule)
            if self.args.verbose:
                for line in rule.doc.split("\n"):
                    print("    ", line)

def
list_rules
(
self
)
:
for
rule
in
sorted
(
self
.
all_rules
,
key
=
lambda
rule
:
rule
.
name
)
:
print
(
rule
)
if
self
.
args
.
verbose
:
for
line
in
rule
.
doc
.
split
(
"\n"
)
:
print
(
"    "
,
line
)


def report(self, linenumber, filename, severity, message, rulename, char):
        """Report a rule violation"""

        if self._print_filename is not None:
            # we print the filename only once. self._print_filename
            # will get reset each time a new file is processed.
            print("+ " + self._print_filename)
            self._print_filename = None

        if severity in (WARNING, ERROR):
            self.counts[severity] += 1
        else:
            self.counts["other"] += 1

        print(self.args.format.format(linenumber=linenumber, filename=filename,
                                      severity=severity, message=message.encode('utf-8'),
                                      rulename=rulename, char=char))


def _get_rules(self, cls):
        """Returns a list of rules of a given class

        Rules are treated as singletons - we only instantiate each
        rule once.
        """

        result = []
        for rule_class in cls.__subclasses__():
            rule_name = rule_class.__name__.lower()
            if rule_name not in self._rules:
                rule = rule_class(self)
                self._rules[rule_name] = rule
            result.append(self._rules[rule_name])
        return result

def _get_rules ( self , cls ) : result = [ ] for rule_class in cls . __subclasses__ ( ) : rule_name = rule_class . __name__ . lower ( ) if rule_name not in self . _rules : rule = rule_class ( self ) self . _rules [ rule_name ] = rule result . append ( self . _rules [ rule_name ] ) return result

Returns a list of rules of a given class Rules are treated as singletons - we only instantiate each rule once .


def _load_rule_file(self, filename):
        """Import the given rule file"""
        if not (os.path.exists(filename)):
            sys.stderr.write("rflint: %s: No such file or directory\n" % filename)
            return
        try:
            basename = os.path.basename(filename)
            (name, ext) = os.path.splitext(basename)
            imp.load_source(name, filename)
        except Exception as e:
            sys.stderr.write("rflint: %s: exception while loading: %s\n" % (filename, str(e)))


def parse_and_process_args(self, args):
        """Handle the parsing of command line arguments."""

        parser = argparse.ArgumentParser(
            prog="python -m rflint",
            description="A style checker for robot framework plain text files.",
            formatter_class=argparse.RawDescriptionHelpFormatter,
            epilog = (
                "You can use 'all' in place of RULENAME to refer to all rules. \n"
                "\n"
                "For example: '--ignore all --warn DuplicateTestNames' will ignore all\n"
                "rules except DuplicateTestNames.\n"
                "\n"
                "FORMAT is a string that performs a substitution on the following \n"
                "patterns: {severity}, {linenumber}, {char}, {message}, and {rulename}.\n"
                "\n"
                "For example: --format 'line: {linenumber}: message: {message}'. \n"
                "\n"
                "ARGUMENTFILE is a filename with contents that match the format of \n"
                "standard robot framework argument files\n"
                "\n"
                "If you give a directory as an argument, all files in the directory\n"
                "with the suffix .txt, .robot or .tsv will be processed. With the \n"
                "--recursive option, subfolders within the directory will also be\n"
                "processed."
                )
            )
        parser.add_argument("--error", "-e", metavar="RULENAME", action=SetErrorAction,
                            help="Assign a severity of ERROR to the given RULENAME")
        parser.add_argument("--ignore", "-i", metavar="RULENAME", action=SetIgnoreAction,
                            help="Ignore the given RULENAME")
        parser.add_argument("--warning", "-w", metavar="RULENAME", action=SetWarningAction,
                            help="Assign a severity of WARNING for the given RULENAME")
        parser.add_argument("--list", "-l", action="store_true",
                            help="show a list of known rules and exit")
        parser.add_argument("--describe", "-d", action="store_true",
                            help="describe the given rules")
        parser.add_argument("--no-filenames", action="store_false", dest="print_filenames",
                            default=True,
                            help="suppress the printing of filenames")
        parser.add_argument("--format", "-f",
                            help="Define the output format",
                            default='{severity}: {linenumber}, {char}: {message} ({rulename})')
        parser.add_argument("--version", action="store_true", default=False,
                            help="Display version number and exit")
        parser.add_argument("--verbose", "-v", action="store_true", default=False,
                            help="Give verbose output")
        parser.add_argument("--configure", "-c", action=ConfigureAction,
                            help="Configure a rule")
        parser.add_argument("--recursive", "-r", action="store_true", default=False,
                            help="Recursively scan subfolders in a directory")
        parser.add_argument("--rulefile", "-R", action=RulefileAction,
                            help="import additional rules from the given RULEFILE")
        parser.add_argument("--argumentfile", "-A", action=ArgfileLoader,
                            help="read arguments from the given file")
        parser.add_argument('args', metavar="file", nargs=argparse.REMAINDER)

        # create a custom namespace, in which we can store a reference to
        # our rules. This lets the custom argument actions access the list
        # of rules
        ns = argparse.Namespace()
        setattr(ns, "app", self)
        args = parser.parse_args(args, ns)

        Rule.output_format = args.format

        return args

Handle the parsing of command line arguments .


def from_resolver(cls, spec_resolver):
        """Creates a customized Draft4ExtendedValidator.

        :param spec_resolver: resolver for the spec
        :type resolver: :class:`jsonschema.RefResolver`
        """
        spec_validators = cls._get_spec_validators(spec_resolver)
        return validators.extend(Draft4Validator, spec_validators)

def from_resolver ( cls , spec_resolver ) : spec_validators = cls . _get_spec_validators ( spec_resolver ) return validators . extend ( Draft4Validator , spec_validators )

Creates a customized Draft4ExtendedValidator .


def construct_mapping(self, node, deep=False):
        """While yaml supports integer keys, these are not valid in
        json, and will break jsonschema. This method coerces all keys
        to strings.
        """
        mapping = super(ExtendedSafeConstructor, self).construct_mapping(
            node, deep)

        return {
            (str(key) if isinstance(key, int) else key): mapping[key]
            for key in mapping
        }

def construct_mapping ( self , node , deep = False ) : mapping = super ( ExtendedSafeConstructor , self ) . construct_mapping ( node , deep ) return { ( str ( key ) if isinstance ( key , int ) else key ) : mapping [ key ] for key in mapping }

While yaml supports integer keys these are not valid in json and will break jsonschema . This method coerces all keys to strings .


def read_yaml_file(path, loader=ExtendedSafeLoader):
    """Open a file, read it and return its contents."""
    with open(path) as fh:
        return load(fh, loader)

def read_yaml_file ( path , loader = ExtendedSafeLoader ) : with open ( path ) as fh : return load ( fh , loader )

Open a file read it and return its contents .


def from_spec_resolver(cls, spec_resolver):
        """Creates validators generator for the spec resolver.

        :param spec_resolver: resolver for the spec
        :type instance_resolver: :class:`jsonschema.RefResolver`
        """
        deref = DerefValidatorDecorator(spec_resolver)
        for key, validator_callable in iteritems(cls.validators):
            yield key, deref(validator_callable)

def from_spec_resolver ( cls , spec_resolver ) : deref = DerefValidatorDecorator ( spec_resolver ) for key , validator_callable in iteritems ( cls . validators ) : yield key , deref ( validator_callable )

Creates validators generator for the spec resolver .


def create_model_path(model_path):
    """
    Creates a path to model files
    model_path - string
    """
    if not model_path.startswith("/") and not model_path.startswith("models/"):
        model_path="/" + model_path
    if not model_path.startswith("models"):
        model_path = "models" + model_path
    if not model_path.endswith(".p"):
        model_path+=".p"

    return model_path

def create_model_path ( model_path ) : if not model_path . startswith ( "/" ) and not model_path . startswith ( "models/" ) : model_path = "/" + model_path if not model_path . startswith ( "models" ) : model_path = "models" + model_path if not model_path . endswith ( ".p" ) : model_path += ".p" return model_path


def sub_chars(string):
    """
    Strips illegal characters from a string.  Used to sanitize input essays.
    Removes all non-punctuation, digit, or letter characters.
    Returns sanitized string.
    string - string
    """
    #Define replacement patterns
    sub_pat = r"[^A-Za-z\.\?!,';:]"
    char_pat = r"\."
    com_pat = r","
    ques_pat = r"\?"
    excl_pat = r"!"
    sem_pat = r";"
    col_pat = r":"
    whitespace_pat = r"\s{1,}"

    #Replace text.  Ordering is very important!
    nstring = re.sub(sub_pat, " ", string)
    nstring = re.sub(char_pat," .", nstring)
    nstring = re.sub(com_pat, " ,", nstring)
    nstring = re.sub(ques_pat, " ?", nstring)
    nstring = re.sub(excl_pat, " !", nstring)
    nstring = re.sub(sem_pat, " ;", nstring)
    nstring = re.sub(col_pat, " :", nstring)
    nstring = re.sub(whitespace_pat, " ", nstring)

    return nstring

Strips illegal characters from a string . Used to sanitize input essays . Removes all non - punctuation digit or letter characters . Returns sanitized string . string - string


def spell_correct(string):
    """
    Uses aspell to spell correct an input string.
    Requires aspell to be installed and added to the path.
    Returns the spell corrected string if aspell is found, original string if not.
    string - string
    """

    # Create a temp file so that aspell could be used
    # By default, tempfile will delete this file when the file handle is closed.
    f = tempfile.NamedTemporaryFile(mode='w')
    f.write(string)
    f.flush()
    f_path = os.path.abspath(f.name)
    try:
        p = os.popen(aspell_path + " -a < " + f_path + " --sug-mode=ultra")

        # Aspell returns a list of incorrect words with the above flags
        incorrect = p.readlines()
        p.close()

    except Exception:
        log.exception("aspell process failed; could not spell check")
        # Return original string if aspell fails
        return string,0, string

    finally:
        f.close()

    incorrect_words = list()
    correct_spelling = list()
    for i in range(1, len(incorrect)):
        if(len(incorrect[i]) > 10):
            #Reformat aspell output to make sense
            match = re.search(":", incorrect[i])
            if hasattr(match, "start"):
                begstring = incorrect[i][2:match.start()]
                begmatch = re.search(" ", begstring)
                begword = begstring[0:begmatch.start()]

                sugstring = incorrect[i][match.start() + 2:]
                sugmatch = re.search(",", sugstring)
                if hasattr(sugmatch, "start"):
                    sug = sugstring[0:sugmatch.start()]

                    incorrect_words.append(begword)
                    correct_spelling.append(sug)

    #Create markup based on spelling errors
    newstring = string
    markup_string = string
    already_subbed=[]
    for i in range(0, len(incorrect_words)):
        sub_pat = r"\b" + incorrect_words[i] + r"\b"
        sub_comp = re.compile(sub_pat)
        newstring = re.sub(sub_comp, correct_spelling[i], newstring)
        if incorrect_words[i] not in already_subbed:
            markup_string=re.sub(sub_comp,'<bs>' + incorrect_words[i] + "</bs>", markup_string)
            already_subbed.append(incorrect_words[i])

    return newstring,len(incorrect_words),markup_string

Uses aspell to spell correct an input string . Requires aspell to be installed and added to the path . Returns the spell corrected string if aspell is found original string if not . string - string


def f7(seq):
    """
    Makes a list unique
    """
    seen = set()
    seen_add = seen.add
    return [x for x in seq if x not in seen and not seen_add(x)]

def f7 ( seq ) : seen = set ( ) seen_add = seen . add return [ x for x in seq if x not in seen and not seen_add ( x ) ]


def count_list(the_list):
    """
    Generates a count of the number of times each unique item appears in a list
    """
    count = the_list.count
    result = [(item, count(item)) for item in set(the_list)]
    result.sort()
    return result

def count_list ( the_list ) : count = the_list . count result = [ ( item , count ( item ) ) for item in set ( the_list ) ] result . sort ( ) return result


def regenerate_good_tokens(string):
    """
    Given an input string, part of speech tags the string, then generates a list of
    ngrams that appear in the string.
    Used to define grammatically correct part of speech tag sequences.
    Returns a list of part of speech tag sequences.
    """
    toks = nltk.word_tokenize(string)
    pos_string = nltk.pos_tag(toks)
    pos_seq = [tag[1] for tag in pos_string]
    pos_ngrams = ngrams(pos_seq, 2, 4)
    sel_pos_ngrams = f7(pos_ngrams)
    return sel_pos_ngrams

def regenerate_good_tokens ( string ) : toks = nltk . word_tokenize ( string ) pos_string = nltk . pos_tag ( toks ) pos_seq = [ tag [ 1 ] for tag in pos_string ] pos_ngrams = ngrams ( pos_seq , 2 , 4 ) sel_pos_ngrams = f7 ( pos_ngrams ) return sel_pos_ngrams

Given an input string part of speech tags the string then generates a list of ngrams that appear in the string . Used to define grammatically correct part of speech tag sequences . Returns a list of part of speech tag sequences .


def edit_distance(s1, s2):
    """
    Calculates string edit distance between string 1 and string 2.
    Deletion, insertion, substitution, and transposition all increase edit distance.
    """
    d = {}
    lenstr1 = len(s1)
    lenstr2 = len(s2)
    for i in xrange(-1, lenstr1 + 1):
        d[(i, -1)] = i + 1
    for j in xrange(-1, lenstr2 + 1):
        d[(-1, j)] = j + 1

    for i in xrange(lenstr1):
        for j in xrange(lenstr2):
            if s1[i] == s2[j]:
                cost = 0
            else:
                cost = 1
            d[(i, j)] = min(
                d[(i - 1, j)] + 1, # deletion
                d[(i, j - 1)] + 1, # insertion
                d[(i - 1, j - 1)] + cost, # substitution
            )
            if i and j and s1[i] == s2[j - 1] and s1[i - 1] == s2[j]:
                d[(i, j)] = min(d[(i, j)], d[i - 2, j - 2] + cost) # transposition

    return d[lenstr1 - 1, lenstr2 - 1]

def
edit_distance
(
s1
,
s2
)
:
d
=
{
}
lenstr1
=
len
(
s1
)
lenstr2
=
len
(
s2
)
for
i
in
xrange
(
1
,
lenstr1
+
1
)
:
d
[
(
i
,
1
)
]
=
i
+
1
for
j
in
xrange
(
1
,
lenstr2
+
1
)
:
d
[
(
1
,
j
)
]
=
j
+
1
for
i
in
xrange
(
lenstr1
)
:
for
j
in
xrange
(
lenstr2
)
:
if
s1
[
i
]
==
s2
[
j
]
:
cost
=
0
else
:
cost
=
1
d
[
(
i
,
j
)
]
=
min
(
d
[
(
i
1
,
j
)
]
+
1
,
# deletion
d
[
(
i
,
j
1
)
]
+
1
,
# insertion
d
[
(
i
1
,
j
1
)
]
+
cost
,
# substitution
)
if
i
and
j
and
s1
[
i
]
==
s2
[
j
1
]
and
s1
[
i
1
]
==
s2
[
j
]
:
d
[
(
i
,
j
)
]
=
min
(
d
[
(
i
,
j
)
]
,
d
[
i
2
,
j
2
]
+
cost
)
# transposition
return
d
[
lenstr1
1
,
lenstr2
1
]

Calculates string edit distance between string 1 and string 2 . Deletion insertion substitution and transposition all increase edit distance .


def gen_preds(clf, arr):
    """
    Generates predictions on a novel data array using a fit classifier
    clf is a classifier that has already been fit
    arr is a data array identical in dimension to the array clf was trained on
    Returns the array of predictions.
    """
    if(hasattr(clf, "predict_proba")):
        ret = clf.predict(arr)
        # pred_score=preds.argmax(1)+min(x._score)
    else:
        ret = clf.predict(arr)
    return ret

Generates predictions on a novel data array using a fit classifier clf is a classifier that has already been fit arr is a data array identical in dimension to the array clf was trained on Returns the array of predictions .


def calc_list_average(l):
    """
    Calculates the average value of a list of numbers
    Returns a float
    """
    total = 0.0
    for value in l:
        total += value
    return total / len(l)

def calc_list_average ( l ) : total = 0.0 for value in l : total += value return total / len ( l )


def quadratic_weighted_kappa(rater_a, rater_b, min_rating=None, max_rating=None):
    """
    Calculates kappa correlation between rater_a and rater_b.
    Kappa measures how well 2 quantities vary together.
    rater_a is a list of rater a scores
    rater_b is a list of rater b scores
    min_rating is an optional argument describing the minimum rating possible on the data set
    max_rating is an optional argument describing the maximum rating possible on the data set
    Returns a float corresponding to the kappa correlation
    """
    assert(len(rater_a) == len(rater_b))
    rater_a = [int(a) for a in rater_a]
    rater_b = [int(b) for b in rater_b]
    if min_rating is None:
        min_rating = min(rater_a + rater_b)
    if max_rating is None:
        max_rating = max(rater_a + rater_b)
    conf_mat = confusion_matrix(rater_a, rater_b,
        min_rating, max_rating)
    num_ratings = len(conf_mat)
    num_scored_items = float(len(rater_a))

    hist_rater_a = histogram(rater_a, min_rating, max_rating)
    hist_rater_b = histogram(rater_b, min_rating, max_rating)

    numerator = 0.0
    denominator = 0.0

    if(num_ratings > 1):
        for i in range(num_ratings):
            for j in range(num_ratings):
                expected_count = (hist_rater_a[i] * hist_rater_b[j]
                                  / num_scored_items)
                d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)
                numerator += d * conf_mat[i][j] / num_scored_items
                denominator += d * expected_count / num_scored_items

        return 1.0 - numerator / denominator
    else:
        return 1.0

def quadratic_weighted_kappa ( rater_a , rater_b , min_rating = None , max_rating = None ) : assert ( len ( rater_a ) == len ( rater_b ) ) rater_a = [ int ( a ) for a in rater_a ] rater_b = [ int ( b ) for b in rater_b ] if min_rating is None : min_rating = min ( rater_a + rater_b ) if max_rating is None : max_rating = max ( rater_a + rater_b ) conf_mat = confusion_matrix ( rater_a , rater_b , min_rating , max_rating ) num_ratings = len ( conf_mat ) num_scored_items = float ( len ( rater_a ) ) hist_rater_a = histogram ( rater_a , min_rating , max_rating ) hist_rater_b = histogram ( rater_b , min_rating , max_rating ) numerator = 0.0 denominator = 0.0 if ( num_ratings > 1 ) : for i in range ( num_ratings ) : for j in range ( num_ratings ) : expected_count = ( hist_rater_a [ i ] * hist_rater_b [ j ] / num_scored_items ) d = pow ( i - j , 2.0 ) / pow ( num_ratings - 1 , 2.0 ) numerator += d * conf_mat [ i ] [ j ] / num_scored_items denominator += d * expected_count / num_scored_items return 1.0 - numerator / denominator else : return 1.0

Calculates kappa correlation between rater_a and rater_b . Kappa measures how well 2 quantities vary together . rater_a is a list of rater a scores rater_b is a list of rater b scores min_rating is an optional argument describing the minimum rating possible on the data set max_rating is an optional argument describing the maximum rating possible on the data set Returns a float corresponding to the kappa correlation


def confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):
    """
    Generates a confusion matrix between rater_a and rater_b
    A confusion matrix shows how often 2 values agree and disagree
    See quadratic_weighted_kappa for argument descriptions
    """
    assert(len(rater_a) == len(rater_b))
    rater_a = [int(a) for a in rater_a]
    rater_b = [int(b) for b in rater_b]
    min_rating = int(min_rating)
    max_rating = int(max_rating)
    if min_rating is None:
        min_rating = min(rater_a)
    if max_rating is None:
        max_rating = max(rater_a)
    num_ratings = int(max_rating - min_rating + 1)
    conf_mat = [[0 for i in range(num_ratings)]
                for j in range(num_ratings)]
    for a, b in zip(rater_a, rater_b):
        conf_mat[int(a - min_rating)][int(b - min_rating)] += 1
    return conf_mat

def confusion_matrix ( rater_a , rater_b , min_rating = None , max_rating = None ) : assert ( len ( rater_a ) == len ( rater_b ) ) rater_a = [ int ( a ) for a in rater_a ] rater_b = [ int ( b ) for b in rater_b ] min_rating = int ( min_rating ) max_rating = int ( max_rating ) if min_rating is None : min_rating = min ( rater_a ) if max_rating is None : max_rating = max ( rater_a ) num_ratings = int ( max_rating - min_rating + 1 ) conf_mat = [ [ 0 for i in range ( num_ratings ) ] for j in range ( num_ratings ) ] for a , b in zip ( rater_a , rater_b ) : conf_mat [ int ( a - min_rating ) ] [ int ( b - min_rating ) ] += 1 return conf_mat


def histogram(ratings, min_rating=None, max_rating=None):
    """
    Generates a frequency count of each rating on the scale
    ratings is a list of scores
    Returns a list of frequencies
    """
    ratings = [int(r) for r in ratings]
    if min_rating is None:
        min_rating = min(ratings)
    if max_rating is None:
        max_rating = max(ratings)
    num_ratings = int(max_rating - min_rating + 1)
    hist_ratings = [0 for x in range(num_ratings)]
    for r in ratings:
        hist_ratings[r - min_rating] += 1
    return hist_ratings

def histogram ( ratings , min_rating = None , max_rating = None ) : ratings = [ int ( r ) for r in ratings ] if min_rating is None : min_rating = min ( ratings ) if max_rating is None : max_rating = max ( ratings ) num_ratings = int ( max_rating - min_rating + 1 ) hist_ratings = [ 0 for x in range ( num_ratings ) ] for r in ratings : hist_ratings [ r - min_rating ] += 1 return hist_ratings


def encode_plus(s):
    """
    Literally encodes the plus sign
    input is a string
    returns the string with plus signs encoded
    """
    regex = r"\+"
    pat = re.compile(regex)
    return pat.sub("%2B", s)

def encode_plus ( s ) : regex = r"\+" pat = re . compile ( regex ) return pat . sub ( "%2B" , s )


def initialize_dictionaries(self, e_set, max_feats2 = 200):
        """
        Initializes dictionaries from an essay set object
        Dictionaries must be initialized prior to using this to extract features
        e_set is an input essay set
        returns a confirmation of initialization
        """
        if(hasattr(e_set, '_type')):
            if(e_set._type == "train"):
                #normal text (unstemmed) useful words/bigrams
                nvocab = util_functions.get_vocab(e_set._text, e_set._score, max_feats2 = max_feats2)
                #stemmed and spell corrected vocab useful words/ngrams
                svocab = util_functions.get_vocab(e_set._clean_stem_text, e_set._score, max_feats2 = max_feats2)
                #dictionary trained on proper vocab
                self._normal_dict = CountVectorizer(ngram_range=(1,2), vocabulary=nvocab)
                #dictionary trained on proper vocab
                self._stem_dict = CountVectorizer(ngram_range=(1,2), vocabulary=svocab)
                self.dict_initialized = True
                #Average spelling errors in set. needed later for spelling detection
                self._mean_spelling_errors=sum(e_set._spelling_errors)/float(len(e_set._spelling_errors))
                self._spell_errors_per_character=sum(e_set._spelling_errors)/float(sum([len(t) for t in e_set._text]))
                #Gets the number and positions of grammar errors
                good_pos_tags,bad_pos_positions=self._get_grammar_errors(e_set._pos,e_set._text,e_set._tokens)
                self._grammar_errors_per_character=(sum(good_pos_tags)/float(sum([len(t) for t in e_set._text])))
                #Generate bag of words features
                bag_feats=self.gen_bag_feats(e_set)
                #Sum of a row of bag of words features (topical words in an essay)
                f_row_sum=numpy.sum(bag_feats[:,:])
                #Average index of how "topical" essays are
                self._mean_f_prop=f_row_sum/float(sum([len(t) for t in e_set._text]))
                ret = "ok"
            else:
                raise util_functions.InputError(e_set, "needs to be an essay set of the train type.")
        else:
            raise util_functions.InputError(e_set, "wrong input. need an essay set object")
        return ret


def get_good_pos_ngrams(self):
        """
        Gets a set of gramatically correct part of speech sequences from an input file called essaycorpus.txt
        Returns the set and caches the file
        """
        if(os.path.isfile(NGRAM_PATH)):
            good_pos_ngrams = pickle.load(open(NGRAM_PATH, 'rb'))
        elif os.path.isfile(ESSAY_CORPUS_PATH):
            essay_corpus = open(ESSAY_CORPUS_PATH).read()
            essay_corpus = util_functions.sub_chars(essay_corpus)
            good_pos_ngrams = util_functions.regenerate_good_tokens(essay_corpus)
            pickle.dump(good_pos_ngrams, open(NGRAM_PATH, 'wb'))
        else:
            #Hard coded list in case the needed files cannot be found
            good_pos_ngrams=['NN PRP', 'NN PRP .', 'NN PRP . DT', 'PRP .', 'PRP . DT', 'PRP . DT NNP', '. DT',
             '. DT NNP', '. DT NNP NNP', 'DT NNP', 'DT NNP NNP', 'DT NNP NNP NNP', 'NNP NNP',
             'NNP NNP NNP', 'NNP NNP NNP NNP', 'NNP NNP NNP .', 'NNP NNP .', 'NNP NNP . TO',
             'NNP .', 'NNP . TO', 'NNP . TO NNP', '. TO', '. TO NNP', '. TO NNP NNP',
             'TO NNP', 'TO NNP NNP']

        return set(good_pos_ngrams)

Gets a set of gramatically correct part of speech sequences from an input file called essaycorpus . txt Returns the set and caches the file


def gen_length_feats(self, e_set):
        """
        Generates length based features from an essay set
        Generally an internal function called by gen_feats
        Returns an array of length features
        e_set - EssaySet object
        """
        text = e_set._text
        lengths = [len(e) for e in text]
        word_counts = [max(len(t),1) for t in e_set._tokens]
        comma_count = [e.count(",") for e in text]
        ap_count = [e.count("'") for e in text]
        punc_count = [e.count(".") + e.count("?") + e.count("!") for e in text]
        chars_per_word = [lengths[m] / float(word_counts[m]) for m in xrange(0, len(text))]

        good_pos_tags,bad_pos_positions= self._get_grammar_errors(e_set._pos,e_set._text,e_set._tokens)
        good_pos_tag_prop = [good_pos_tags[m] / float(word_counts[m]) for m in xrange(0, len(text))]

        length_arr = numpy.array((
        lengths, word_counts, comma_count, ap_count, punc_count, chars_per_word, good_pos_tags,
        good_pos_tag_prop)).transpose()

        return length_arr.copy()

def gen_length_feats ( self , e_set ) : text = e_set . _text lengths = [ len ( e ) for e in text ] word_counts = [ max ( len ( t ) , 1 ) for t in e_set . _tokens ] comma_count = [ e . count ( "," ) for e in text ] ap_count = [ e . count ( "'" ) for e in text ] punc_count = [ e . count ( "." ) + e . count ( "?" ) + e . count ( "!" ) for e in text ] chars_per_word = [ lengths [ m ] / float ( word_counts [ m ] ) for m in xrange ( 0 , len ( text ) ) ] good_pos_tags , bad_pos_positions = self . _get_grammar_errors ( e_set . _pos , e_set . _text , e_set . _tokens ) good_pos_tag_prop = [ good_pos_tags [ m ] / float ( word_counts [ m ] ) for m in xrange ( 0 , len ( text ) ) ] length_arr = numpy . array ( ( lengths , word_counts , comma_count , ap_count , punc_count , chars_per_word , good_pos_tags , good_pos_tag_prop ) ) . transpose ( ) return length_arr . copy ( )


def gen_bag_feats(self, e_set):
        """
        Generates bag of words features from an input essay set and trained FeatureExtractor
        Generally called by gen_feats
        Returns an array of features
        e_set - EssaySet object
        """
        if(hasattr(self, '_stem_dict')):
            sfeats = self._stem_dict.transform(e_set._clean_stem_text)
            nfeats = self._normal_dict.transform(e_set._text)
            bag_feats = numpy.concatenate((sfeats.toarray(), nfeats.toarray()), axis=1)
        else:
            raise util_functions.InputError(self, "Dictionaries must be initialized prior to generating bag features.")
        return bag_feats.copy()


def gen_feats(self, e_set):
        """
        Generates bag of words, length, and prompt features from an essay set object
        returns an array of features
        e_set - EssaySet object
        """
        bag_feats = self.gen_bag_feats(e_set)
        length_feats = self.gen_length_feats(e_set)
        prompt_feats = self.gen_prompt_feats(e_set)
        overall_feats = numpy.concatenate((length_feats, prompt_feats, bag_feats), axis=1)
        overall_feats = overall_feats.copy()

        return overall_feats

def gen_feats ( self , e_set ) : bag_feats = self . gen_bag_feats ( e_set ) length_feats = self . gen_length_feats ( e_set ) prompt_feats = self . gen_prompt_feats ( e_set ) overall_feats = numpy . concatenate ( ( length_feats , prompt_feats , bag_feats ) , axis = 1 ) overall_feats = overall_feats . copy ( ) return overall_feats

Generates bag of words length and prompt features from an essay set object returns an array of features e_set - EssaySet object


def gen_prompt_feats(self, e_set):
        """
        Generates prompt based features from an essay set object and internal prompt variable.
        Generally called internally by gen_feats
        Returns an array of prompt features
        e_set - EssaySet object
        """
        prompt_toks = nltk.word_tokenize(e_set._prompt)
        expand_syns = []
        for word in prompt_toks:
            synonyms = util_functions.get_wordnet_syns(word)
            expand_syns.append(synonyms)
        expand_syns = list(chain.from_iterable(expand_syns))
        prompt_overlap = []
        prompt_overlap_prop = []
        for j in e_set._tokens:
            tok_length=len(j)
            if(tok_length==0):
                tok_length=1
            prompt_overlap.append(len([i for i in j if i in prompt_toks]))
            prompt_overlap_prop.append(prompt_overlap[len(prompt_overlap) - 1] / float(tok_length))
        expand_overlap = []
        expand_overlap_prop = []
        for j in e_set._tokens:
            tok_length=len(j)
            if(tok_length==0):
                tok_length=1
            expand_overlap.append(len([i for i in j if i in expand_syns]))
            expand_overlap_prop.append(expand_overlap[len(expand_overlap) - 1] / float(tok_length))

        prompt_arr = numpy.array((prompt_overlap, prompt_overlap_prop, expand_overlap, expand_overlap_prop)).transpose()

        return prompt_arr.copy()

def gen_prompt_feats ( self , e_set ) : prompt_toks = nltk . word_tokenize ( e_set . _prompt ) expand_syns = [ ] for word in prompt_toks : synonyms = util_functions . get_wordnet_syns ( word ) expand_syns . append ( synonyms ) expand_syns = list ( chain . from_iterable ( expand_syns ) ) prompt_overlap = [ ] prompt_overlap_prop = [ ] for j in e_set . _tokens : tok_length = len ( j ) if ( tok_length == 0 ) : tok_length = 1 prompt_overlap . append ( len ( [ i for i in j if i in prompt_toks ] ) ) prompt_overlap_prop . append ( prompt_overlap [ len ( prompt_overlap ) - 1 ] / float ( tok_length ) ) expand_overlap = [ ] expand_overlap_prop = [ ] for j in e_set . _tokens : tok_length = len ( j ) if ( tok_length == 0 ) : tok_length = 1 expand_overlap . append ( len ( [ i for i in j if i in expand_syns ] ) ) expand_overlap_prop . append ( expand_overlap [ len ( expand_overlap ) - 1 ] / float ( tok_length ) ) prompt_arr = numpy . array ( ( prompt_overlap , prompt_overlap_prop , expand_overlap , expand_overlap_prop ) ) . transpose ( ) return prompt_arr . copy ( )

Generates prompt based features from an essay set object and internal prompt variable . Generally called internally by gen_feats Returns an array of prompt features e_set - EssaySet object


def update_prompt(self, prompt_text):
        """
        Update the default prompt string, which is "".
        prompt_text should be a string.
        Returns the prompt as a confirmation.
        """
        if(isinstance(prompt_text, basestring)):
            self._prompt = util_functions.sub_chars(prompt_text)
            ret = self._prompt
        else:
            raise util_functions.InputError(prompt_text, "Invalid prompt. Need to enter a string value.")
        return ret

Update the default prompt string which is . prompt_text should be a string . Returns the prompt as a confirmation .


def get_algorithms(algorithm):
    """
    Gets two classifiers for each type of algorithm, and returns them.  First for predicting, second for cv error.
    type - one of util_functions.AlgorithmTypes
    """
    if algorithm == util_functions.AlgorithmTypes.classification:
        clf = sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learn_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
        clf2=sklearn.ensemble.GradientBoostingClassifier(n_estimators=100, learn_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
    else:
        clf = sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learn_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
        clf2=sklearn.ensemble.GradientBoostingRegressor(n_estimators=100, learn_rate=.05,
            max_depth=4, random_state=1,min_samples_leaf=3)
    return clf, clf2

Gets two classifiers for each type of algorithm and returns them . First for predicting second for cv error . type - one of util_functions . AlgorithmTypes


def extract_features_and_generate_model_predictors(predictor_set, algorithm=util_functions.AlgorithmTypes.regression):
    """
    Extracts features and generates predictors based on a given predictor set
    predictor_set - a PredictorSet object that has been initialized with data
    type - one of util_functions.AlgorithmType
    """
    if(algorithm not in [util_functions.AlgorithmTypes.regression, util_functions.AlgorithmTypes.classification]):
        algorithm = util_functions.AlgorithmTypes.regression

    f = predictor_extractor.PredictorExtractor()
    f.initialize_dictionaries(predictor_set)

    train_feats = f.gen_feats(predictor_set)

    clf,clf2 = get_algorithms(algorithm)
    cv_error_results=get_cv_error(clf2,train_feats,predictor_set._target)

    try:
        set_score = numpy.asarray(predictor_set._target, dtype=numpy.int)
        clf.fit(train_feats, set_score)
    except ValueError:
        log.exception("Not enough classes (0,1,etc) in sample.")
        set_score = predictor_set._target
        set_score[0]=1
        set_score[1]=0
        clf.fit(train_feats, set_score)

    return f, clf, cv_error_results

Extracts features and generates predictors based on a given predictor set predictor_set - a PredictorSet object that has been initialized with data type - one of util_functions . AlgorithmType


def extract_features_and_generate_model(essays, algorithm=util_functions.AlgorithmTypes.regression):
    """
    Feed in an essay set to get feature vector and classifier
    essays must be an essay set object
    additional array is an optional argument that can specify
    a numpy array of values to add in
    returns a trained FeatureExtractor object and a trained classifier
    """
    f = feature_extractor.FeatureExtractor()
    f.initialize_dictionaries(essays)

    train_feats = f.gen_feats(essays)

    set_score = numpy.asarray(essays._score, dtype=numpy.int)
    if len(util_functions.f7(list(set_score)))>5:
        algorithm = util_functions.AlgorithmTypes.regression
    else:
        algorithm = util_functions.AlgorithmTypes.classification

    clf,clf2 = get_algorithms(algorithm)

    cv_error_results=get_cv_error(clf2,train_feats,essays._score)

    try:
        clf.fit(train_feats, set_score)
    except ValueError:
        log.exception("Not enough classes (0,1,etc) in sample.")
        set_score[0]=1
        set_score[1]=0
        clf.fit(train_feats, set_score)

    return f, clf, cv_error_results


def dump_model_to_file(prompt_string, feature_ext, classifier, text, score, model_path):
    """
    Writes out a model to a file.
    prompt string is a string containing the prompt
    feature_ext is a trained FeatureExtractor object
    classifier is a trained classifier
    model_path is the path of write out the model file to
    """
    model_file = {'prompt': prompt_string, 'extractor': feature_ext, 'model': classifier, 'text' : text, 'score' : score}
    pickle.dump(model_file, file=open(model_path, "w"))

def dump_model_to_file ( prompt_string , feature_ext , classifier , text , score , model_path ) : model_file = { 'prompt' : prompt_string , 'extractor' : feature_ext , 'model' : classifier , 'text' : text , 'score' : score } pickle . dump ( model_file , file = open ( model_path , "w" ) )

Writes out a model to a file . prompt string is a string containing the prompt feature_ext is a trained FeatureExtractor object classifier is a trained classifier model_path is the path of write out the model file to


def create_essay_set_and_dump_model(text,score,prompt,model_path,additional_array=None):
    """
    Function that creates essay set, extracts features, and writes out model
    See above functions for argument descriptions
    """
    essay_set=create_essay_set(text,score,prompt)
    feature_ext,clf=extract_features_and_generate_model(essay_set,additional_array)
    dump_model_to_file(prompt,feature_ext,clf,model_path)

def create_essay_set_and_dump_model ( text , score , prompt , model_path , additional_array = None ) : essay_set = create_essay_set ( text , score , prompt ) feature_ext , clf = extract_features_and_generate_model ( essay_set , additional_array ) dump_model_to_file ( prompt , feature_ext , clf , model_path )

Function that creates essay set extracts features and writes out model See above functions for argument descriptions


def initialize_dictionaries(self, p_set):
        """
        Initialize dictionaries with the textual inputs in the PredictorSet object
        p_set - PredictorSet object that has had data fed in
        """
        success = False
        if not (hasattr(p_set, '_type')):
            error_message = "needs to be an essay set of the train type."
            log.exception(error_message)
            raise util_functions.InputError(p_set, error_message)

        if not (p_set._type == "train"):
            error_message = "needs to be an essay set of the train type."
            log.exception(error_message)
            raise util_functions.InputError(p_set, error_message)

        div_length=len(p_set._essay_sets)
        if div_length==0:
            div_length=1

        #Ensures that even with a large amount of input textual features, training time stays reasonable
        max_feats2 = int(math.floor(200/div_length))
        for i in xrange(0,len(p_set._essay_sets)):
            self._extractors.append(FeatureExtractor())
            self._extractors[i].initialize_dictionaries(p_set._essay_sets[i], max_feats2=max_feats2)
            self._initialized = True
            success = True
        return success


def gen_feats(self, p_set):
        """
        Generates features based on an iput p_set
        p_set - PredictorSet
        """
        if self._initialized!=True:
            error_message = "Dictionaries have not been initialized."
            log.exception(error_message)
            raise util_functions.InputError(p_set, error_message)

        textual_features = []
        for i in xrange(0,len(p_set._essay_sets)):
            textual_features.append(self._extractors[i].gen_feats(p_set._essay_sets[i]))

        textual_matrix = numpy.concatenate(textual_features, axis=1)
        predictor_matrix = numpy.array(p_set._numeric_features)

        print textual_matrix.shape
        print predictor_matrix.shape

        overall_matrix = numpy.concatenate((textual_matrix, predictor_matrix), axis=1)

        return overall_matrix.copy()


def order_error(self, first_tag, second_tag, line):
        """Reports an OrderError. Error message will state that
        first_tag came before second_tag.
        """
        self.error = True
        msg = ERROR_MESSAGES['A_BEFORE_B'].format(first_tag, second_tag, line)
        self.logger.log(msg)

def order_error ( self , first_tag , second_tag , line ) : self . error = True msg = ERROR_MESSAGES [ 'A_BEFORE_B' ] . format ( first_tag , second_tag , line ) self . logger . log ( msg )

Reports an OrderError . Error message will state that first_tag came before second_tag .


def write_creation_info(creation_info, out):
    """
    Write the creation info to out.
    """
    out.write('# Creation Info\n\n')
    # Write sorted creators
    for creator in sorted(creation_info.creators):
        write_value('Creator', creator, out)

    # write created
    write_value('Created', creation_info.created_iso_format, out)
    # possible comment
    if creation_info.has_comment:
        write_text_value('CreatorComment', creation_info.comment, out)

def
write_creation_info
(
creation_info
,
out
)
:
out
.
write
(
'# Creation Info\n\n'
)
# Write sorted creators
for
creator
in
sorted
(
creation_info
.
creators
)
:
write_value
(
'Creator'
,
creator
,
out
)
# write created
write_value
(
'Created'
,
creation_info
.
created_iso_format
,
out
)
# possible comment
if
creation_info
.
has_comment
:
write_text_value
(
'CreatorComment'
,
creation_info
.
comment
,
out
)


def write_review(review, out):
    """
    Write the fields of a single review to out.
    """
    out.write('# Review\n\n')
    write_value('Reviewer', review.reviewer, out)
    write_value('ReviewDate', review.review_date_iso_format, out)
    if review.has_comment:
        write_text_value('ReviewComment', review.comment, out)

def
write_review
(
review
,
out
)
:
out
.
write
(
'# Review\n\n'
)
write_value
(
'Reviewer'
,
review
.
reviewer
,
out
)
write_value
(
'ReviewDate'
,
review
.
review_date_iso_format
,
out
)
if
review
.
has_comment
:
write_text_value
(
'ReviewComment'
,
review
.
comment
,
out
)

Write the fields of a single review to out .


def write_annotation(annotation, out):
    """
    Write the fields of a single annotation to out.
    """
    out.write('# Annotation\n\n')
    write_value('Annotator', annotation.annotator, out)
    write_value('AnnotationDate', annotation.annotation_date_iso_format, out)
    if annotation.has_comment:
        write_text_value('AnnotationComment', annotation.comment, out)
    write_value('AnnotationType', annotation.annotation_type, out)
    write_value('SPDXREF', annotation.spdx_id, out)

def
write_annotation
(
annotation
,
out
)
:
out
.
write
(
'# Annotation\n\n'
)
write_value
(
'Annotator'
,
annotation
.
annotator
,
out
)
write_value
(
'AnnotationDate'
,
annotation
.
annotation_date_iso_format
,
out
)
if
annotation
.
has_comment
:
write_text_value
(
'AnnotationComment'
,
annotation
.
comment
,
out
)
write_value
(
'AnnotationType'
,
annotation
.
annotation_type
,
out
)
write_value
(
'SPDXREF'
,
annotation
.
spdx_id
,
out
)

Write the fields of a single annotation to out .


def write_file(spdx_file, out):
    """
    Write a file fields to out.
    """
    out.write('# File\n\n')
    write_value('FileName', spdx_file.name, out)
    write_value('SPDXID', spdx_file.spdx_id, out)
    if spdx_file.has_optional_field('type'):
        write_file_type(spdx_file.type, out)
    write_value('FileChecksum', spdx_file.chk_sum.to_tv(), out)
    if isinstance(spdx_file.conc_lics, (document.LicenseConjunction, document.LicenseDisjunction)):
        write_value('LicenseConcluded', u'({0})'.format(spdx_file.conc_lics), out)
    else:
        write_value('LicenseConcluded', spdx_file.conc_lics, out)

    # write sorted list
    for lics in sorted(spdx_file.licenses_in_file):
        write_value('LicenseInfoInFile', lics, out)

    if isinstance(spdx_file.copyright, six.string_types):
        write_text_value('FileCopyrightText', spdx_file.copyright, out)
    else:
        write_value('FileCopyrightText', spdx_file.copyright, out)

    if spdx_file.has_optional_field('license_comment'):
        write_text_value('LicenseComments', spdx_file.license_comment, out)

    if spdx_file.has_optional_field('comment'):
        write_text_value('FileComment', spdx_file.comment, out)

    if spdx_file.has_optional_field('notice'):
        write_text_value('FileNotice', spdx_file.notice, out)

    for contributor in sorted(spdx_file.contributors):
        write_value('FileContributor', contributor, out)

    for dependency in sorted(spdx_file.dependencies):
        write_value('FileDependency', dependency, out)

    names = spdx_file.artifact_of_project_name
    homepages = spdx_file.artifact_of_project_home
    uris = spdx_file.artifact_of_project_uri

    for name, homepage, uri in sorted(zip_longest(names, homepages, uris)):
        write_value('ArtifactOfProjectName', name, out)
        if homepage is not None:
            write_value('ArtifactOfProjectHomePage', homepage, out)
        if uri is not None:
            write_value('ArtifactOfProjectURI', uri, out)

def
write_file
(
spdx_file
,
out
)
:
out
.
write
(
'# File\n\n'
)
write_value
(
'FileName'
,
spdx_file
.
name
,
out
)
write_value
(
'SPDXID'
,
spdx_file
.
spdx_id
,
out
)
if
spdx_file
.
has_optional_field
(
'type'
)
:
write_file_type
(
spdx_file
.
type
,
out
)
write_value
(
'FileChecksum'
,
spdx_file
.
chk_sum
.
to_tv
(
)
,
out
)
if
isinstance
(
spdx_file
.
conc_lics
,
(
document
.
LicenseConjunction
,
document
.
LicenseDisjunction
)
)
:
write_value
(
'LicenseConcluded'
,
u'({0})'
.
format
(
spdx_file
.
conc_lics
)
,
out
)
else
:
write_value
(
'LicenseConcluded'
,
spdx_file
.
conc_lics
,
out
)
# write sorted list
for
lics
in
sorted
(
spdx_file
.
licenses_in_file
)
:
write_value
(
'LicenseInfoInFile'
,
lics
,
out
)
if
isinstance
(
spdx_file
.
copyright
,
six
.
string_types
)
:
write_text_value
(
'FileCopyrightText'
,
spdx_file
.
copyright
,
out
)
else
:
write_value
(
'FileCopyrightText'
,
spdx_file
.
copyright
,
out
)
if
spdx_file
.
has_optional_field
(
'license_comment'
)
:
write_text_value
(
'LicenseComments'
,
spdx_file
.
license_comment
,
out
)
if
spdx_file
.
has_optional_field
(
'comment'
)
:
write_text_value
(
'FileComment'
,
spdx_file
.
comment
,
out
)
if
spdx_file
.
has_optional_field
(
'notice'
)
:
write_text_value
(
'FileNotice'
,
spdx_file
.
notice
,
out
)
for
contributor
in
sorted
(
spdx_file
.
contributors
)
:
write_value
(
'FileContributor'
,
contributor
,
out
)
for
dependency
in
sorted
(
spdx_file
.
dependencies
)
:
write_value
(
'FileDependency'
,
dependency
,
out
)
names
=
spdx_file
.
artifact_of_project_name
homepages
=
spdx_file
.
artifact_of_project_home
uris
=
spdx_file
.
artifact_of_project_uri
for
name
,
homepage
,
uri
in
sorted
(
zip_longest
(
names
,
homepages
,
uris
)
)
:
write_value
(
'ArtifactOfProjectName'
,
name
,
out
)
if
homepage
is
not
:
write_value
(
'ArtifactOfProjectHomePage'
,
homepage
,
out
)
if
uri
is
not
:
write_value
(
'ArtifactOfProjectURI'
,
uri
,
out
)


def write_package(package, out):
    """
    Write a package fields to out.
    """
    out.write('# Package\n\n')
    write_value('PackageName', package.name, out)
    if package.has_optional_field('version'):
        write_value('PackageVersion', package.version, out)
    write_value('PackageDownloadLocation', package.download_location, out)

    if package.has_optional_field('summary'):
        write_text_value('PackageSummary', package.summary, out)

    if package.has_optional_field('source_info'):
        write_text_value('PackageSourceInfo', package.source_info, out)

    if package.has_optional_field('file_name'):
        write_value('PackageFileName', package.file_name, out)

    if package.has_optional_field('supplier'):
        write_value('PackageSupplier', package.supplier, out)

    if package.has_optional_field('originator'):
        write_value('PackageOriginator', package.originator, out)

    if package.has_optional_field('check_sum'):
        write_value('PackageChecksum', package.check_sum.to_tv(), out)

    write_value('PackageVerificationCode', format_verif_code(package), out)

    if package.has_optional_field('description'):
        write_text_value('PackageDescription', package.description, out)

    if isinstance(package.license_declared, (document.LicenseConjunction,
        document.LicenseDisjunction)):
        write_value('PackageLicenseDeclared', u'({0})'.format(package.license_declared), out)
    else:
        write_value('PackageLicenseDeclared', package.license_declared, out)

    if isinstance(package.conc_lics, (document.LicenseConjunction,
        document.LicenseDisjunction)):
        write_value('PackageLicenseConcluded', u'({0})'.format(package.conc_lics), out)
    else:
        write_value('PackageLicenseConcluded', package.conc_lics, out)

    # Write sorted list of licenses.
    for lics in sorted(package.licenses_from_files):
        write_value('PackageLicenseInfoFromFiles', lics, out)

    if package.has_optional_field('license_comment'):
        write_text_value('PackageLicenseComments', package.license_comment, out)

    # cr_text is either free form text or NONE or NOASSERTION.
    if isinstance(package.cr_text, six.string_types):
        write_text_value('PackageCopyrightText', package.cr_text, out)
    else:
        write_value('PackageCopyrightText', package.cr_text, out)

    if package.has_optional_field('homepage'):
        write_value('PackageHomePage', package.homepage, out)

    # Write sorted files.
    for spdx_file in sorted(package.files):
        write_separators(out)
        write_file(spdx_file, out)


def write_extracted_licenses(lics, out):
    """
    Write extracted licenses fields to out.
    """
    write_value('LicenseID', lics.identifier, out)

    if lics.full_name is not None:
        write_value('LicenseName', lics.full_name, out)

    if lics.comment is not None:
        write_text_value('LicenseComment', lics.comment, out)

    for xref in sorted(lics.cross_ref):
        write_value('LicenseCrossReference', xref, out)

    write_text_value('ExtractedText', lics.text, out)

def write_extracted_licenses ( lics , out ) : write_value ( 'LicenseID' , lics . identifier , out ) if lics . full_name is not None : write_value ( 'LicenseName' , lics . full_name , out ) if lics . comment is not None : write_text_value ( 'LicenseComment' , lics . comment , out ) for xref in sorted ( lics . cross_ref ) : write_value ( 'LicenseCrossReference' , xref , out ) write_text_value ( 'ExtractedText' , lics . text , out )

Write extracted licenses fields to out .


def write_document(document, out, validate=True):
    """
    Write an SPDX tag value document.
    - document - spdx.document instance.
    - out - file like object that will be written to.
    Optionally `validate` the document before writing and raise
    InvalidDocumentError if document.validate returns False.
    """
    messages = []
    messages = document.validate(messages)
    if validate and messages:
        raise InvalidDocumentError(messages)

    # Write out document information
    out.write('# Document Information\n\n')
    write_value('SPDXVersion', str(document.version), out)
    write_value('DataLicense', document.data_license.identifier, out)
    write_value('DocumentName', document.name, out)
    write_value('SPDXID', 'SPDXRef-DOCUMENT', out)
    write_value('DocumentNamespace', document.namespace, out)
    if document.has_comment:
        write_text_value('DocumentComment', document.comment, out)
    for doc_ref in document.ext_document_references:
        doc_ref_str = ' '.join([doc_ref.external_document_id,
                                doc_ref.spdx_document_uri,
                                doc_ref.check_sum.identifier + ':' +
                                doc_ref.check_sum.value])
        write_value('ExternalDocumentRef', doc_ref_str, out)
    write_separators(out)
    # Write out creation info
    write_creation_info(document.creation_info, out)
    write_separators(out)

    # Writesorted reviews
    for review in sorted(document.reviews):
        write_review(review, out)
        write_separators(out)

    #Write sorted annotations
    for annotation in sorted(document.annotations):
        write_annotation(annotation, out)
        write_separators(out)

    # Write out package info
    write_package(document.package, out)
    write_separators(out)

    out.write('# Extracted Licenses\n\n')
    for lic in sorted(document.extracted_licenses):
        write_extracted_licenses(lic, out)
        write_separators(out)

def
write_document
(
document
,
out
,
validate
=
True
)
:
messages
=
[
]
messages
=
document
.
validate
(
messages
)
if
validate
and
messages
:
raise
InvalidDocumentError
(
messages
)
# Write out document information
out
.
write
(
'# Document Information\n\n'
)
write_value
(
'SPDXVersion'
,
str
(
document
.
version
)
,
out
)
write_value
(
'DataLicense'
,
document
.
data_license
.
identifier
,
out
)
write_value
(
'DocumentName'
,
document
.
name
,
out
)
write_value
(
'SPDXID'
,
'SPDXRef-DOCUMENT'
,
out
)
write_value
(
'DocumentNamespace'
,
document
.
namespace
,
out
)
if
document
.
has_comment
:
write_text_value
(
'DocumentComment'
,
document
.
comment
,
out
)
for
doc_ref
in
document
.
ext_document_references
:
doc_ref_str
=
' '
.
join
(
[
doc_ref
.
external_document_id
,
doc_ref
.
spdx_document_uri
,
doc_ref
.
check_sum
.
identifier
+
':'
+
doc_ref
.
check_sum
.
value
]
)
write_value
(
'ExternalDocumentRef'
,
doc_ref_str
,
out
)
write_separators
(
out
)
# Write out creation info
write_creation_info
(
document
.
creation_info
,
out
)
write_separators
(
out
)
# Writesorted reviews
for
review
in
sorted
(
document
.
reviews
)
:
write_review
(
review
,
out
)
write_separators
(
out
)
#Write sorted annotations
for
annotation
in
sorted
(
document
.
annotations
)
:
write_annotation
(
annotation
,
out
)
write_separators
(
out
)
# Write out package info
write_package
(
document
.
package
,
out
)
write_separators
(
out
)
out
.
write
(
'# Extracted Licenses\n\n'
)
for
lic
in
sorted
(
document
.
extracted_licenses
)
:
write_extracted_licenses
(
lic
,
out
)
write_separators
(
out
)

Write an SPDX tag value document . - document - spdx . document instance . - out - file like object that will be written to . Optionally validate the document before writing and raise InvalidDocumentError if document . validate returns False .


def checksum_from_sha1(value):
    """
    Return an spdx.checksum.Algorithm instance representing the SHA1
    checksum or None if does not match CHECKSUM_RE.
    """
    # More constrained regex at lexer level
    CHECKSUM_RE = re.compile('SHA1:\s*([\S]+)', re.UNICODE)
    match = CHECKSUM_RE.match(value)
    if match:
        return checksum.Algorithm(identifier='SHA1', value=match.group(1))
    else:
        return None

def
checksum_from_sha1
(
value
)
:
# More constrained regex at lexer level
CHECKSUM_RE
=
re
.
compile
(
'SHA1:\s*([\S]+)'
,
re
.
UNICODE
)
match
=
CHECKSUM_RE
.
match
(
value
)
if
match
:
return
checksum
.
Algorithm
(
identifier
=
'SHA1'
,
value
=
match
.
group
(
1
)
)
else
:
return

Return an spdx . checksum . Algorithm instance representing the SHA1 checksum or None if does not match CHECKSUM_RE .


def str_from_text(text):
    """
    Return content of a free form text block as a string.
    """
    REGEX = re.compile('<text>((.|\n)+)</text>', re.UNICODE)
    match = REGEX.match(text)
    if match:
        return match.group(1)
    else:
        return None

def str_from_text ( text ) : REGEX = re . compile ( '<text>((.|\n)+)</text>' , re . UNICODE ) match = REGEX . match ( text ) if match : return match . group ( 1 ) else : return None

Return content of a free form text block as a string .


def set_doc_version(self, doc, value):
        """
        Set the document version.
        Raise SPDXValueError if malformed value, CardinalityError
        if already defined
        """
        if not self.doc_version_set:
            self.doc_version_set = True
            m = self.VERS_STR_REGEX.match(value)
            if m is None:
                raise SPDXValueError('Document::Version')
            else:
                doc.version = version.Version(major=int(m.group(1)),
                                              minor=int(m.group(2)))
                return True
        else:
            raise CardinalityError('Document::Version')

def set_doc_version ( self , doc , value ) : if not self . doc_version_set : self . doc_version_set = True m = self . VERS_STR_REGEX . match ( value ) if m is None : raise SPDXValueError ( 'Document::Version' ) else : doc . version = version . Version ( major = int ( m . group ( 1 ) ) , minor = int ( m . group ( 2 ) ) ) return True else : raise CardinalityError ( 'Document::Version' )

Set the document version . Raise SPDXValueError if malformed value CardinalityError if already defined


def set_doc_data_lics(self, doc, lics):
        """Sets the document data license.
        Raises value error if malformed value, CardinalityError
        if already defined.
        """
        if not self.doc_data_lics_set:
            self.doc_data_lics_set = True
            if validations.validate_data_lics(lics):
                doc.data_license = document.License.from_identifier(lics)
                return True
            else:
                raise SPDXValueError('Document::DataLicense')
        else:
            raise CardinalityError('Document::DataLicense')

def set_doc_data_lics ( self , doc , lics ) : if not self . doc_data_lics_set : self . doc_data_lics_set = True if validations . validate_data_lics ( lics ) : doc . data_license = document . License . from_identifier ( lics ) return True else : raise SPDXValueError ( 'Document::DataLicense' ) else : raise CardinalityError ( 'Document::DataLicense' )

Sets the document data license . Raises value error if malformed value CardinalityError if already defined .


def set_doc_name(self, doc, name):
        """Sets the document name.
        Raises CardinalityError if already defined.
        """
        if not self.doc_name_set:
            doc.name = name
            self.doc_name_set = True
            return True
        else:
            raise CardinalityError('Document::Name')

def set_doc_name ( self , doc , name ) : if not self . doc_name_set : doc . name = name self . doc_name_set = True return True else : raise CardinalityError ( 'Document::Name' )

Sets the document name . Raises CardinalityError if already defined .


def set_doc_spdx_id(self, doc, doc_spdx_id_line):
        """Sets the document SPDX Identifier.
        Raises value error if malformed value, CardinalityError
        if already defined.
        """
        if not self.doc_spdx_id_set:
            if doc_spdx_id_line == 'SPDXRef-DOCUMENT':
                doc.spdx_id = doc_spdx_id_line
                self.doc_spdx_id_set = True
                return True
            else:
                raise SPDXValueError('Document::SPDXID')
        else:
            raise CardinalityError('Document::SPDXID')

def set_doc_spdx_id ( self , doc , doc_spdx_id_line ) : if not self . doc_spdx_id_set : if doc_spdx_id_line == 'SPDXRef-DOCUMENT' : doc . spdx_id = doc_spdx_id_line self . doc_spdx_id_set = True return True else : raise SPDXValueError ( 'Document::SPDXID' ) else : raise CardinalityError ( 'Document::SPDXID' )

Sets the document SPDX Identifier . Raises value error if malformed value CardinalityError if already defined .


def set_doc_comment(self, doc, comment):
        """Sets document comment, Raises CardinalityError if
        comment already set.
        Raises SPDXValueError if comment is not free form text.
        """
        if not self.doc_comment_set:
            self.doc_comment_set = True
            if validations.validate_doc_comment(comment):
                doc.comment = str_from_text(comment)
                return True
            else:
                raise SPDXValueError('Document::Comment')
        else:
            raise CardinalityError('Document::Comment')

def set_doc_comment ( self , doc , comment ) : if not self . doc_comment_set : self . doc_comment_set = True if validations . validate_doc_comment ( comment ) : doc . comment = str_from_text ( comment ) return True else : raise SPDXValueError ( 'Document::Comment' ) else : raise CardinalityError ( 'Document::Comment' )

Sets document comment Raises CardinalityError if comment already set . Raises SPDXValueError if comment is not free form text .


def set_doc_namespace(self, doc, namespace):
        """Sets the document namespace.
        Raise SPDXValueError if malformed value, CardinalityError
        if already defined.
        """
        if not self.doc_namespace_set:
            self.doc_namespace_set = True
            if validations.validate_doc_namespace(namespace):
                doc.namespace = namespace
                return True
            else:
                raise SPDXValueError('Document::Namespace')
        else:
            raise CardinalityError('Document::Comment')

def set_doc_namespace ( self , doc , namespace ) : if not self . doc_namespace_set : self . doc_namespace_set = True if validations . validate_doc_namespace ( namespace ) : doc . namespace = namespace return True else : raise SPDXValueError ( 'Document::Namespace' ) else : raise CardinalityError ( 'Document::Comment' )

Sets the document namespace . Raise SPDXValueError if malformed value CardinalityError if already defined .


def reset_document(self):
        """Resets the state to allow building new documents"""
        # FIXME: this state does not make sense
        self.doc_version_set = False
        self.doc_comment_set = False
        self.doc_namespace_set = False
        self.doc_data_lics_set = False
        self.doc_name_set = False
        self.doc_spdx_id_set = False

def
reset_document
(
self
)
:
# FIXME: this state does not make sense
self
.
doc_version_set
=
False
self
.
doc_comment_set
=
False
self
.
doc_namespace_set
=
False
self
.
doc_data_lics_set
=
False
self
.
doc_name_set
=
False
self
.
doc_spdx_id_set
=
False


def set_spdx_doc_uri(self, doc, spdx_doc_uri):
        """
        Sets the `spdx_document_uri` attribute of the `ExternalDocumentRef`
        object.
        """
        if validations.validate_doc_namespace(spdx_doc_uri):
            doc.ext_document_references[-1].spdx_document_uri = spdx_doc_uri
        else:
            raise SPDXValueError('Document::ExternalDocumentRef')

def set_spdx_doc_uri ( self , doc , spdx_doc_uri ) : if validations . validate_doc_namespace ( spdx_doc_uri ) : doc . ext_document_references [ - 1 ] . spdx_document_uri = spdx_doc_uri else : raise SPDXValueError ( 'Document::ExternalDocumentRef' )

Sets the spdx_document_uri attribute of the ExternalDocumentRef object .


def build_tool(self, doc, entity):
        """Builds a tool object out of a string representation.
        Returns built tool. Raises SPDXValueError if failed to extract
        tool name or name is malformed
        """
        match = self.tool_re.match(entity)
        if match and validations.validate_tool_name(match.group(self.TOOL_NAME_GROUP)):
            name = match.group(self.TOOL_NAME_GROUP)
            return creationinfo.Tool(name)
        else:
            raise SPDXValueError('Failed to extract tool name')

def
build_tool
(
self
,
doc
,
entity
)
:
match
=
self
.
tool_re
.
match
(
entity
)
if
match
and
validations
.
validate_tool_name
(
match
.
group
(
self
.
TOOL_NAME_GROUP
)
)
:
name
=
match
.
group
(
self
.
TOOL_NAME_GROUP
)
return
creationinfo
.
Tool
(
name
)
else
:
raise
SPDXValueError
(
'Failed to extract tool name'
)

Builds a tool object out of a string representation . Returns built tool . Raises SPDXValueError if failed to extract tool name or name is malformed


def add_creator(self, doc, creator):
        """Adds a creator to the document's creation info.
        Returns true if creator is valid.
        Creator must be built by an EntityBuilder.
        Raises SPDXValueError if not a creator type.
        """
        if validations.validate_creator(creator):
            doc.creation_info.add_creator(creator)
            return True
        else:
            raise SPDXValueError('CreationInfo::Creator')

def add_creator ( self , doc , creator ) : if validations . validate_creator ( creator ) : doc . creation_info . add_creator ( creator ) return True else : raise SPDXValueError ( 'CreationInfo::Creator' )

Adds a creator to the document s creation info . Returns true if creator is valid . Creator must be built by an EntityBuilder . Raises SPDXValueError if not a creator type .


def set_created_date(self, doc, created):
        """Sets created date, Raises CardinalityError if
        created date already set.
        Raises SPDXValueError if created is not a date.
        """
        if not self.created_date_set:
            self.created_date_set = True
            date = utils.datetime_from_iso_format(created)
            if date is not None:
                doc.creation_info.created = date
                return True
            else:
                raise SPDXValueError('CreationInfo::Date')
        else:
            raise CardinalityError('CreationInfo::Created')

def set_created_date ( self , doc , created ) : if not self . created_date_set : self . created_date_set = True date = utils . datetime_from_iso_format ( created ) if date is not None : doc . creation_info . created = date return True else : raise SPDXValueError ( 'CreationInfo::Date' ) else : raise CardinalityError ( 'CreationInfo::Created' )

Sets created date Raises CardinalityError if created date already set . Raises SPDXValueError if created is not a date .


def set_lics_list_ver(self, doc, value):
        """Sets the license list version, Raises CardinalityError if
        already set, SPDXValueError if incorrect value.
        """
        if not self.lics_list_ver_set:
            self.lics_list_ver_set = True
            vers = version.Version.from_str(value)
            if vers is not None:
                doc.creation_info.license_list_version = vers
                return True
            else:
                raise SPDXValueError('CreationInfo::LicenseListVersion')
        else:
            raise CardinalityError('CreationInfo::LicenseListVersion')

def set_lics_list_ver ( self , doc , value ) : if not self . lics_list_ver_set : self . lics_list_ver_set = True vers = version . Version . from_str ( value ) if vers is not None : doc . creation_info . license_list_version = vers return True else : raise SPDXValueError ( 'CreationInfo::LicenseListVersion' ) else : raise CardinalityError ( 'CreationInfo::LicenseListVersion' )

Sets the license list version Raises CardinalityError if already set SPDXValueError if incorrect value .


def reset_creation_info(self):
        """
        Resets builder state to allow building new creation info."""
        # FIXME: this state does not make sense
        self.created_date_set = False
        self.creation_comment_set = False
        self.lics_list_ver_set = False

def
reset_creation_info
(
self
)
:
# FIXME: this state does not make sense
self
.
created_date_set
=
False
self
.
creation_comment_set
=
False
self
.
lics_list_ver_set
=
False

Resets builder state to allow building new creation info .


def add_reviewer(self, doc, reviewer):
        """Adds a reviewer to the SPDX Document.
        Reviwer is an entity created by an EntityBuilder.
        Raises SPDXValueError if not a valid reviewer type.
        """
        # Each reviewer marks the start of a new review object.
        # FIXME: this state does not make sense
        self.reset_reviews()
        if validations.validate_reviewer(reviewer):
            doc.add_review(review.Review(reviewer=reviewer))
            return True
        else:
            raise SPDXValueError('Review::Reviewer')

Adds a reviewer to the SPDX Document . Reviwer is an entity created by an EntityBuilder . Raises SPDXValueError if not a valid reviewer type .


def add_review_date(self, doc, reviewed):
        """Sets the review date. Raises CardinalityError if
        already set. OrderError if no reviewer defined before.
        Raises SPDXValueError if invalid reviewed value.
        """
        if len(doc.reviews) != 0:
            if not self.review_date_set:
                self.review_date_set = True
                date = utils.datetime_from_iso_format(reviewed)
                if date is not None:
                    doc.reviews[-1].review_date = date
                    return True
                else:
                    raise SPDXValueError('Review::ReviewDate')
            else:
                raise CardinalityError('Review::ReviewDate')
        else:
            raise OrderError('Review::ReviewDate')

def add_review_date ( self , doc , reviewed ) : if len ( doc . reviews ) != 0 : if not self . review_date_set : self . review_date_set = True date = utils . datetime_from_iso_format ( reviewed ) if date is not None : doc . reviews [ - 1 ] . review_date = date return True else : raise SPDXValueError ( 'Review::ReviewDate' ) else : raise CardinalityError ( 'Review::ReviewDate' ) else : raise OrderError ( 'Review::ReviewDate' )

Sets the review date . Raises CardinalityError if already set . OrderError if no reviewer defined before . Raises SPDXValueError if invalid reviewed value .


def add_review_comment(self, doc, comment):
        """Sets the review comment. Raises CardinalityError if
        already set. OrderError if no reviewer defined before.
        Raises SPDXValueError if comment is not free form text.
        """
        if len(doc.reviews) != 0:
            if not self.review_comment_set:
                self.review_comment_set = True
                if validations.validate_review_comment(comment):
                    doc.reviews[-1].comment = str_from_text(comment)
                    return True
                else:
                    raise SPDXValueError('ReviewComment::Comment')
            else:
                raise CardinalityError('ReviewComment')
        else:
            raise OrderError('ReviewComment')

def add_review_comment ( self , doc , comment ) : if len ( doc . reviews ) != 0 : if not self . review_comment_set : self . review_comment_set = True if validations . validate_review_comment ( comment ) : doc . reviews [ - 1 ] . comment = str_from_text ( comment ) return True else : raise SPDXValueError ( 'ReviewComment::Comment' ) else : raise CardinalityError ( 'ReviewComment' ) else : raise OrderError ( 'ReviewComment' )

Sets the review comment . Raises CardinalityError if already set . OrderError if no reviewer defined before . Raises SPDXValueError if comment is not free form text .


def reset_annotations(self):
        """Resets the builder's state to allow building new annotations."""
        # FIXME: this state does not make sense
        self.annotation_date_set = False
        self.annotation_comment_set = False
        self.annotation_type_set = False
        self.annotation_spdx_id_set = False

def
reset_annotations
(
self
)
:
# FIXME: this state does not make sense
self
.
annotation_date_set
=
False
self
.
annotation_comment_set
=
False
self
.
annotation_type_set
=
False
self
.
annotation_spdx_id_set
=
False

Resets the builder s state to allow building new annotations .


def add_annotator(self, doc, annotator):
        """Adds an annotator to the SPDX Document.
        Annotator is an entity created by an EntityBuilder.
        Raises SPDXValueError if not a valid annotator type.
        """
        # Each annotator marks the start of a new annotation object.
        # FIXME: this state does not make sense
        self.reset_annotations()
        if validations.validate_annotator(annotator):
            doc.add_annotation(annotation.Annotation(annotator=annotator))
            return True
        else:
            raise SPDXValueError('Annotation::Annotator')

Adds an annotator to the SPDX Document . Annotator is an entity created by an EntityBuilder . Raises SPDXValueError if not a valid annotator type .


def add_annotation_date(self, doc, annotation_date):
        """Sets the annotation date. Raises CardinalityError if
        already set. OrderError if no annotator defined before.
        Raises SPDXValueError if invalid value.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_date_set:
                self.annotation_date_set = True
                date = utils.datetime_from_iso_format(annotation_date)
                if date is not None:
                    doc.annotations[-1].annotation_date = date
                    return True
                else:
                    raise SPDXValueError('Annotation::AnnotationDate')
            else:
                raise CardinalityError('Annotation::AnnotationDate')
        else:
            raise OrderError('Annotation::AnnotationDate')

def add_annotation_date ( self , doc , annotation_date ) : if len ( doc . annotations ) != 0 : if not self . annotation_date_set : self . annotation_date_set = True date = utils . datetime_from_iso_format ( annotation_date ) if date is not None : doc . annotations [ - 1 ] . annotation_date = date return True else : raise SPDXValueError ( 'Annotation::AnnotationDate' ) else : raise CardinalityError ( 'Annotation::AnnotationDate' ) else : raise OrderError ( 'Annotation::AnnotationDate' )

Sets the annotation date . Raises CardinalityError if already set . OrderError if no annotator defined before . Raises SPDXValueError if invalid value .


def add_annotation_comment(self, doc, comment):
        """Sets the annotation comment. Raises CardinalityError if
        already set. OrderError if no annotator defined before.
        Raises SPDXValueError if comment is not free form text.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_comment_set:
                self.annotation_comment_set = True
                if validations.validate_annotation_comment(comment):
                    doc.annotations[-1].comment = str_from_text(comment)
                    return True
                else:
                    raise SPDXValueError('AnnotationComment::Comment')
            else:
                raise CardinalityError('AnnotationComment::Comment')
        else:
            raise OrderError('AnnotationComment::Comment')

def add_annotation_comment ( self , doc , comment ) : if len ( doc . annotations ) != 0 : if not self . annotation_comment_set : self . annotation_comment_set = True if validations . validate_annotation_comment ( comment ) : doc . annotations [ - 1 ] . comment = str_from_text ( comment ) return True else : raise SPDXValueError ( 'AnnotationComment::Comment' ) else : raise CardinalityError ( 'AnnotationComment::Comment' ) else : raise OrderError ( 'AnnotationComment::Comment' )

Sets the annotation comment . Raises CardinalityError if already set . OrderError if no annotator defined before . Raises SPDXValueError if comment is not free form text .


def add_annotation_type(self, doc, annotation_type):
        """Sets the annotation type. Raises CardinalityError if
        already set. OrderError if no annotator defined before.
        Raises SPDXValueError if invalid value.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_type_set:
                self.annotation_type_set = True
                if validations.validate_annotation_type(annotation_type):
                    doc.annotations[-1].annotation_type = annotation_type
                    return True
                else:
                    raise SPDXValueError('Annotation::AnnotationType')
            else:
                raise CardinalityError('Annotation::AnnotationType')
        else:
            raise OrderError('Annotation::AnnotationType')

def add_annotation_type ( self , doc , annotation_type ) : if len ( doc . annotations ) != 0 : if not self . annotation_type_set : self . annotation_type_set = True if validations . validate_annotation_type ( annotation_type ) : doc . annotations [ - 1 ] . annotation_type = annotation_type return True else : raise SPDXValueError ( 'Annotation::AnnotationType' ) else : raise CardinalityError ( 'Annotation::AnnotationType' ) else : raise OrderError ( 'Annotation::AnnotationType' )

Sets the annotation type . Raises CardinalityError if already set . OrderError if no annotator defined before . Raises SPDXValueError if invalid value .


def set_annotation_spdx_id(self, doc, spdx_id):
        """Sets the annotation SPDX Identifier.
        Raises CardinalityError if already set. OrderError if no annotator
        defined before.
        """
        if len(doc.annotations) != 0:
            if not self.annotation_spdx_id_set:
                self.annotation_spdx_id_set = True
                doc.annotations[-1].spdx_id = spdx_id
                return True
            else:
                raise CardinalityError('Annotation::SPDXREF')
        else:
            raise OrderError('Annotation::SPDXREF')

def set_annotation_spdx_id ( self , doc , spdx_id ) : if len ( doc . annotations ) != 0 : if not self . annotation_spdx_id_set : self . annotation_spdx_id_set = True doc . annotations [ - 1 ] . spdx_id = spdx_id return True else : raise CardinalityError ( 'Annotation::SPDXREF' ) else : raise OrderError ( 'Annotation::SPDXREF' )

Sets the annotation SPDX Identifier . Raises CardinalityError if already set . OrderError if no annotator defined before .


def reset_package(self):
        """Resets the builder's state in order to build new packages."""
        # FIXME: this state does not make sense
        self.package_set = False
        self.package_vers_set = False
        self.package_file_name_set = False
        self.package_supplier_set = False
        self.package_originator_set = False
        self.package_down_location_set = False
        self.package_home_set = False
        self.package_verif_set = False
        self.package_chk_sum_set = False
        self.package_source_info_set = False
        self.package_conc_lics_set = False
        self.package_license_declared_set = False
        self.package_license_comment_set = False
        self.package_cr_text_set = False
        self.package_summary_set = False
        self.package_desc_set = False

def
reset_package
(
self
)
:
# FIXME: this state does not make sense
self
.
package_set
=
False
self
.
package_vers_set
=
False
self
.
package_file_name_set
=
False
self
.
package_supplier_set
=
False
self
.
package_originator_set
=
False
self
.
package_down_location_set
=
False
self
.
package_home_set
=
False
self
.
package_verif_set
=
False
self
.
package_chk_sum_set
=
False
self
.
package_source_info_set
=
False
self
.
package_conc_lics_set
=
False
self
.
package_license_declared_set
=
False
self
.
package_license_comment_set
=
False
self
.
package_cr_text_set
=
False
self
.
package_summary_set
=
False
self
.
package_desc_set
=
False

Resets the builder s state in order to build new packages .


def create_package(self, doc, name):
        """Creates a package for the SPDX Document.
        name - any string.
        Raises CardinalityError if package already defined.
        """
        if not self.package_set:
            self.package_set = True
            doc.package = package.Package(name=name)
            return True
        else:
            raise CardinalityError('Package::Name')

def create_package ( self , doc , name ) : if not self . package_set : self . package_set = True doc . package = package . Package ( name = name ) return True else : raise CardinalityError ( 'Package::Name' )

Creates a package for the SPDX Document . name - any string . Raises CardinalityError if package already defined .


def set_pkg_vers(self, doc, version):
        """Sets package version, if not already set.
        version - Any string.
        Raises CardinalityError if already has a version.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_vers_set:
            self.package_vers_set = True
            doc.package.version = version
            return True
        else:
            raise CardinalityError('Package::Version')

def set_pkg_vers ( self , doc , version ) : self . assert_package_exists ( ) if not self . package_vers_set : self . package_vers_set = True doc . package . version = version return True else : raise CardinalityError ( 'Package::Version' )

Sets package version if not already set . version - Any string . Raises CardinalityError if already has a version . Raises OrderError if no package previously defined .


def set_pkg_file_name(self, doc, name):
        """Sets the package file name, if not already set.
        name - Any string.
        Raises CardinalityError if already has a file_name.
        Raises OrderError if no pacakge previously defined.
        """
        self.assert_package_exists()
        if not self.package_file_name_set:
            self.package_file_name_set = True
            doc.package.file_name = name
            return True
        else:
            raise CardinalityError('Package::FileName')

def set_pkg_file_name ( self , doc , name ) : self . assert_package_exists ( ) if not self . package_file_name_set : self . package_file_name_set = True doc . package . file_name = name return True else : raise CardinalityError ( 'Package::FileName' )

Sets the package file name if not already set . name - Any string . Raises CardinalityError if already has a file_name . Raises OrderError if no pacakge previously defined .


def set_pkg_supplier(self, doc, entity):
        """Sets the package supplier, if not already set.
        entity - Organization, Person or NoAssert.
        Raises CardinalityError if already has a supplier.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_supplier_set:
            self.package_supplier_set = True
            if validations.validate_pkg_supplier(entity):
                doc.package.supplier = entity
                return True
            else:
                raise SPDXValueError('Package::Supplier')
        else:
            raise CardinalityError('Package::Supplier')

def set_pkg_supplier ( self , doc , entity ) : self . assert_package_exists ( ) if not self . package_supplier_set : self . package_supplier_set = True if validations . validate_pkg_supplier ( entity ) : doc . package . supplier = entity return True else : raise SPDXValueError ( 'Package::Supplier' ) else : raise CardinalityError ( 'Package::Supplier' )

Sets the package supplier if not already set . entity - Organization Person or NoAssert . Raises CardinalityError if already has a supplier . Raises OrderError if no package previously defined .


def set_pkg_originator(self, doc, entity):
        """Sets the package originator, if not already set.
        entity - Organization, Person or NoAssert.
        Raises CardinalityError if already has an originator.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_originator_set:
            self.package_originator_set = True
            if validations.validate_pkg_originator(entity):
                doc.package.originator = entity
                return True
            else:
                raise SPDXValueError('Package::Originator')
        else:
            raise CardinalityError('Package::Originator')

def set_pkg_originator ( self , doc , entity ) : self . assert_package_exists ( ) if not self . package_originator_set : self . package_originator_set = True if validations . validate_pkg_originator ( entity ) : doc . package . originator = entity return True else : raise SPDXValueError ( 'Package::Originator' ) else : raise CardinalityError ( 'Package::Originator' )

Sets the package originator if not already set . entity - Organization Person or NoAssert . Raises CardinalityError if already has an originator . Raises OrderError if no package previously defined .


def set_pkg_down_location(self, doc, location):
        """Sets the package download location, if not already set.
        location - A string
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        """
        self.assert_package_exists()
        if not self.package_down_location_set:
            self.package_down_location_set = True
            doc.package.download_location = location
            return True
        else:
            raise CardinalityError('Package::DownloadLocation')

def set_pkg_down_location ( self , doc , location ) : self . assert_package_exists ( ) if not self . package_down_location_set : self . package_down_location_set = True doc . package . download_location = location return True else : raise CardinalityError ( 'Package::DownloadLocation' )

Sets the package download location if not already set . location - A string Raises CardinalityError if already defined . Raises OrderError if no package previously defined .


def set_pkg_home(self, doc, location):
        """Sets the package homepage location if not already set.
        location - A string or None or NoAssert.
        Raises CardinalityError if already defined.
        Raises OrderError if no package previously defined.
        Raises SPDXValueError if location has incorrect value.
        """
        self.assert_package_exists()
        if not self.package_home_set:
            self.package_home_set = True
            if validations.validate_pkg_homepage(location):
                doc.package.homepage = location
                return True
            else:
                raise SPDXValueError('Package::HomePage')
        else:
            raise CardinalityError('Package::HomePage')

def set_pkg_home ( self , doc , location ) : self . assert_package_exists ( ) if not self . package_home_set : self . package_home_set = True if validations . validate_pkg_homepage ( location ) : doc . package . homepage = location return True else : raise SPDXValueError ( 'Package::HomePage' ) else : raise CardinalityError ( 'Package::HomePage' )

Sets the package homepage location if not already set . location - A string or None or NoAssert . Raises CardinalityError if already defined . Raises OrderError if no package previously defined . Raises SPDXValueError if location has incorrect value .