    def encode(s):
    def ensure_int(arg, msg=None):
    def log(level, msg):
    def __init__(self):
    def __getitem__(self, key):
    def __len__(self):
    def __setitem__(self, key, item):
    def __delitem__(self, key):
    def set_citation_data(self, citation_data):
    def as_txt(self):
    def as_csv(self, header=False, sep='|'):
    def as_citation(self):
    def __init__(self, site=None):
    def handle_article(self, art):
    def handle_num_results(self, num_results):
    def parse(self, html):
    def _clean_article(self):
    def _parse_globals(self):
    def _parse_article(self, div):
    def _parse_links(self, span):
    def _tag_has_class(tag, klass):
    def _tag_results_checker(tag):
    def _as_int(obj):
    def _path2url(self, path):
    def _strip_url_arg(self, arg, url):
    def _parse_article(self, div):
    def _parse_article(self, div):
    def __init__(self):
    def set_num_page_results(self, num_page_results):
    def get_url(self):
    def _add_attribute_type(self, key, label, default_value=None):
    def __getitem__(self, key):
    def __setitem__(self, key, item):
    def _parenthesize_phrases(self, query):
    def __init__(self, cluster=None):
    def set_cluster(self, cluster):
    def get_url(self):
    def __init__(self):
    def set_words(self, words):
    def set_words_some(self, words):
    def set_words_none(self, words):
    def set_phrase(self, phrase):
    def set_scope(self, title_only):
    def set_author(self, author):
    def set_pub(self, pub):
    def set_timeframe(self, start=None, end=None):
    def set_include_citations(self, yesorno):
    def set_include_patents(self, yesorno):
    def get_url(self):
    def __init__(self):
    def set_citation_format(self, citform):
    def set_per_page_results(self, per_page_results):
    def is_configured(self):
        def __init__(self, querier):
        def handle_num_results(self, num_results):
        def handle_article(self, art):
    def __init__(self):
    def apply_settings(self, settings):
    def send_query(self, query):
    def get_citation_data(self, article):
    def parse(self, html):
    def add_article(self, art):
    def clear_articles(self):
    def save_cookies(self):
    def _get_http_response(self, url, log_msg=None, err_msg=None):
def txt(querier, with_globals):
def csv(querier, header=False, sep='|'):
def citation_export(querier):
def main():
class Error(Exception):
class FormatError(Error):
class QueryArgumentError(Error):
class ScholarConf(object):
class ScholarUtils(object):
class ScholarArticle(object):
class ScholarArticleParser(object):
class ScholarArticleParser120201(ScholarArticleParser):
class ScholarArticleParser120726(ScholarArticleParser):
class ScholarQuery(object):
class ClusterScholarQuery(ScholarQuery):
class SearchScholarQuery(ScholarQuery):
class ScholarSettings(object):
class ScholarQuerier(object):
    class Parser(ScholarArticleParser120726):
    VERSION = '2.10'
    LOG_LEVEL = 3
    MAX_PAGE_RESULTS = 20 # Current maximum for per-page results
    SCHOLAR_SITE = 'http://scholar.google.com'
    USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'
    COOKIE_JAR_FILE = None
    LOG_LEVELS = {'error': 1,
                  'warn':  2,
                  'info':  3,
                  'debug': 4}
    SCHOLAR_CLUSTER_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
        + 'cluster=%(cluster)s' \
        + '&num=%(num)s'
    SCHOLAR_QUERY_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
        + 'as_q=%(words)s' \
        + '&as_epq=%(phrase)s' \
        + '&as_oq=%(words_some)s' \
        + '&as_eq=%(words_none)s' \
        + '&as_occt=%(scope)s' \
        + '&as_sauthors=%(authors)s' \
        + '&as_publication=%(pub)s' \
        + '&as_ylo=%(ylo)s' \
        + '&as_yhi=%(yhi)s' \
        + '&as_sdt=%(patents)s%%2C5' \
        + '&as_vis=%(citations)s' \
        + '&btnG=&hl=en' \
        + '&num=%(num)s'
    CITFORM_NONE = 0
    CITFORM_REFWORKS = 1
    CITFORM_REFMAN = 2
    CITFORM_ENDNOTE = 3
    CITFORM_BIBTEX = 4
    GET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_settings?' \
        + 'sciifh=1&hl=en&as_sdt=0,5'
    SET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_setprefs?' \
        + 'q=' \
        + '&scisig=%(scisig)s' \
        + '&inststart=0' \
        + '&as_sdt=1,5' \
        + '&as_sdtp=' \
        + '&num=%(num)s' \
        + '&scis1=%(scis)s' \
        + '%(scisf)s' \
        + '&hl=en&lang=all&instq=&inst=569367360547434339&save=' class Error(Exception):
    """Base class for any Scholar error.""" class FormatError(Error):
    """A query argument or setting was formatted incorrectly.""" class QueryArgumentError(Error):
    """A query did not have a suitable set of arguments.""" class ScholarConf(object):
    """Helper class for global settings."""

    VERSION = '2.10'
    LOG_LEVEL = 3
    MAX_PAGE_RESULTS = 20 # Current maximum for per-page results
    SCHOLAR_SITE = 'http://scholar.google.com'

    # USER_AGENT = 'Mozilla/5.0 (X11; U; FreeBSD i386; en-US; rv:1.9.2.9) Gecko/20100913 Firefox/3.6.9'
    # Let's update at this point (3/14):
    USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'

    # If set, we will use this file to read/save cookies to enable
    # cookie use across sessions.
    COOKIE_JAR_FILE = None class ScholarUtils(object):
    """A wrapper for various utensils that come in handy."""

    LOG_LEVELS = {'error': 1,
                  'warn':  2,
                  'info':  3,
                  'debug': 4}

    @staticmethod
    def ensure_int(arg, msg=None):
        try:
            return int(arg)
        except ValueError:
            raise FormatError(msg)

    @staticmethod
    def log(level, msg):
        if level not in ScholarUtils.LOG_LEVELS.keys():
            return
        if ScholarUtils.LOG_LEVELS[level] > ScholarConf.LOG_LEVEL:
            return
        sys.stderr.write('[%5s]  %s' % (level.upper(), msg + '\n'))
        sys.stderr.flush() class ScholarArticle(object):
    """
    A class representing articles listed on Google Scholar.  The class
    provides basic dictionary-like behavior.
    """
    def __init__(self):
        # The triplets for each keyword correspond to (1) the actual
        # value, (2) a user-suitable label for the item, and (3) an
        # ordering index:
        self.attrs = {
            'title':         [None, 'Title',          0],
            'url':           [None, 'URL',            1],
            'year':          [None, 'Year',           2],
            'num_citations': [0,    'Citations',      3],
            'num_versions':  [0,    'Versions',       4],
            'cluster_id':    [None, 'Cluster ID',     5],
            'url_pdf':       [None, 'PDF link',       6],
            'url_citations': [None, 'Citations list', 7],
            'url_versions':  [None, 'Versions list',  8],
            'url_citation':  [None, 'Citation link',  9],
            'excerpt':       [None, 'Excerpt',       10],
        }

        # The citation data in one of the standard export formats,
        # e.g. BibTeX.
        self.citation_data = None

    def __getitem__(self, key):
        if key in self.attrs:
            return self.attrs[key][0]
        return None

    def __len__(self):
        return len(self.attrs)

    def __setitem__(self, key, item):
        if key in self.attrs:
            self.attrs[key][0] = item
        else:
            self.attrs[key] = [item, key, len(self.attrs)]

    def __delitem__(self, key):
        if key in self.attrs:
            del self.attrs[key]

    def set_citation_data(self, citation_data):
        self.citation_data = citation_data

    def as_txt(self):
        # Get items sorted in specified order:
        items = sorted(list(self.attrs.values()), key=lambda item: item[2])
        # Find largest label length:
        max_label_len = max([len(str(item[1])) for item in items])
        fmt = '%%%ds %%s' % max_label_len
        res = []
        for item in items:
            if item[0] is not None:
                res.append(fmt % (item[1], item[0]))
        return '\n'.join(res)

    def as_csv(self, header=False, sep='|'):
        # Get keys sorted in specified order:
        keys = [pair[0] for pair in \
                sorted([(key, val[2]) for key, val in list(self.attrs.items())],
                       key=lambda pair: pair[1])]
        res = []
        if header:
            res.append(sep.join(keys))
        res.append(sep.join([unicode(self.attrs[key][0]) for key in keys]))
        return '\n'.join(res)

    def as_citation(self):
        """
        Reports the article in a standard citation format. This works only
        if you have configured the querier to retrieve a particular
        citation export format. (See ScholarSettings.)
        """
        return self.citation_data or b'' class ScholarArticleParser120726(ScholarArticleParser):
    """
    This class reflects update to the Scholar results page layout that
    Google made 07/26/12.
    """
    def _parse_article(self, div):
        self.article = ScholarArticle()
        self.article.div = div

        for tag in div:
            if not getattr(tag, 'name'):
                continue
            if str(tag).lower().find('.pdf'):
                if tag.find('div', {'class': 'gs_ttss'}):
                    self._parse_links(tag.find('div', {'class': 'gs_ttss'}))

            if tag.name == 'div' and self._tag_has_class(tag, 'gs_ri'):
                # There are (at least) two formats here. In the first
                # one, we have a link, e.g.:
                #
                # <h3 class="gs_rt">
                #   <a href="http://dl.acm.org/citation.cfm?id=972384" class="yC0">
                #     <b>Honeycomb</b>: creating intrusion detection signatures using
                #        honeypots
                #   </a>
                # </h3>
                #
                # In the other, there's no actual link -- it's what
                # Scholar renders as "CITATION" in the HTML:
                #
                # <h3 class="gs_rt">
                #   <span class="gs_ctu">
                #     <span class="gs_ct1">[CITATION]</span>
                #     <span class="gs_ct2">[C]</span>
                #   </span>
                #   <b>Honeycomb</b> automated ids signature creation using honeypots
                # </h3>
                #
                # We now distinguish the two.
                try:
                    atag = tag.h3.a
                    self.article['title'] = ''.join(atag.findAll(text=True))
                    self.article['url'] = self._path2url(atag['href'])
                    if self.article['url'].endswith('.pdf'):
                        self.article['url_pdf'] = self.article['url']
                except:
                    # Remove a few spans that have unneeded content (e.g. [CITATION])
                    for span in tag.h3.findAll(name='span'):
                        span.clear()
                    self.article['title'] = ''.join(tag.h3.findAll(text=True))

                if tag.find('div', {'class': 'gs_a'}):
                    year = self.year_re.findall(tag.find('div', {'class': 'gs_a'}).text)
                    self.article['year'] = year[0] if len(year) > 0 else None

                if tag.find('div', {'class': 'gs_fl'}):
                    self._parse_links(tag.find('div', {'class': 'gs_fl'}))

                if tag.find('div', {'class': 'gs_rs'}):
                    # These are the content excerpts rendered into the results.
                    raw_text = tag.find('div', {'class': 'gs_rs'}).findAll(text=True)
                    if len(raw_text) > 0:
                        raw_text = ''.join(raw_text)
                        raw_text = raw_text.replace('\n', '')
                        self.article['excerpt'] = raw_text     def encode(s):
    def ensure_int(arg, msg=None):
    def log(level, msg):
    def __init__(self):
    def __getitem__(self, key):
    def __len__(self):
    def __setitem__(self, key, item):
    def __delitem__(self, key):
    def set_citation_data(self, citation_data):
    def as_txt(self):
    def as_csv(self, header=False, sep='|'):
    def as_citation(self):
    def __init__(self, site=None):
    def handle_article(self, art):
    def handle_num_results(self, num_results):
    def parse(self, html):
    def _clean_article(self):
    def _parse_globals(self):
    def _parse_article(self, div):
    def _parse_links(self, span):
    def _tag_has_class(tag, klass):
    def _tag_results_checker(tag):
    def _as_int(obj):
    def _path2url(self, path):
    def _strip_url_arg(self, arg, url):
    def _parse_article(self, div):
    def _parse_article(self, div):
    def __init__(self):
    def set_num_page_results(self, num_page_results):
    def get_url(self):
    def _add_attribute_type(self, key, label, default_value=None):
    def __getitem__(self, key):
    def __setitem__(self, key, item):
    def _parenthesize_phrases(self, query):
    def __init__(self, cluster=None):
    def set_cluster(self, cluster):
    def get_url(self):
    def __init__(self):
    def set_words(self, words):
    def set_words_some(self, words):
    def set_words_none(self, words):
    def set_phrase(self, phrase):
    def set_scope(self, title_only):
    def set_author(self, author):
    def set_pub(self, pub):
    def set_timeframe(self, start=None, end=None):
    def set_include_citations(self, yesorno):
    def set_include_patents(self, yesorno):
    def get_url(self):
    def __init__(self):
    def set_citation_format(self, citform):
    def set_per_page_results(self, per_page_results):
    def is_configured(self):
        def __init__(self, querier):
        def handle_num_results(self, num_results):
        def handle_article(self, art):
    def __init__(self):
    def apply_settings(self, settings):
    def send_query(self, query):
    def get_citation_data(self, article):
    def parse(self, html):
    def add_article(self, art):
    def clear_articles(self):
    def save_cookies(self):
    def _get_http_response(self, url, log_msg=None, err_msg=None):
def txt(querier, with_globals):
def csv(querier, header=False, sep='|'):
def citation_export(querier):
def main():
class Error(Exception):
class FormatError(Error):
class QueryArgumentError(Error):
class ScholarConf(object):
class ScholarUtils(object):
class ScholarArticle(object):
class ScholarArticleParser(object):
class ScholarArticleParser120201(ScholarArticleParser):
class ScholarArticleParser120726(ScholarArticleParser):
class ScholarQuery(object):
class ClusterScholarQuery(ScholarQuery):
class SearchScholarQuery(ScholarQuery):
class ScholarSettings(object):
class ScholarQuerier(object):
    class Parser(ScholarArticleParser120726):
    VERSION = '2.10'
    LOG_LEVEL = 3
    MAX_PAGE_RESULTS = 20 # Current maximum for per-page results
    SCHOLAR_SITE = 'http://scholar.google.com'
    USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0'
    COOKIE_JAR_FILE = None
    LOG_LEVELS = {'error': 1,
                  'warn':  2,
                  'info':  3,
                  'debug': 4}
    SCHOLAR_CLUSTER_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
        + 'cluster=%(cluster)s' \
        + '&num=%(num)s'
    SCHOLAR_QUERY_URL = ScholarConf.SCHOLAR_SITE + '/scholar?' \
        + 'as_q=%(words)s' \
        + '&as_epq=%(phrase)s' \
        + '&as_oq=%(words_some)s' \
        + '&as_eq=%(words_none)s' \
        + '&as_occt=%(scope)s' \
        + '&as_sauthors=%(authors)s' \
        + '&as_publication=%(pub)s' \
        + '&as_ylo=%(ylo)s' \
        + '&as_yhi=%(yhi)s' \
        + '&as_sdt=%(patents)s%%2C5' \
        + '&as_vis=%(citations)s' \
        + '&btnG=&hl=en' \
        + '&num=%(num)s'
    CITFORM_NONE = 0
    CITFORM_REFWORKS = 1
    CITFORM_REFMAN = 2
    CITFORM_ENDNOTE = 3
    CITFORM_BIBTEX = 4
    GET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_settings?' \
        + 'sciifh=1&hl=en&as_sdt=0,5'
    SET_SETTINGS_URL = ScholarConf.SCHOLAR_SITE + '/scholar_setprefs?' \
        + 'q=' \
        + '&scisig=%(scisig)s' \
        + '&inststart=0' \
        + '&as_sdt=1,5' \
        + '&as_sdtp=' \
        + '&num=%(num)s' \
        + '&scis1=%(scis)s' \
        + '%(scisf)s' \
        + '&hl=en&lang=all&instq=&inst=569367360547434339&save=' def work_by_varname(varname, year=None):
    """Load work by varname

    Doctest:

    .. doctest::

        >>> reload()
        >>> work = work_by_varname('murta2014a')
        >>> work.year
        2014
    """
    if year is None:
        year = int(parse_varname(varname, 2) or -1)
    module = "y{}.py".format(year) if isinstance(year, int) else year
    if module not in WORK_CACHE:
        return
    worklist = WORK_CACHE[module]
    return getattr(worklist, varname, None)
import sys
import time
from collections import deque
from random import uniform
from selenium import webdriver
from selenium.webdriver.remote.errorhandler import NoSuchElementException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.common.exceptions import TimeoutException
from .scholar import quote, encode, unquote
from .scholar import Error, FormatError, QueryArgumentError, ScholarConf
from .scholar import ScholarUtils, ScholarArticle, ScholarArticleParser120726
from .scholar import ScholarQuery, BeautifulSoup
from .operations import work_by_varname
        from . import config
    """Click on selector"""
    element = parent.find_element_by_css_selector(selector)
    element.click()
    return element


class URLQuery(ScholarQuery):
    """Represent a Google Scholar query using a generic query
    We use it to navigate on the citations"""

    def __init__(self, url, start=None):
        ScholarQuery.__init__(self)
        self.start = start

        self.url = url


    def get_url(self):
        url = self.url
        if self.start is not None:
            url += "&" if "?" in url else "?"
            url += "start={}".format(self.start)
        return url


class Result(object):
    """Represent a result with articles"""

    def __init__(self, query, html):
        self.articles = []
        self.query = query
        self.html = html
        self.next_page = None
        self.prev_page = None

    def set_navigation(self, driver, name, text):
        try:
            link = driver.find_element_by_link_text(text)
            setattr(self, name, URLQuery(link.get_attribute("href")))
        except NoSuchElementException:
            pass


class AddArticleTask(object):
    """Task that adds an article to the result"""

    def __init__(self, result, article):
        self.article = article
        self.result = result

    def get_citation_data(self, querier):
        """
        Given an article, retrieves citation link. Note, this requires that
        you adjusted the settings to tell Google Scholar to actually
        provide this information, *prior* to retrieving the article.
        """
        if self.article["url_citation"] is None:
            return False
        if self.article.citation_data is not None:
            return True

        ScholarUtils.log("info", "retrieving citation export data: {}".format(
            self.article["url_citation"]
        ))
        data = querier.get_response(url=self.article["url_citation"],
                                    log_msg="citation data response",
                                    err_msg="requesting citation data failed",
                                    condition=(By.TAG_NAME, "pre"))
        soup = BeautifulSoup(data, "html.parser")
        data = soup.text
        if data is None:
            querier.tasks.appendleft(self)
            raise Error("Connection error")

        self.article.set_citation_data(data)
        return True

    def apply(self, querier):
        if len(self.result.articles) < getattr(self.result.query, "num_results", 10000):
            self.get_citation_data(querier)
            self.result.articles.append(self.article)


class ParseTask(ScholarArticleParser120726):
    """Task that parsers articles"""

    def __init__(self, result):
        ScholarArticleParser120726.__init__(self)
        self.result = result
        self.querier = None
        self.articles = []

    def handle_num_results(self, num_results):
        if self.result is not None and self.result.query is not None:
            self.result.query["num_results"] = num_results

    def handle_article(self, art):
        self.articles.append(art)

    def apply(self, querier):
        self.querier = querier
        self.parse(self.result.html)
        for art in reversed(self.articles):
            self.querier.tasks.appendleft(AddArticleTask(self.result, art))


class QueryTask(object):
    """Task that queries the scholar"""

    def __init__(self, query):
        self.query = query

    def apply(self, querier):
        html = querier.get_response(url=self.query.get_url(),
                                    log_msg="dump of query response HTML",
                                    err_msg="results retrieval failed",
                                    condition=(By.ID, "gs_ab"))

        if html is None:
            querier.tasks.appendleft(self)

            raise Error("Connection error")