text: """Shared support for scanning document type declarations in HTML and XHTML.
  
  This module is used as a foundation for the HTMLParser and sgmllib
  modules (indirectly, for htmllib as well).  It has no documented
  public API and should not be used directly.
  
  """
  
  import re
  
  _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
  _declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
  _commentclose = re.compile(r'--\s*>')
  _markedsectionclose = re.compile(r']\s*]\s*>')
  
  # An analysis of the MS-Word extensions is available at
  # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
  
  _msmarkedsectionclose = re.compile(r']\s*>')
  
  del re
  
  
  class ParserBase:
      """Parser base class which provides some common support methods used
      by the SGML/HTML and XHTML parsers."""
  
      def __init__(self):
          if self.__class__ is ParserBase:
              raise RuntimeError(
                  "markupbase.ParserBase must be subclassed")
  
      def error(self, message):
          raise NotImplementedError(
              "subclasses of ParserBase must override error()")
  
      def reset(self):
          self.lineno = 1
          self.offset = 0
  
      def getpos(self):
          """Return current line number and offset."""
          return self.lineno, self.offset
  
      # Internal -- update line number and offset.  This should be
      # called for each piece of data exactly once, in order -- in other
      # words the concatenation of all the input strings to this
      # function should be exactly the entire input.
      def updatepos(self, i, j):
          if i >= j:
              return j
          rawdata = self.rawdata
          nlines = rawdata.count("\n", i, j)
          if nlines:
              self.lineno = self.lineno + nlines
              pos = rawdata.rindex("\n", i, j) # Should not fail
              self.offset = j-(pos+1)
          else:
              self.offset = self.offset + j-i
          return j
  
      _decl_otherchars = ''
  
      # Internal -- parse declaration (for use by subclasses).
      def parse_declaration(self, i):
          # This is some sort of declaration; in "HTML as
          # deployed," this should only be the document type
          # declaration ("<!DOCTYPE html...>").
          # ISO 8879:1986, however, has more complex
          # declaration syntax for elements in <!...>, including:
          # --comment--
          # [marked section]
          # name in the following list: ENTITY, DOCTYPE, ELEMENT,
          # ATTLIST, NOTATION, SHORTREF, USEMAP,
          # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
          rawdata = self.rawdata
          j = i + 2
          assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
          if rawdata[j:j+1] == ">":
              # the empty comment <!>
              return j + 1
          if rawdata[j:j+1] in ("-", ""):
              # Start of comment followed by buffer boundary,
              # or just a buffer boundary.
              return -1
          # A simple, practical version could look like: ((name|stringlit) S*) + '>'
          n = len(rawdata)
          if rawdata[j:j+2] == '--': #comment
              # Locate --.*-- as the body of the comment
              return self.parse_comment(i)
          elif rawdata[j] == '[': #marked section
              # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
              # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
              # Note that this is extended by Microsoft Office "Save as Web" function
              # to include [if...] and [endif].
              return self.parse_marked_section(i)
          else: #all other declaration elements
              decltype, j = self._scan_name(j, i)
          if j < 0:
              return j
          if decltype == "doctype":
              self._decl_otherchars = ''
          while j < n:
              c = rawdata[j]
              if c == ">":
                  # end of declaration syntax
                  data = rawdata[i+2:j]
                  if decltype == "doctype":
                      self.handle_decl(data)
                  else:
                      # According to the HTML5 specs sections "8.2.4.44 Bogus
                      # comment state" and "8.2.4.45 Markup declaration open
                      # state", a comment token should be emitted.
                      # Calling unknown_decl provides more flexibility though.
                      self.unknown_decl(data)
                  return j + 1
              if c in "\"'":
                  m = _declstringlit_match(rawdata, j)
                  if not m:
                      return -1 # incomplete
                  j = m.end()
              elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
                  name, j = self._scan_name(j, i)
              elif c in self._decl_otherchars:
                  j = j + 1
              elif c == "[":
                  # this could be handled in a separate doctype parser
                  if decltype == "doctype":
                      j = self._parse_doctype_subset(j + 1, i)
                  elif decltype in ("attlist", "linktype", "link", "element"):
                      # must tolerate []'d groups in a content model in an element declaration
                      # also in data attribute specifications of attlist declaration
                      # also link type declaration subsets in linktype declarations
                      # also link attribute specification lists in link declarations
                      self.error("unsupported '[' char in %s declaration" % decltype)
                  else:
                      self.error("unexpected '[' char in declaration")
              else:
                  self.error(
                      "unexpected %r char in declaration" % rawdata[j])
              if j < 0:
                  return j
          return -1 # incomplete
  
      # Internal -- parse a marked section
      # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
      def parse_marked_section(self, i, report=1):
          rawdata= self.rawdata
          assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
          sectName, j = self._scan_name( i+3, i )
          if j < 0:
              return j
          if sectName in ("temp", "cdata", "ignore", "include", "rcdata"):
              # look for standard ]]> ending
              match= _markedsectionclose.search(rawdata, i+3)
          elif sectName in ("if", "else", "endif"):
              # look for MS Office ]> ending
              match= _msmarkedsectionclose.search(rawdata, i+3)
          else:
              self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
          if not match:
              return -1
          if report:
              j = match.start(0)
              self.unknown_decl(rawdata[i+3: j])
          return match.end(0)
  
      # Internal -- parse comment, return length or -1 if not terminated
      def parse_comment(self, i, report=1):
          rawdata = self.rawdata
          if rawdata[i:i+4] != '<!--':
              self.error('unexpected call to parse_comment()')
          match = _commentclose.search(rawdata, i+4)
          if not match:
              return -1
          if report:
              j = match.start(0)
              self.handle_comment(rawdata[i+4: j])
          return match.end(0)
  
      # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
      # returning the index just past any whitespace following the trailing ']'.
      def _parse_doctype_subset(self, i, declstartpos):
          rawdata = self.rawdata
          n = len(rawdata)
          j = i
          while j < n:
              c = rawdata[j]
              if c == "<":
                  s = rawdata[j:j+2]
                  if s == "<":
                      # end of buffer; incomplete
                      return -1
                  if s != "<!":
                      self.updatepos(declstartpos, j + 1)
                      self.error("unexpected char in internal subset (in %r)" % s)
                  if (j + 2) == n:
                      # end of buffer; incomplete
                      return -1
                  if (j + 4) > n:
                      # end of buffer; incomplete
                      return -1
                  if rawdata[j:j+4] == "<!--":
                      j = self.parse_comment(j, report=0)
                      if j < 0:
                          return j
                      continue
                  name, j = self._scan_name(j + 2, declstartpos)
                  if j == -1:
                      return -1
                  if name not in ("attlist", "element", "entity", "notation"):
                      self.updatepos(declstartpos, j + 2)
                      self.error(
                          "unknown declaration %r in internal subset" % name)
                  # handle the individual names
                  meth = getattr(self, "_parse_doctype_" + name)
                  j = meth(j, declstartpos)
                  if j < 0:
                      return j
              elif c == "%":
                  # parameter entity reference
                  if (j + 1) == n:
                      # end of buffer; incomplete
                      return -1
                  s, j = self._scan_name(j + 1, declstartpos)
                  if j < 0:
                      return j
                  if rawdata[j] == ";":
                      j = j + 1
              elif c == "]":
                  j = j + 1
                  while j < n and rawdata[j].isspace():
                      j = j + 1
                  if j < n:
                      if rawdata[j] == ">":
                          return j
                      self.updatepos(declstartpos, j)
                      self.error("unexpected char after internal subset")
                  else:
                      return -1
              elif c.isspace():
                  j = j + 1
              else:
                  self.updatepos(declstartpos, j)
                  self.error("unexpected char %r in internal subset" % c)
          # end of buffer reached
          return -1
  
      # Internal -- scan past <!ELEMENT declarations
      def _parse_doctype_element(self, i, declstartpos):
          name, j = self._scan_name(i, declstartpos)
          if j == -1:
              return -1
          # style content model; just skip until '>'
          rawdata = self.rawdata
          if '>' in rawdata[j:]:
              return rawdata.find(">", j) + 1
          return -1
  
      # Internal -- scan past <!ATTLIST declarations
      def _parse_doctype_attlist(self, i, declstartpos):
          rawdata = self.rawdata
          name, j = self._scan_name(i, declstartpos)
          c = rawdata[j:j+1]
          if c == "":
              return -1
          if c == ">":
              return j + 1
          while 1:
              # scan a series of attribute descriptions; simplified:
              #   name type [value] [#constraint]
              name, j = self._scan_name(j, declstartpos)
              if j < 0:
                  return j
              c = rawdata[j:j+1]
              if c == "":
                  return -1
              if c == "(":
                  # an enumerated type; look for ')'
                  if ")" in rawdata[j:]:
                      j = rawdata.find(")", j) + 1
                  else:
                      return -1
                  while rawdata[j:j+1].isspace():
                      j = j + 1
                  if not rawdata[j:]:
                      # end of buffer, incomplete
                      return -1
              else:
                  name, j = self._scan_name(j, declstartpos)
              c = rawdata[j:j+1]
              if not c:
                  return -1
              if c in "'\"":
                  m = _declstringlit_match(rawdata, j)
                  if m:
                      j = m.end()
                  else:
                      return -1
                  c = rawdata[j:j+1]
                  if not c:
                      return -1
              if c == "#":
                  if rawdata[j:] == "#":
                      # end of buffer
                      return -1
                  name, j = self._scan_name(j + 1, declstartpos)
                  if j < 0:
                      return j
                  c = rawdata[j:j+1]
                  if not c:
                      return -1
              if c == '>':
                  # all done
                  return j + 1
  
      # Internal -- scan past <!NOTATION declarations
      def _parse_doctype_notation(self, i, declstartpos):
          name, j = self._scan_name(i, declstartpos)
          if j < 0:
              return j
          rawdata = self.rawdata
          while 1:
              c = rawdata[j:j+1]
              if not c:
                  # end of buffer; incomplete
                  return -1
              if c == '>':
                  return j + 1
              if c in "'\"":
                  m = _declstringlit_match(rawdata, j)
                  if not m:
                      return -1
                  j = m.end()
              else:
                  name, j = self._scan_name(j, declstartpos)
                  if j < 0:
                      return j
  
      # Internal -- scan past <!ENTITY declarations
      def _parse_doctype_entity(self, i, declstartpos):
          rawdata = self.rawdata
          if rawdata[i:i+1] == "%":
              j = i + 1
              while 1:
                  c = rawdata[j:j+1]
                  if not c:
                      return -1
                  if c.isspace():
                      j = j + 1
                  else:
                      break
          else:
              j = i
          name, j = self._scan_name(j, declstartpos)
          if j < 0:
              return j
          while 1:
              c = self.rawdata[j:j+1]
              if not c:
                  return -1
              if c in "'\"":
                  m = _declstringlit_match(rawdata, j)
                  if m:
                      j = m.end()
                  else:
                      return -1    # incomplete
              elif c == ">":
                  return j + 1
              else:
                  name, j = self._scan_name(j, declstartpos)
                  if j < 0:
                      return j
  
      # Internal -- scan a name token and the new position and the token, or
      # return -1 if we've reached the end of the buffer.
      def _scan_name(self, i, declstartpos):
          rawdata = self.rawdata
          n = len(rawdata)
          if i == n:
              return None, -1
          m = _declname_match(rawdata, i)
          if m:
              s = m.group()
              name = s.strip()
              if (i + len(s)) == n:
                  return None, -1  # end of buffer
              return name.lower(), m.end()
          else:
              self.updatepos(declstartpos, i)
              self.error("expected name token at %r"
                         % rawdata[declstartpos:declstartpos+20])
  
      # To be overridden -- handlers for unknown objects
      def unknown_decl(self, data):
          pass
  