<reponame>ikedim01/secscan<gh_stars>0
# AUTOGENERATED! DO NOT EDIT! File to edit: 10_scrape13G.ipynb (unless otherwise specified).

__all__ = ['default13GDir', 'getSec13NshAndPctFromText2', 'addNshAndPct', 'cusipChecksum', 'monthNameToIso',
           'getMonthPatStr', 'parseEventDate', 'parse13GD', 'scraper13G', 'reOPTS', 'aggregatePatStr',
           'percentOfClassPatStr', 'typeOfRepPatStr', 'form13PiecesPat1', 'form13PiecesPat2', 'form13PiecesPat3',
           'nSharesPatStr', 'nPctBarePatStr', 'nPctWithPctPatStr', 'nshAndPctPat1Pref', 'form13NshAndPctPats1',
           'form13NshAndPctPat2', 'form13NshAndPctPat3', 'purposePat', 'strictCusipPatStr', 'cusipPatStr',
           'cusipNumberPatStr', 'cusipSearchPats', 'spaceDashPat', 'monthNames', 'monthAbbrevStrs', 'monthPatStr',
           'monthDayPatStr', 'possCommaPatStr', 'yearPatStr', 'dateOfEventPatStr', 'dateOfEventAtStartPatStr',
           'dateOfEventAtEndPatStr', 'dateOfEventMonthPat1', 'dateOfEventMonthRevPat1', 'dateOfEventMonthPat2',
           'dateOfEventMonthRevPat2', 'isoSepPatStr', 'dateOfEventIsoPat1', 'dateOfEventIsoRevPat1',
           'dateOfEventIsoPat2', 'dateOfEventIsoRevPat2', 'whitespacePat', 'updateCik13GDPos', 'cikSymStr',
           'calcBonusMap']

# Cell

import collections
import datetime
import itertools
import os
import re

from secscan import utils, dailyList, basicInfo, infoScraper

default13GDir = os.path.join(utils.stockDataRoot,'scraped13G')

# Cell

reOPTS = re.IGNORECASE|re.DOTALL
# aggregatePatStr = r'aggregate.?\s+amount\s+(?:ben|own)'
# percentOfClassPatStr = r'percent\s+(?:of|or)\s+class\s+(?:re|pr)'
# typeOfRepPatStr = r'type.?\s+(?:of|or)\s+(?:rep|per)'
aggregatePatStr = r'aggregated?\s+amount\s+ben'
percentOfClassPatStr = r'percent(?:age)?\s+of\s+class\s+(?:re|pr)'
typeOfRepPatStr = r'type\s+of\s+(?:rep|per)'
item9PatStr,item11PatStr,item12PatStr,item13PatStr,item14PatStr = (
    r'item\s+' + itemNo + r'\s*:' for itemNo in ('9','11','12','13','14'))
form13PiecesPat1 = re.compile(r'.*?'.join([aggregatePatStr,percentOfClassPatStr,typeOfRepPatStr]),reOPTS)
form13PiecesPat2 = re.compile(r'.*?'.join([item9PatStr,item11PatStr,item12PatStr]),reOPTS)
form13PiecesPat3 = re.compile(r'.*?'.join([item11PatStr,item13PatStr,item14PatStr]),reOPTS)
# nSharesPatStr = r'\D(?!9\s|9\D\D)(\d+(?:[,.]\d\d\d)*)' # try to avoid taking item number 9 as share count
nSharesPatStr = r'(?<!\d)(?!9\s|9\D\D)(\d+[,.\d]*)' # try to avoid taking item number 9 as share count
nPctBarePatStr = r'(\d+(?:\.\d*)?|\.\d+)'
nPctWithPctPatStr = r'((?:\d+(?:[\.,]\d*)?|[\.,]\d+)\s*%)'
nshAndPctPat1Pref = r'.*?'.join([aggregatePatStr,nSharesPatStr,percentOfClassPatStr])
form13NshAndPctPats1 = [
    re.compile(r'.*?'.join([nshAndPctPat1Pref,nPctWithPctPatStr,typeOfRepPatStr]),reOPTS),
    # if percentage isn't followed by a % character, look for a plain number but
    # try to avoid using the 9 in "mentioned in item 9" verbiage for the percentage
    re.compile(r'.*?'.join([nshAndPctPat1Pref,r'\D9(?!\.\d)\D.*?'+nPctBarePatStr,typeOfRepPatStr]),reOPTS),
    # ditto for "mentioned in item 11" verbiage
    re.compile(r'.*?'.join([nshAndPctPat1Pref,r'\D11(?!\.\d)\D.*?'+nPctBarePatStr,typeOfRepPatStr]),reOPTS),
    re.compile(r'.*?'.join([nshAndPctPat1Pref,nPctBarePatStr,typeOfRepPatStr]),reOPTS)
]
form13NshAndPctPat2 = re.compile(r'.*?'.join([item9PatStr,nSharesPatStr,item11PatStr,
                                              nPctWithPctPatStr,item12PatStr]), reOPTS)
form13NshAndPctPat3 = re.compile(r'.*?'.join([item11PatStr,nSharesPatStr,item13PatStr,
                                              nPctWithPctPatStr,item14PatStr]), reOPTS)
def getSec13NshAndPctFromText2(txt,accNo, debug=False) :
    "Returns a list [(nShares, percent) ... ] parsed from form 13G or 13D."
    if debug : print(txt)
    res = []
    pat1Pieces = form13PiecesPat1.findall(txt)
    for piece in pat1Pieces :
        if debug : print('********1',piece)
        if not any(addNshAndPct(pat.match(piece),res) for pat in form13NshAndPctPats1) :
            print("??????1", accNo, piece)
    if res :
        return res
    pat2Pieces = form13PiecesPat2.findall(txt)
    for piece in pat2Pieces :
        if debug : print('********2',piece)
        if not addNshAndPct(form13NshAndPctPat2.match(piece),res) :
            print("??????2", accNo, piece)
    if res :
        return res
    pat3Pieces = form13PiecesPat3.findall(txt)
    for piece in pat3Pieces :
        if debug : print('********3',piece)
        if not addNshAndPct(form13NshAndPctPat3.match(piece),res) :
            print("??????3", accNo, piece)
    return res
def addNshAndPct(m,res) :
    if not m :
        return False
    # print(m.groups())
    nSh,pct = m.groups()
    if nSh in ['10','10.','11','11.','12','12.'] :
        nSh = '999'
    if pct in ['9','9.','11','11.','12','12.','14','14.'] :
        return False
    # nSh = nSh.replace('.',',')
    pct = pct.replace(',','.').replace('%','').rstrip()
    res.append((nSh,pct))
    return True

purposePat = re.compile(r'4\s*\.?\s*purpose\s*of\s*(?:the\s*)?transaction(?:\s*\.?\s*)(.{1,20000}?)'
                        + r'(?:\s*(?:item\s*)?5\s*\.?\s*interest'
                            + r'|\s*(?:item\s*)?6\s*\.?\s*contracts'
                            + r'|\s*(?:item\s*)?7\s*\.?\s*material'
                            + r'|\s*after\s*reasonable\s*inquiry'
                            + r'|\s*$'
                        + r')', reOPTS)

def cusipChecksum(cusip) :
    s = 0
    for i,c in enumerate(cusip[:8]) :
        if c.isdigit() :
            v = ord(c) - ord('0')
        elif c.isalpha() :
            v = 10 + ord(c.upper()) - ord('A')
        if (i&1) == 1 :
            v *= 2
        s += (v//10) + (v%10)
    return str((10 - (s%10)) % 10)

strictCusipPatStr = r'[\dA-Z]\d[\dA-Z]\d[\dA-Z]{4}\d'
cusipPatStr = (r'[\dA-Z]\d[\dA-Z][-_\s]*\d[-_\s]*[\dA-Z][-_\s]*[\dA-Z]'
                + r'(?:[-_\s]*[\dA-Z]{2}(?:[-_\s]*\d)?)?')
cusipNumberPatStr = r'cusip\s*(?:number|#|no)'
cusipSearchPats = [re.compile(patStr, reOPTS) for patStr in [
    r'.{1,3000}?[^\dA-Z](' + cusipPatStr + r')[^2-9A-Z]{0,200}?\s*' + cusipNumberPatStr,
    r'.{1,3000}?\s*' + cusipNumberPatStr + r'[^\dA-Z]{0,200}?(' + cusipPatStr + r')[^\dA-Z]',
    r'.{1,2000}?\s(' + strictCusipPatStr + r')\s',
]]
spaceDashPat = re.compile(r'[-\s]*')

monthNames = ['January', 'February', 'March', 'April', 'May', 'June',
              'July', 'August', 'September', 'October', 'November', 'December']
monthAbbrevStrs = ''.join(monthName[:3].lower() for monthName in monthNames)
def monthNameToIso(monthName) :
    return str(1+(monthAbbrevStrs.find(monthName[:3].lower())//3)).zfill(2)
def getMonthPatStr() :
    monthNamePatStrs = []
    for monthName in monthNames :
        monthNamePatStr = monthName[:3]
        if monthName != 'May' :
            monthNamePatStr += r'(?:'
            if monthName == 'September' :
                monthNamePatStr += r't|t\.|'
            monthNamePatStr += monthName[3:]
            monthNamePatStr += r'|\.)?'
        monthNamePatStrs.append(monthNamePatStr)
    return '(' + '|'.join(monthNamePatStrs) + ')'
monthPatStr = getMonthPatStr()
monthDayPatStr = r'(\d\d?)(?:\s*th|\s*st)?'
possCommaPatStr = r'[.,\s]'
yearPatStr = r'(\d\d\s*\d\d)'
dateOfEventPatStr = r'dates?\s*of(?:\s*the)?\s*events?\s*which'
dateOfEventAtStartPatStr = r'.{1,3000}?'+dateOfEventPatStr+r'.{0,120}?'
dateOfEventAtEndPatStr = r'[^\d].{0,120}?'+dateOfEventPatStr
dateOfEventMonthPat1 = re.compile(r'.{1,3000}?[^\dA-Z]'
                                  + r'\s*'.join([monthPatStr,monthDayPatStr,possCommaPatStr,yearPatStr])
                                  + dateOfEventAtEndPatStr, reOPTS)
dateOfEventMonthRevPat1 = re.compile(dateOfEventAtStartPatStr + r'[^\dA-Z]'
                                     + r'\s*'.join([monthPatStr,monthDayPatStr,possCommaPatStr,yearPatStr])
                                     + r'[^\d]', reOPTS)
dateOfEventMonthPat2 = re.compile(r'.{1,3000}?[^\d]'
                                  + r'\s*'.join([monthDayPatStr,monthPatStr,possCommaPatStr,yearPatStr])
                                  + dateOfEventAtEndPatStr, reOPTS)
dateOfEventMonthRevPat2 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                     + r'\s*'.join([monthDayPatStr,monthPatStr,possCommaPatStr,yearPatStr])
                                     + r'[^\d]', reOPTS)
isoSepPatStr = r'\s*[-/]\s*'
dateOfEventIsoPat1 = re.compile(r'.{1,3000}?[^\d]'
                                + isoSepPatStr.join([r'(\d\d?)',r'(\d\d?)',r'(\d\d(?:\d\d)?)'])
                                + dateOfEventAtEndPatStr, reOPTS)
dateOfEventIsoRevPat1 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                   + isoSepPatStr.join([r'(\d\d?)',r'(\d\d?)',r'(\d\d(?:\d\d)?)'])
                                   + r'[^\d]', reOPTS)
dateOfEventIsoPat2 = re.compile(r'.{1,3000}?[^\d]'
                                + isoSepPatStr.join([r'(\d\d\d\d)',r'(\d\d?)',r'(\d\d?)'])
                                + dateOfEventAtEndPatStr, reOPTS)
dateOfEventIsoRevPat2 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                   + isoSepPatStr.join([r'(\d\d\d\d)',r'(\d\d?)',r'(\d\d?)'])
                                   + r'[^\d]', reOPTS)
whitespacePat = re.compile(r'\s*', reOPTS)
def parseEventDate(info,mainText) :
    m = dateOfEventMonthPat1.match(mainText) or dateOfEventMonthRevPat1.match(mainText)
    if m :
        info['eventDate'] = '-'.join([whitespacePat.sub('',m.group(3)),
                                      monthNameToIso(m.group(1)),m.group(2).zfill(2)])
        return
    m = dateOfEventMonthPat2.match(mainText) or dateOfEventMonthRevPat2.match(mainText)
    if m :
        info['eventDate'] = '-'.join([whitespacePat.sub('',m.group(3)),
                                      monthNameToIso(m.group(2)),m.group(1).zfill(2)])
        return
    m = dateOfEventIsoPat1.match(mainText) or dateOfEventIsoRevPat1.match(mainText)
    if m :
        info['eventDate'] = '-'.join([('20' if len(m.group(3))==2 else '')+m.group(3),
                                      m.group(1).zfill(2),m.group(2).zfill(2)])
        return
    m = dateOfEventIsoPat2.match(mainText) or dateOfEventIsoRevPat2.match(mainText)
    if m :
        info['eventDate'] = '-'.join([m.group(1),m.group(2).zfill(2),m.group(3).zfill(2)])
        return
    print('NO EVENT DATE!', end=' ')


def parse13GD(accNo, formType=None, info=None, textLimit=basicInfo.defaultTextLimit, debug=False) :
    if info is None :
        info = basicInfo.getSecFormInfo(accNo, formType=formType)
    if 'filedByCik' not in info :
        print('No filed by CIK!', end=' ')
    links = info['links']
    if len(links) == 0 :
        print('NO LINKS LIST!', end=' ')
        info['positions'] = []
    else :
        toFormat = 'text' if links[0][3].endswith('.txt') else 'souptext'
        mainText = utils.downloadSecUrl(links[0][3], toFormat=toFormat)
        parseEventDate(info,mainText)
        info['positions'] = getSec13NshAndPctFromText2(mainText,accNo, debug=debug)
        for cusipSearchPat in cusipSearchPats :
            m = cusipSearchPat.match(mainText)
            if m is not None :
                break
        if m is None :
            if not ('0001504304' in info['ciks'] or '0001067621' in info['ciks']) :
                # suppress the message for 0001504304 - Bulldog Investors
                # and 0001067621 - <NAME>
                # - they don't report CUSIPs in their filings
                print('no CUSIP found!', end=' ')
        else :
            cusip = spaceDashPat.sub('',m.group(1))
            if len(cusip) == 6 :
                print('adding 10 to CUSIP', cusip, end=' ')
                cusip = cusip + '10'
            if len(cusip) == 8 :
                print('adding checksum to CUSIP', cusip, end=' ')
                if cusipChecksum('0'+cusip[:7]) == cusip[7] :
                    cusip = '0'+cusip
                else :
                    cusip = cusip + cusipChecksum(cusip)
            if len(cusip)!=9 or cusip[8]!=cusipChecksum(cusip) :
                print('invalid CUSIP!', cusip, end=' ')
            info['cusip'] = cusip.upper()
            # print('CUSIP-'+cusip,end=' ')
        if formType is None :
            formType = links[0][2]
        if formType.upper().startswith('SC 13D') :
            m = purposePat.search(mainText)
            if m is None :
                print('no purpose!', end=' ')
            else :
                info['purpose'] = m.group(1)[:textLimit]
    if len(info['positions']) == 0 :
        print('no positions found!', end=' ')
    return info #,mainText

class scraper13G(infoScraper.scraperBase) :
    @utils.delegates(infoScraper.scraperBase.__init__)
    def __init__(self, infoDir=default13GDir, **kwargs) :
        super().__init__(infoDir, 'SC 13G', **kwargs)
    def init_for_13D(self, infoDir, **kwargs) :
        super().__init__(infoDir, 'SC 13D', **kwargs)
    def scrapeInfo(self, accNo, formType=None) :
        return parse13GD(accNo, formType=formType), None
    def rescrapeInfo(self, accNo, info) :
        return parse13GD(accNo, info=info)

# Cell

def updateCik13GDPos(scrapers, cik13GDPosMap=None,
                     cusipNames=None, cikNames=None, includeTickers=False) :
    """
    Generate or update a combined dict of percentage holdings:
        cik13GDPosMap: cik -> {cusip -> (eventDate, accNo, pct)}
    based on a list of 13G and 13D scrapers - pct is in the range [0.0 .. 100.0],
    as given in the 13G and 13D filings.

    If cusipNames and cikNames are supplied, both should be dicts, and CUSIPs
    encountered in 13G/D filings will have the corresponding CIK names added
    to the CUSIP names based on the filings (i.e. each 13D/G filing includes
    a CUSIP, and a corresponding subject CIK). If includeTickers is True,
    the ticker names will also be added based on the CIK to ticker file
    provided by the SEC.
    """
    if cik13GDPosMap is None :
        cik13GDPosMap = collections.defaultdict(dict)
    cikTo13GDs = collections.defaultdict(list)
    count = 0
    extraCusipNames = None if (cusipNames is None or cikNames is None) else {
        # CUSIP -> CIK mapping for some stocks that don't currently appear in any 13D or 13G filings:
        '931142103' : ('0000','walmart','104169'),
        '084670108' : ('0000','berkshire','1067983'),
    }
    for scraper in scrapers :
        for dStr, accNoToInfo in scraper.infoMap.items() :
            for accNo, info in accNoToInfo.items() :
                if info == 'ERROR' :
                    print('*** ERROR in ',accNo)
                elif 'filedByCik' not in info :
                    print('*** No filed-by CIK in',accNo)
                elif 'cusip' not in info :
                    print('No CUSIP in',accNo)
                else :
                    if len(info['positions']) == 0 :
                        print('*** No positions found in',accNo)
                        maxPctPos = 0.0
                    else :
                        maxPctPos = max(float(pct) for _,pct in info['positions'])
                    if 'eventDate' not in info :
                        eventDate = (utils.toDate(dStr)-datetime.timedelta(7)).isoformat()
                        print(f'No event date in {accNo}; using {eventDate}')
                    else :
                        eventDate = info['eventDate']
                    cusip = info['cusip']
                    filedByCik = info['filedByCik']
                    cikTo13GDs[filedByCik.lstrip('0')].append((cusip, eventDate, accNo,maxPctPos))
                    count += 1
                    if extraCusipNames is not None :
                        subjectCik = [cik for cik in info['ciks'] if cik!=filedByCik]
                        if len(subjectCik) != 1 :
                            print(f"missing or ambiguous subject CIK '{accNo}'")
                        elif cusip not in extraCusipNames or extraCusipNames[cusip][0] < dStr :
                            subjectCik = subjectCik[0].lstrip('0')
                            if subjectCik in cikNames :
                                extraCusipNames[cusip] = (dStr, cikNames[subjectCik], subjectCik)
                            else :
                                print(f"subject CIK {subjectCik} name not found '{accNo}'")
    if extraCusipNames is not None :
        count1 = count2 = 0
        if includeTickers :
            cikToTickers = dailyList.getCikToTickersMap()
        else :
            cikToTickers = collections.defaultdict(list)
        for cusip,name in cusipNames.items() :
            if cusip in extraCusipNames and 'CIK-' not in name :
                _,subjectCikName,subjectCik = extraCusipNames[cusip]
                if subjectCikName[:8].strip().lower() != name[:8].strip().lower() :
                    cusipNames[cusip] += f' - {subjectCikName}'
                cusipNames[cusip] += cikSymStr(subjectCik,cikToTickers[subjectCik])
                count1 += 1
        for cusip,(_,subjectCikName,subjectCik) in extraCusipNames.items() :
            if cusip not in cusipNames :
                cusipNames[cusip] = f'- {subjectCikName}{cikSymStr(subjectCik,cikToTickers[subjectCik])}'
                count2 += 1
        print('count1',count1,'count2',count2)
    print('total of',len(cikTo13GDs),'ciks,',count,'13G/D filings')
    for cik, cik13GDList in cikTo13GDs.items() :
        posMap = cik13GDPosMap[cik]
        for tup in cik13GDList :
            cusip = tup[0]
            if cusip not in posMap or posMap[cusip] < tup[1:] :
                posMap[cusip] = tup[1:]
    return cik13GDPosMap
def cikSymStr(cik,tickers) :
    return ' (' + ', '.join(sorted(tickers)[:8] + (['...'] if len(tickers)>8 else []) + ['CIK-'+cik])+ ')'

def calcBonusMap(cik13GDPosMap, max13GDBonus=0.2, min13GDBonus=0.02, max13GDCount=100,
                 allCusipCounter=None) :
    """
    Calculate "bonus fractions" for cusips where a 13G or 13D has been filed.
    13GD bonus fractions are 1.0/#positions, but restricted to [min13GDBonus..max13GDBonus]
    If max13GDCount is not None, restricts to investors with at most max13GDCount combined 13G
    and 13D positions. For positions between [1.0% .. 5.0%), the bonus is cut in half;
    positions below 1.0% aren't given a bonus.

    If supplied, allCusipCounter should be a Counter, and it will be updated to count
    all investors that have any position in each stock.

    Returns a dict: cik -> {cusip -> bonusfrac}
    """
    res = {}
    for cik,posMap in cik13GDPosMap.items() :
        if allCusipCounter is not None :
            allCusipCounter.update(posMap.keys())
        fullCusips, halfCusips = [], []
        for cusip,pos in posMap.items() :
            if pos[-1] >= 5.0 :
                fullCusips.append(cusip)
            elif pos[-1] >= 1.0 :
                halfCusips.append(cusip)
        totNCusips = len(fullCusips) + len(halfCusips)
        if totNCusips>0 and (max13GDCount is None or totNCusips<=max13GDCount) :
            bonus = min(max13GDBonus,max(min13GDBonus,1/totNCusips))
            res[cik] = dict((cusip,bonus) for cusip in fullCusips)
            res[cik].update((cusip,bonus*0.5) for cusip in halfCusips)
    return res
# AUTOGENERATED! DO NOT EDIT! File to edit: 10_scrape13G.ipynb (unless otherwise specified).

__all__ = ['default13GDir', 'getSec13NshAndPctFromText2', 'addNshAndPct', 'cusipChecksum', 'monthNameToIso',
           'getMonthPatStr', 'parseEventDate', 'parse13GD', 'scraper13G', 'reOPTS', 'aggregatePatStr',
           'percentOfClassPatStr', 'typeOfRepPatStr', 'form13PiecesPat1', 'form13PiecesPat2', 'form13PiecesPat3',
           'nSharesPatStr', 'nPctBarePatStr', 'nPctWithPctPatStr', 'nshAndPctPat1Pref', 'form13NshAndPctPats1',
           'form13NshAndPctPat2', 'form13NshAndPctPat3', 'purposePat', 'strictCusipPatStr', 'cusipPatStr',
           'cusipNumberPatStr', 'cusipSearchPats', 'spaceDashPat', 'monthNames', 'monthAbbrevStrs', 'monthPatStr',
           'monthDayPatStr', 'possCommaPatStr', 'yearPatStr', 'dateOfEventPatStr', 'dateOfEventAtStartPatStr',
           'dateOfEventAtEndPatStr', 'dateOfEventMonthPat1', 'dateOfEventMonthRevPat1', 'dateOfEventMonthPat2',
           'dateOfEventMonthRevPat2', 'isoSepPatStr', 'dateOfEventIsoPat1', 'dateOfEventIsoRevPat1',
           'dateOfEventIsoPat2', 'dateOfEventIsoRevPat2', 'whitespacePat', 'updateCik13GDPos', 'cikSymStr',
           'calcBonusMap']

# Cell

import collections
import datetime
import itertools
import os
import re

from secscan import utils, dailyList, basicInfo, infoScraper

default13GDir = os.path.join(utils.stockDataRoot,'scraped13G')

# Cell

reOPTS = re.IGNORECASE|re.DOTALL
# aggregatePatStr = r'aggregate.?\s+amount\s+(?:ben|own)'
# percentOfClassPatStr = r'percent\s+(?:of|or)\s+class\s+(?:re|pr)'
# typeOfRepPatStr = r'type.?\s+(?:of|or)\s+(?:rep|per)'
aggregatePatStr = r'aggregated?\s+amount\s+ben'
percentOfClassPatStr = r'percent(?:age)?\s+of\s+class\s+(?:re|pr)'
typeOfRepPatStr = r'type\s+of\s+(?:rep|per)'
item9PatStr,item11PatStr,item12PatStr,item13PatStr,item14PatStr = (
    r'item\s+' + itemNo + r'\s*:' for itemNo in ('9','11','12','13','14'))
form13PiecesPat1 = re.compile(r'.*?'.join([aggregatePatStr,percentOfClassPatStr,typeOfRepPatStr]),reOPTS)
form13PiecesPat2 = re.compile(r'.*?'.join([item9PatStr,item11PatStr,item12PatStr]),reOPTS)
form13PiecesPat3 = re.compile(r'.*?'.join([item11PatStr,item13PatStr,item14PatStr]),reOPTS)
# nSharesPatStr = r'\D(?!9\s|9\D\D)(\d+(?:[,.]\d\d\d)*)' # try to avoid taking item number 9 as share count
nSharesPatStr = r'(?<!\d)(?!9\s|9\D\D)(\d+[,.\d]*)' # try to avoid taking item number 9 as share count
nPctBarePatStr = r'(\d+(?:\.\d*)?|\.\d+)'
nPctWithPctPatStr = r'((?:\d+(?:[\.,]\d*)?|[\.,]\d+)\s*%)'
nshAndPctPat1Pref = r'.*?'.join([aggregatePatStr,nSharesPatStr,percentOfClassPatStr])
form13NshAndPctPats1 = [
    re.compile(r'.*?'.join([nshAndPctPat1Pref,nPctWithPctPatStr,typeOfRepPatStr]),reOPTS),
    # if percentage isn't followed by a % character, look for a plain number but
    # try to avoid using the 9 in "mentioned in item 9" verbiage for the percentage
    re.compile(r'.*?'.join([nshAndPctPat1Pref,r'\D9(?!\.\d)\D.*?'+nPctBarePatStr,typeOfRepPatStr]),reOPTS),
    # ditto for "mentioned in item 11" verbiage
    re.compile(r'.*?'.join([nshAndPctPat1Pref,r'\D11(?!\.\d)\D.*?'+nPctBarePatStr,typeOfRepPatStr]),reOPTS),
    re.compile(r'.*?'.join([nshAndPctPat1Pref,nPctBarePatStr,typeOfRepPatStr]),reOPTS)
]
form13NshAndPctPat2 = re.compile(r'.*?'.join([item9PatStr,nSharesPatStr,item11PatStr,
                                              nPctWithPctPatStr,item12PatStr]), reOPTS)
form13NshAndPctPat3 = re.compile(r'.*?'.join([item11PatStr,nSharesPatStr,item13PatStr,
                                              nPctWithPctPatStr,item14PatStr]), reOPTS)
def getSec13NshAndPctFromText2(txt,accNo, debug=False) :
    "Returns a list [(nShares, percent) ... ] parsed from form 13G or 13D."
    if debug : print(txt)
    res = []
    pat1Pieces = form13PiecesPat1.findall(txt)
    for piece in pat1Pieces :
        if debug : print('********1',piece)
        if not any(addNshAndPct(pat.match(piece),res) for pat in form13NshAndPctPats1) :
            print("??????1", accNo, piece)
    if res :
        return res
    pat2Pieces = form13PiecesPat2.findall(txt)
    for piece in pat2Pieces :
        if debug : print('********2',piece)
        if not addNshAndPct(form13NshAndPctPat2.match(piece),res) :
            print("??????2", accNo, piece)
    if res :
        return res
    pat3Pieces = form13PiecesPat3.findall(txt)
    for piece in pat3Pieces :
        if debug : print('********3',piece)
        if not addNshAndPct(form13NshAndPctPat3.match(piece),res) :
            print("??????3", accNo, piece)
    return res
def addNshAndPct(m,res) :
    if not m :
        return False
    # print(m.groups())
    nSh,pct = m.groups()
    if nSh in ['10','10.','11','11.','12','12.'] :
        nSh = '999'
    if pct in ['9','9.','11','11.','12','12.','14','14.'] :
        return False
    # nSh = nSh.replace('.',',')
    pct = pct.replace(',','.').replace('%','').rstrip()
    res.append((nSh,pct))
    return True

purposePat = re.compile(r'4\s*\.?\s*purpose\s*of\s*(?:the\s*)?transaction(?:\s*\.?\s*)(.{1,20000}?)'
                        + r'(?:\s*(?:item\s*)?5\s*\.?\s*interest'
                            + r'|\s*(?:item\s*)?6\s*\.?\s*contracts'
                            + r'|\s*(?:item\s*)?7\s*\.?\s*material'
                            + r'|\s*after\s*reasonable\s*inquiry'
                            + r'|\s*$'
                        + r')', reOPTS)

def cusipChecksum(cusip) :
    s = 0
    for i,c in enumerate(cusip[:8]) :
        if c.isdigit() :
            v = ord(c) - ord('0')
        elif c.isalpha() :
            v = 10 + ord(c.upper()) - ord('A')
        if (i&1) == 1 :
            v *= 2
        s += (v//10) + (v%10)
    return str((10 - (s%10)) % 10)

strictCusipPatStr = r'[\dA-Z]\d[\dA-Z]\d[\dA-Z]{4}\d'
cusipPatStr = (r'[\dA-Z]\d[\dA-Z][-_\s]*\d[-_\s]*[\dA-Z][-_\s]*[\dA-Z]'
                + r'(?:[-_\s]*[\dA-Z]{2}(?:[-_\s]*\d)?)?')
cusipNumberPatStr = r'cusip\s*(?:number|#|no)'
cusipSearchPats = [re.compile(patStr, reOPTS) for patStr in [
    r'.{1,3000}?[^\dA-Z](' + cusipPatStr + r')[^2-9A-Z]{0,200}?\s*' + cusipNumberPatStr,
    r'.{1,3000}?\s*' + cusipNumberPatStr + r'[^\dA-Z]{0,200}?(' + cusipPatStr + r')[^\dA-Z]',
    r'.{1,2000}?\s(' + strictCusipPatStr + r')\s',
]]
spaceDashPat = re.compile(r'[-\s]*')

monthNames = ['January', 'February', 'March', 'April', 'May', 'June',
              'July', 'August', 'September', 'October', 'November', 'December']
monthAbbrevStrs = ''.join(monthName[:3].lower() for monthName in monthNames)
def monthNameToIso(monthName) :
    return str(1+(monthAbbrevStrs.find(monthName[:3].lower())//3)).zfill(2)
def getMonthPatStr() :
    monthNamePatStrs = []
    for monthName in monthNames :
        monthNamePatStr = monthName[:3]
        if monthName != 'May' :
            monthNamePatStr += r'(?:'
            if monthName == 'September' :
                monthNamePatStr += r't|t\.|'
            monthNamePatStr += monthName[3:]
            monthNamePatStr += r'|\.)?'
        monthNamePatStrs.append(monthNamePatStr)
    return '(' + '|'.join(monthNamePatStrs) + ')'
monthPatStr = getMonthPatStr()
monthDayPatStr = r'(\d\d?)(?:\s*th|\s*st)?'
possCommaPatStr = r'[.,\s]'
yearPatStr = r'(\d\d\s*\d\d)'
dateOfEventPatStr = r'dates?\s*of(?:\s*the)?\s*events?\s*which'
dateOfEventAtStartPatStr = r'.{1,3000}?'+dateOfEventPatStr+r'.{0,120}?'
dateOfEventAtEndPatStr = r'[^\d].{0,120}?'+dateOfEventPatStr
dateOfEventMonthPat1 = re.compile(r'.{1,3000}?[^\dA-Z]'
                                  + r'\s*'.join([monthPatStr,monthDayPatStr,possCommaPatStr,yearPatStr])
                                  + dateOfEventAtEndPatStr, reOPTS)
dateOfEventMonthRevPat1 = re.compile(dateOfEventAtStartPatStr + r'[^\dA-Z]'
                                     + r'\s*'.join([monthPatStr,monthDayPatStr,possCommaPatStr,yearPatStr])
                                     + r'[^\d]', reOPTS)
dateOfEventMonthPat2 = re.compile(r'.{1,3000}?[^\d]'
                                  + r'\s*'.join([monthDayPatStr,monthPatStr,possCommaPatStr,yearPatStr])
                                  + dateOfEventAtEndPatStr, reOPTS)
dateOfEventMonthRevPat2 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                     + r'\s*'.join([monthDayPatStr,monthPatStr,possCommaPatStr,yearPatStr])
                                     + r'[^\d]', reOPTS)
isoSepPatStr = r'\s*[-/]\s*'
dateOfEventIsoPat1 = re.compile(r'.{1,3000}?[^\d]'
                                + isoSepPatStr.join([r'(\d\d?)',r'(\d\d?)',r'(\d\d(?:\d\d)?)'])
                                + dateOfEventAtEndPatStr, reOPTS)
dateOfEventIsoRevPat1 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                   + isoSepPatStr.join([r'(\d\d?)',r'(\d\d?)',r'(\d\d(?:\d\d)?)'])
                                   + r'[^\d]', reOPTS)
dateOfEventIsoPat2 = re.compile(r'.{1,3000}?[^\d]'
                                + isoSepPatStr.join([r'(\d\d\d\d)',r'(\d\d?)',r'(\d\d?)'])
                                + dateOfEventAtEndPatStr, reOPTS)
dateOfEventIsoRevPat2 = re.compile(dateOfEventAtStartPatStr + r'[^\d]'
                                   + isoSepPatStr.join([r'(\d\d\d\d)',r'(\d\d?)',r'(\d\d?)'])
                                   + r'[^\d]', reOPTS)
whitespacePat = re.compile(r'\s*', reOPTS)
def parseEventDate(info,mainText) :
    m = dateOfEventMonthPat1.match(mainText) or dateOfEventMonthRevPat1.match(mainText)
    if m :
        info['eventDate'] = '-'.join([whitespacePat.sub('',m.group(3)),
                                      monthNameToIso(m.group(1)),m.group(2).zfill(2)])
        return
    m = dateOfEventMonthPat2.match(mainText) or dateOfEventMonthRevPat2.match(mainText)
    if m :
        info['eventDate'] = '-'.join([whitespacePat.sub('',m.group(3)),
                                      monthNameToIso(m.group(2)),m.group(1).zfill(2)])
        return
    m = dateOfEventIsoPat1.match(mainText) or dateOfEventIsoRevPat1.match(mainText)
    if m :
        info['eventDate'] = '-'.join([('20' if len(m.group(3))==2 else '')+m.group(3),
                                      m.group(1).zfill(2),m.group(2).zfill(2)])
        return
    m = dateOfEventIsoPat2.match(mainText) or dateOfEventIsoRevPat2.match(mainText)
    if m :
        info['eventDate'] = '-'.join([m.group(1),m.group(2).zfill(2),m.group(3).zfill(2)])
        return
    print('NO EVENT DATE!', end=' ')


def parse13GD(accNo, formType=None, info=None, textLimit=basicInfo.defaultTextLimit, debug=False) :
    if info is None :
        info = basicInfo.getSecFormInfo(accNo, formType=formType)
    if 'filedByCik' not in info :
        print('No filed by CIK!', end=' ')
    links = info['links']
    if len(links) == 0 :
        print('NO LINKS LIST!', end=' ')
        info['positions'] = []
    else :
        toFormat = 'text' if links[0][3].endswith('.txt') else 'souptext'
        mainText = utils.downloadSecUrl(links[0][3], toFormat=toFormat)
        parseEventDate(info,mainText)
        info['positions'] = getSec13NshAndPctFromText2(mainText,accNo, debug=debug)
        for cusipSearchPat in cusipSearchPats :
            m = cusipSearchPat.match(mainText)
            if m is not None :
                break
        if m is None :
            if not ('0001504304' in info['ciks'] or '0001067621' in info['ciks']) :
                # suppress the message for 0001504304 - Bulldog Investors
                # and 0001067621 - <NAME>
                # - they don't report CUSIPs in their filings
                print('no CUSIP found!', end=' ')
        else :
            cusip = spaceDashPat.sub('',m.group(1))
            if len(cusip) == 6 :
                print('adding 10 to CUSIP', cusip, end=' ')
                cusip = cusip + '10'
            if len(cusip) == 8 :
                print('adding checksum to CUSIP', cusip, end=' ')
                if cusipChecksum('0'+cusip[:7]) == cusip[7] :
                    cusip = '0'+cusip
                else :
                    cusip = cusip + cusipChecksum(cusip)
            if len(cusip)!=9 or cusip[8]!=cusipChecksum(cusip) :
                print('invalid CUSIP!', cusip, end=' ')
            info['cusip'] = cusip.upper()
            # print('CUSIP-'+cusip,end=' ')
        if formType is None :
            formType = links[0][2]
        if formType.upper().startswith('SC 13D') :
            m = purposePat.search(mainText)
            if m is None :
                print('no purpose!', end=' ')
            else :
                info['purpose'] = m.group(1)[:textLimit]
    if len(info['positions']) == 0 :
        print('no positions found!', end=' ')
    return info #,mainText

class scraper13G(infoScraper.scraperBase) :
    @utils.delegates(infoScraper.scraperBase.__init__)
    def __init__(self, infoDir=default13GDir, **kwargs) :
        super().__init__(infoDir, 'SC 13G', **kwargs)
    def init_for_13D(self, infoDir, **kwargs) :
        super().__init__(infoDir, 'SC 13D', **kwargs)
    def scrapeInfo(self, accNo, formType=None) :
        return parse13GD(accNo, formType=formType), None
    def rescrapeInfo(self, accNo, info) :
        return parse13GD(accNo, info=info)

# Cell

def updateCik13GDPos(scrapers, cik13GDPosMap=None,
                     cusipNames=None, cikNames=None, includeTickers=False) :
    """
    Generate or update a combined dict of percentage holdings:
        cik13GDPosMap: cik -> {cusip -> (eventDate, accNo, pct)}
    based on a list of 13G and 13D scrapers - pct is in the range [0.0 .. 100.0],
    as given in the 13G and 13D filings.

    If cusipNames and cikNames are supplied, both should be dicts, and CUSIPs
    encountered in 13G/D filings will have the corresponding CIK names added
    to the CUSIP names based on the filings (i.e. each 13D/G filing includes
    a CUSIP, and a corresponding subject CIK). If includeTickers is True,
    the ticker names will also be added based on the CIK to ticker file
    provided by the SEC.
    """
    if cik13GDPosMap is None :
        cik13GDPosMap = collections.defaultdict(dict)
    cikTo13GDs = collections.defaultdict(list)
    count = 0
    extraCusipNames = None if (cusipNames is None or cikNames is None) else {
        # CUSIP -> CIK mapping for some stocks that don't currently appear in any 13D or 13G filings:
        '931142103' : ('0000','walmart','104169'),
        '084670108' : ('0000','berkshire','1067983'),
    }
    for scraper in scrapers :
        for dStr, accNoToInfo in scraper.infoMap.items() :
            for accNo, info in accNoToInfo.items() :
                if info == 'ERROR' :
                    print('*** ERROR in ',accNo)
                elif 'filedByCik' not in info :
                    print('*** No filed-by CIK in',accNo)
                elif 'cusip' not in info :
                    print('No CUSIP in',accNo)
                else :
                    if len(info['positions']) == 0 :
                        print('*** No positions found in',accNo)
                        maxPctPos = 0.0
                    else :
                        maxPctPos = max(float(pct) for _,pct in info['positions'])
                    if 'eventDate' not in info :
                        eventDate = (utils.toDate(dStr)-datetime.timedelta(7)).isoformat()
                        print(f'No event date in {accNo}; using {eventDate}')
                    else :
                        eventDate = info['eventDate']
                    cusip = info['cusip']
                    filedByCik = info['filedByCik']
                    cikTo13GDs[filedByCik.lstrip('0')].append((cusip, eventDate, accNo,maxPctPos))
                    count += 1
                    if extraCusipNames is not None :
                        subjectCik = [cik for cik in info['ciks'] if cik!=filedByCik]
                        if len(subjectCik) != 1 :
                            print(f"missing or ambiguous subject CIK '{accNo}'")
                        elif cusip not in extraCusipNames or extraCusipNames[cusip][0] < dStr :
                            subjectCik = subjectCik[0].lstrip('0')
                            if subjectCik in cikNames :
                                extraCusipNames[cusip] = (dStr, cikNames[subjectCik], subjectCik)
                            else :
                                print(f"subject CIK {subjectCik} name not found '{accNo}'")
    if extraCusipNames is not None :
        count1 = count2 = 0
        if includeTickers :
            cikToTickers = dailyList.getCikToTickersMap()
        else :
            cikToTickers = collections.defaultdict(list)
        for cusip,name in cusipNames.items() :
            if cusip in extraCusipNames and 'CIK-' not in name :
                _,subjectCikName,subjectCik = extraCusipNames[cusip]
                if subjectCikName[:8].strip().lower() != name[:8].strip().lower() :
                    cusipNames[cusip] += f' - {subjectCikName}'
                cusipNames[cusip] += cikSymStr(subjectCik,cikToTickers[subjectCik])
                count1 += 1
        for cusip,(_,subjectCikName,subjectCik) in extraCusipNames.items() :
            if cusip not in cusipNames :
                cusipNames[cusip] = f'- {subjectCikName}{cikSymStr(subjectCik,cikToTickers[subjectCik])}'
                count2 += 1
        print('count1',count1,'count2',count2)
    print('total of',len(cikTo13GDs),'ciks,',count,'13G/D filings')
    for cik, cik13GDList in cikTo13GDs.items() :
        posMap = cik13GDPosMap[cik]
        for tup in cik13GDList :
            cusip = tup[0]
            if cusip not in posMap or posMap[cusip] < tup[1:] :
                posMap[cusip] = tup[1:]
    return cik13GDPosMap
def cikSymStr(cik,tickers) :
    return ' (' + ', '.join(sorted(tickers)[:8] + (['...'] if len(tickers)>8 else []) + ['CIK-'+cik])+ ')'

def calcBonusMap(cik13GDPosMap, max13GDBonus=0.2, min13GDBonus=0.02, max13GDCount=100,
                 allCusipCounter=None) :
    """
    Calculate "bonus fractions" for cusips where a 13G or 13D has been filed.
    13GD bonus fractions are 1.0/#positions, but restricted to [min13GDBonus..max13GDBonus]
    If max13GDCount is not None, restricts to investors with at most max13GDCount combined 13G
    and 13D positions. For positions between [1.0% .. 5.0%), the bonus is cut in half;
    positions below 1.0% aren't given a bonus.

    If supplied, allCusipCounter should be a Counter, and it will be updated to count
    all investors that have any position in each stock.

    Returns a dict: cik -> {cusip -> bonusfrac}
    """
    res = {}
    for cik,posMap in cik13GDPosMap.items() :
        if allCusipCounter is not None :
            allCusipCounter.update(posMap.keys())
        fullCusips, halfCusips = [], []
        for cusip,pos in posMap.items() :
            if pos[-1] >= 5.0 :
                fullCusips.append(cusip)
            elif pos[-1] >= 1.0 :
                halfCusips.append(cusip)
        totNCusips = len(fullCusips) + len(halfCusips)
        if totNCusips>0 and (max13GDCount is None or totNCusips<=max13GDCount) :
            bonus = min(max13GDBonus,max(min13GDBonus,1/totNCusips))
            res[cik] = dict((cusip,bonus) for cusip in fullCusips)
            res[cik].update((cusip,bonus*0.5) for cusip in halfCusips)
    return res
# AUTOGENERATED! DO NOT EDIT! File to edit: 10_scrape13G.ipynb (unless otherwise specified). # Cell # Cell # aggregatePatStr = r'aggregate.?\s+amount\s+(?:ben|own)' # percentOfClassPatStr = r'percent\s+(?:of|or)\s+class\s+(?:re|pr)' # typeOfRepPatStr = r'type.?\s+(?:of|or)\s+(?:rep|per)' # nSharesPatStr = r'\D(?!9\s|9\D\D)(\d+(?:[,.]\d\d\d)*)' # try to avoid taking item number 9 as share count # try to avoid taking item number 9 as share count # if percentage isn't followed by a % character, look for a plain number but # try to avoid using the 9 in "mentioned in item 9" verbiage for the percentage # ditto for "mentioned in item 11" verbiage # print(m.groups()) # nSh = nSh.replace('.',',') #|no)' # suppress the message for 0001504304 - Bulldog Investors # and 0001067621 - <NAME> # - they don't report CUSIPs in their filings # print('CUSIP-'+cusip,end=' ') #,mainText # Cell Generate or update a combined dict of percentage holdings:         cik13GDPosMap: cik -> {cusip -> (eventDate, accNo, pct)}     based on a list of 13G and 13D scrapers - pct is in the range [0.0 .. 100.0],     as given in the 13G and 13D filings.      If cusipNames and cikNames are supplied, both should be dicts, and CUSIPs     encountered in 13G/D filings will have the corresponding CIK names added     to the CUSIP names based on the filings (i.e. each 13D/G filing includes     a CUSIP, and a corresponding subject CIK). If includeTickers is True,     the ticker names will also be added based on the CIK to ticker file     provided by the SEC. # CUSIP -> CIK mapping for some stocks that don't currently appear in any 13D or 13G filings: Calculate "bonus fractions" for cusips where a 13G or 13D has been filed.     13GD bonus fractions are 1.0/#positions, but restricted to [min13GDBonus..max13GDBonus]     If max13GDCount is not None, restricts to investors with at most max13GDCount combined 13G     and 13D positions. For positions between [1.0% .. 5.0%), the bonus is cut in half;     positions below 1.0% aren't given a bonus.      If supplied, allCusipCounter should be a Counter, and it will be updated to count     all investors that have any position in each stock.      Returns a dict: cik -> {cusip -> bonusfrac}