class QQ(DWM):  # v.qq.com
    handle_list = ['/v\.qq\.com/x/cover/']

    def __init__(self):
        DWM.__init__(self, 'auto')
        self.extra_headers['Referer'] = SWF_REFERER

    def getvkey(self, url, vid, vt, resolution, filename):
        rand = random.random()
        ckey = echo_ckeyv3(vid, PLAYER_GUID, rand,
                           player_version=PLAYER_VERSION, platform=PLATFORM)
        params = {
            'guid': PLAYER_GUID,
            'platform': PLATFORM,
            'vt': vt,
            'linkver': 2,
            'vid': vid,
            'lnk': vid,
            'charge': 0,
            'cKey': ckey,
            'encryptVer': '5.4',
            'otype': 'xml',
            'filename': filename,
            'ehost': url,
            'format': resolution,
            'appver': PLAYER_VERSION,
            'ran': rand,
        }

        hutf = self.get_hutf('http://vv.video.qq.com/getvkey',
                             postdata=urllib.urlencode(params))
        mp = MyHtmlParser(tidy=False)
        mp.feed(hutf)
        debug(str(mp.root_node))
        return {
            'filename': mp.select('filename')[0].text,
            'br': float(mp.select('br')[0].text),
            'key': mp.select('key')[0].text
        }

    def getvclip(self, url, vid, vt, resolution, idx):
        rand = random.random()
        ckey = echo_ckeyv3(vid, PLAYER_GUID, rand,
                           player_version=PLAYER_VERSION, platform=PLATFORM)
        params = {
            'buffer': 0,
            'guid': PLAYER_GUID,
            'vt': vt,
            'ltime': 77,
            'fmt': 'auto',
            'vid': vid,
            'platform': PLATFORM,
            'cKey': ckey,
            'format': resolution,
            'speed': random.randint(1000, 3000),
            'encryptVer': '5.4',
            'idx': idx,
            'appver': PLAYER_VERSION,
            'ehost': url,
            'dltype': 1,
            'charge': 0,
            'otype': 'xml',
            'ran': '%.16f' % rand
        }

        hutf = self.get_hutf('http://vv.video.qq.com/getvclip',
                             postdata=urllib.urlencode(params))
        mp = MyHtmlParser(tidy=False)
        mp.feed(hutf)

        return {
            'filename': mp.select('vi>fn')[0].text,
            'br': float(mp.select('vi>br')[0].text),
            'fmt': mp.select('vi>fmt')[0].text,
            'key': mp.select('vi>key')[0].text,
            'md5': mp.select('vi>md5')[0].text,
            'fs': int(mp.select('vi>fs')[0].text)
        }

    def getvinfo(self, url, fmt='shd'):
        hutf = self.get_hutf(url)
        #echo(hutf)
        ss = SelStr('script[r-notemplate=true]', hutf)
        for s in ss:
            if 'VIDEO_INFO' in s.text:
                data = s.text
                break
        match = re.search('var\s+COVER_INFO\s?=\s?({[^;]+);', data)
        cover_info = to_dict(match.group(1))
        match = re.search('var\s+VIDEO_INFO\s?=\s?({[^;]+);', data)
        video_info = to_dict(match.group(1))
        title = video_info['title']
        vid = video_info['vid']
        echo('title =', title, 'vid =', vid)
        mp = self.get_vinfo_do(url, vid, fmt)
        return title, vid, mp

    def getvinfo_iframe_player(self, url, fmt='shd'):
        # 'http://v.qq.com/iframe/player.html?vid=i00167ai266&tiny=0&auto=0'
        res = urlparse(url)
        qs = parse_qs(res.query)
        vid = qs['vid'][0]
        mp = self.get_vinfo_do(url, vid, fmt)
        return self.title, vid, mp

    def get_vinfo_do(self, url, vid, fmt):
        #getvinfo(target_dir, page_url, video_info['vid'])
        rand = random.random()
        ckey = echo_ckeyv3(vid, PLAYER_GUID, rand,
                           player_version=PLAYER_VERSION, platform=PLATFORM)
        params = {
            'newplatform': PLATFORM,
            'guid': PLAYER_GUID,
            'pid': PLAYER_PID,
            'speed': random.randint(5000, 9000),
            'vids': vid,
            'fp2p': 1,
            'dtype': 3,
            'linkver': 2,
            'ehost': url,
            'fhdswitch': 0,
            'cKey': ckey,
            'vid': vid,
            'appver': PLAYER_VERSION,
            'ran': '%.16f' % rand,
            'utype': 0,
            'encryptVer': '5.4',
            'defnpayver': 1,
            'charge': 0,
            'ip': '',
            'otype': 'xml',
            'platform': PLATFORM,
        }
        if fmt:
            params['defaultfmt'] = fmt
            params['defn'] = fmt
        hutf = self.get_html('http://vv.video.qq.com/getvinfo',
                             postdata=urllib.urlencode(params))
        #echo(hutf)
        mp = MyHtmlParser(tidy=False)
        mp.feed(hutf)
        debug(str(mp.root_node))
        return mp

    def query_info(self, url):
        #url = 'https://v.qq.com/x/cover/ijilh0frmu96sbf/x0017evzp6n.html'
        if 'iframe/player' in url:
            title, vid, mp = self.getvinfo_iframe_player(url)
        else:
            title, vid, mp = self.getvinfo(url)
        dlt = mp.select('dltype')[0].text.strip()
        if dlt == '3':
            return self.query_info_dlt3(url, title, vid, mp)
        slid = None
        mfs = 0
        # <fi><sl>0</sl><br>1500</br><id>11401</id><name>shd</name><lmt>0</lmt><sb>1</sb><cname>超清;(720P)</cname><fs>304995197</fs></fi>
        for ff in mp.select("fl > fi"):
            #print ff
            fs = int(ff.select("fs")[0].text)
            if fs > mfs:
                name = ff.select("name")[0].text
                fiid = ff.select("id")[0].text
                cname = ff.select("cname")[0].text
                mfs = fs
                slid = fiid
        echo(cname)

        for vi in mp.select('vl>vi'):
            video_type = int(vi.select('videotype')[0].text)
            if video_type == 1:
                video_type = 'flv'
            elif video_type == 2:
                video_type = 'mp4'
            else:
                video_type = 'unknown'

            video_id = vi.select('vid')[0].text

            cdn_host = vi.select('ul>ui>url')[0].text
            # this will much speed up
            cdn_host = "http://videohy.tc.qq.com/vlive.qqvideo.tc.qq.com"
            vt = vi.select('ul>ui>vt')[0].text
            fn = vi.select('fn')[0].text
            fs = int(vi.select('fs')[0].text)
            # failed https://v.qq.com/x/cover/i200hs4ip5a6u7a.html
            clfc = vi.select('cl>fc')
            fc = 0
            if clfc:
                fc = int(clfc[0].text)
            echo('video_id=', video_id)
            echo('cdn_host=', cdn_host)
            echo('fc=', fc)

            urls = []

            if fc == 0:
                vkey = self.getvkey(url, vid, vt, slid, fn)
                debug(vkey)
                filename = vkey.get('filename')
                target_file = os.path.join(target_dir, filename)
                cdn_url = '%s/%s' % (cdn_host, filename)
                key = vkey.get('key')
                br = vkey.get('br')
                urls.append(self.make_url(cdn_url, key, br, video_type, fs))
            else:
                for ci in vi.select('cl>ci'):
                    idx = int(ci.select('idx')[0].text)
                    cd = float(ci.select('cd')[0].text)
                    md5 = ci.select('cmd5')[0].text

                    vclip = self.getvclip(url, vid, vt, slid, idx)

                    filename = vclip['filename']
                    key = vclip['key']

                    cdn_url = '%s/%s' % (cdn_host, filename)
                    echo('cdn_url=', cdn_url)
                    urls.append(self.make_url(cdn_url, key, vclip['br'],
                                              vclip['fmt'], vclip['fs']))
        echo("urls =", urls)
        #k, tsize = self.get_total_size(urls)
        return title, video_type, urls, None

    def make_url(self, url, vkey, br, fmt, size, sp=0):
        params = {
            'sdtfrom': 'v1000',
            'type': 'mp4',
            'vkey': vkey,
            'platform': PLATFORM,
            'br': br,
            'fmt': fmt,
            'sp': sp,
            'guid': PLAYER_GUID
        }
        return '%s?%s' % (url, urllib.urlencode(params))

    def get_playlist(self, url):
        res = urlparse(url)
        pre = "%s://%s" % (res.scheme, res.netloc)
        debug("pre =", pre)
        hutf = self.get_hutf(url)
        urls = []
        #for node in SelStr('a[_stat="videolist:click"]', hutf):
        for node in SelStr('div.mod_episode > span.item > '
                           'a[_stat="videolist:click"][title]', hutf):
            n = node.text.strip()
            try:
                cnt = int(n)
            except ValueError:
                continue
            ii = node.select('i.mark_v > img[alt]')
            if ii and ii[0]['alt'] == U('预告'):
                continue
            u = (n + node['title'], pre + node['href'])
            debug(*u)
            urls.append(u)
        return urls

    def query_info_dlt3(self, url, title, vid, mp):
        #url = 'https://v.qq.com/x/cover/i200hs4ip5a6u7a.html'
        #t, vid, mp = self.getvinfo(url) #, fmt='fhd')
        mi, mz = None, 0
        for fi in mp.select("fl > fi"):
            sz = int(fi.select("fs")[0].text)
            if sz > mz and fi.select('lmt')[0].text == '0':
                mi = fi
                mz = sz
        debug("cname =", mi.select('cname')[0].text)
        vlifs = int(mp.select('vl > vi > fs')[0].text)
        fm = mi.select('name')[0].text.strip()
        if vlifs < mz:
            echo('need query again, current size=%d, best size=%d' % (vlifs,
                                                                      mz))
            echo('new query with fmt=' + fm)
            t, vid, mp = self.getvinfo(url, fmt=fm)
        for ui in mp.select("vl > vi > ul > ui"):
            ur = ui.select('url')[0].text.strip()
            if 'default' in ur:
                break
            #break
        keyid = mp.select('vl > vi > keyid')[0].text.strip()
        debug("ur =", ur)
        pt = ui.select('hls > pt')[0].text.strip()
        tp = ui.select('hls > ftype')[0].text.strip()
        um = ur + pt + "&type=" + tp + "&fmt=" + fm
        debug("um =", um)
        hutf = self.get_hutf(um, raw=True)
        #echo(hutf)
        urls = []
        for line in hutf.split('\n'):
            if keyid in line:
                urls.append(ur + line.strip())
        debug('urls =', urls)
        return title, tp, urls, mz

    def test(self):
        url = 'https://y.qq.com/portal/mv/v/s0017amxyfd.html'   # don't care music
        url = 'https://v.qq.com/x/cover/tu0kfx77pkwk3t6.html?vid=k00205g5xhk' # don't add proxy='auto'
        url = 'http://v.qq.com/iframe/player.html?vid=i00167ai266&tiny=0&auto=0'
        hutf = self.get_hutf(url)
        #title, vid, mp = self.getvinfo(url)
        echo(hutf) def SelStr(sel, data):
    mp = MyHtmlParser(tidy=False)
    mp.feed(data)
    return mp.select(sel) class YOUKU(DWM):
    handle_list = ['.youku.com/']
    stream_types = [
        ('mp4hd3', 'flv'),
        ('hd3',    'flv', '1080P'),
        ('mp4hd2', 'flv'),
        ('hd2',    'flv', '超清'),
        ('mp4hd',  'mp4'),
        ('mp4',    'mp4', '高清'),
        ('flvhd',  'flv', '标清'),
        ('flv',    'flv', '标清'),
        ('3gphd',  '3gp', '标清（3GP）'),
    ]

    f_code_1 = 'becaf9be'
    f_code_2 = 'bf7e5f01'

    ctype = 12  # differ from 86

    @staticmethod
    def trans_e(a, c):
        """str, str->str
        This is an RC4 encryption."""
        f = h = 0
        b = list(range(256))
        result = ''
        while h < 256:
            f = (f + b[h] + ord(a[h % len(a)])) % 256
            b[h], b[f] = b[f], b[h]
            h += 1
        q = f = h = 0
        while q < len(c):
            h = (h + 1) % 256
            f = (f + b[h]) % 256
            b[h], b[f] = b[f], b[h]
            if isinstance(c[q], int):
                result += chr(c[q] ^ b[(b[h] + b[f]) % 256])
            else:
                result += chr(ord(c[q]) ^ b[(b[h] + b[f]) % 256])
            q += 1
        return result

    def generate_ep(self, fileid, sid, token):
        ep = self.trans_e(self.f_code_2,  # use the 86 fcode if using 86
                          sid + '_' + fileid + '_' + token)
        try:
            ep = ''.join(ep).encode('latin1')
        except UnicodeDecodeError:  # py2 ep is ep, py3 need conevert it to bytes
            pass
        #echo("ep =", ep)
        return parse.quote(base64.b64encode(ep), safe='~()*!.\'')

    def __init__(self):
        DWM.__init__(self, proxy="auto")
        #ip = "220.181.111.%d" % random.randint(1, 254)
        #self.extra_headers['X-Forwarded-For'] = ip,
        self.extra_headers['Referer'] = 'http://static.youku.com/'
        # this is key!
        self.extra_headers['Cookie'] = '__ysuid={}'.format(time.time())

    def get_vid_from_url(self, url):
        """Extracts video ID from URL.
        """
        # http://player.youku.com/embed/XMTY3NzI5NTU3Ng==
        #return match1(url, r'youku\.com/v_show/id_([a-zA-Z0-9=]+)') or \
        #  match1(url, r'player\.youku\.com/player\.php/sid/([a-zA-Z0-9=]+)/v\.swf') or \
        #  match1(url, r'loader\.swf\?VideoIDS=([a-zA-Z0-9=]+)') or \
        #  match1(url, r'player\.youku\.com/embed/([a-zA-Z0-9=]+)')
        return match1(url, r'youku\.com/v_show/id_([a-zA-Z0-9=]+)',
                      r'player\.youku\.com/embed/([a-zA-Z0-9=]+)')[0]

    def get_stream(self, data):
        for dat in self.stream_types:
            for ss in data['stream']:
                if dat[0] == ss['stream_type']:
                    echo(dat)
                    return dat[0], dat[1], ss
        echo(data)

    def query_info(self, url):
        # return title, ext, urls, size
        vid = self.get_vid_from_url(url)
        echo("vid =", vid)
        api_url = 'http://play.youku.com/play/get.json?vid=%s&ct=12' % vid
        hutf = self.get_hutf(api_url)
        meta = json.loads(hutf)
        data12 = meta['data']
        data = data12
        debug(data)

        title = data['video']['title']
        sec_ep = data12['security']['encrypt_string']
        sec_ip = data12['security']['ip']
        echo(title, sec_ep, sec_ip)
        stype, ext, stm = self.get_stream(data)

        # get url list
        e_code = self.trans_e(self.f_code_1,
                              base64.b64decode(sec_ep.encode('ascii')))
                              #base64.b64decode(bytes(sec_ep, 'ascii')) )
        sid, token = e_code.split('_')
        echo(sid, token)

        urls = []
        streamfileid = stm['stream_fileid']
        for no, seg in enumerate(stm['segs'], 0):
            k = seg['key']
            if k == -1:
                raise  # we hit the paywall; stop here
            fileid = "%s%02X%s" % (streamfileid[0:8], no, streamfileid[10:])
            ep = self.generate_ep(fileid, sid, token)
            q = parse.urlencode(dict(
                ctype = self.ctype,
                ev    = 1,
                K     = k,
                ep    = parse.unquote(ep),
                oip   = str(sec_ip),
                token = token,
                yxon  = 1
            ))
            u = 'http://k.youku.com/player/getFlvPath/sid/{sid}_00' \
                '/st/{container}/fileid/{fileid}?{q}'.format(
                    sid       = sid,
                    container = ext,
                    fileid    = fileid,
                    q         = q
                )
            html = self.get_html(u, True)
            for i in json.loads(html.decode("utf8")):
                echo(i['server'])
                urls.append(i['server'])
        k, size = self.get_total_size(urls)
        return title, ext, urls, size

    def get_playlist(self, url):
        hutf = self.get_hutf(url)
        #echo(hutf)
        urls = []
        for a in SelStr('div.tvlists div.item a', hutf):
            if not a.select("span.sn_ispreview"):
                urls.append((a.text, a['href']))
        return urls

    def test(self, args):
        url = 'http://list.youku.com/show/id_zcbfffbf2962411de83b1.html'
        url = 'http://v.youku.com/v_show/id_XMTEzMjYzMTIw.html' class DM(DWM):     # http://www.dailymotion.com/embed/video/
    handle_list = ['/www.dailymotion.com/embed/video/',
                   '/www.dailymotion.com/video/']

    def query_info(self, url):
        #url = "http://www.dailymotion.com/embed/video/k7alsxAgBgcMGaachYS"
        #url = "http://www.dailymotion.com/embed/video/k4BjypcByJGUTDl6Bvx"
        hutf = self.get_hutf(url)
        #echo(hutf)
        # "720":[{"type":"application\/x-mpegURL","url":"http:\/\/www.dailymotion.com\/cdn\/manifest\/video\/x2hpv0i.m3u8?auth=1482432896-2562-0fq84z9d-24047244e9a36f0f3fab8388642b74c1&include=720"},{"type":"video\/mp4","url":"http:\/\/www.dailymotion.com\/cdn\/H264-1280x720\/video\/x2hpv0i.mp4?auth=1482432896-2562-pvg451ll-4c251ca9aa8a1bf6f56c88d318eccd65"}]}
        m = re.search("var config = ([^\n]+);", hutf)
        j = json.loads(m.group(1))
        mr, mq, mu, ex = -2, '', '', ''
        qua = ['auto', '144', '240', '380', '480', '720']
        for k, vs in j['metadata']['qualities'].items():
            if k not in qua:
                echo("New qua", k)
                continue
            r = qua.index(k)
            if r > mr:
                for v in vs:
                    t = v.get('type', '')
                    if t.startswith('video/'):
                        mr, mq = r, k
                        mu = v['url']
                        ex = t[-3:]
        echo("ext=%s, mq=%s, url=%s" % (ex, mq, mu))
        title = j['metadata']['title'].strip('.')
        return title, ex, [mu], None

    def test(self):
        url = 'http://www.dailymotion.com/video/k336RLStrzbIGzl96CY'
        url = 'http://www.dailymotion.com/embed/video/k6ELdHzeVXWQmemlHkY&info=0' class OpenLoad(DWM):     # http://openload.co/
    handle_list = ['openload.co/embed/', 'oload\.tv/']

    def __init__(self):
        DWM.__init__(self)
        self.extra_headers['Referer'] = 'https://vjs.zencdn.net/swf/5.1.0/video-js.swf'

    def query_info1(self, url):
        # https://openload.io/embed/igdtpdeGltM/
        # https://openload.co/embed/isCWWnlsZLE/
        # https://openload.io/embed/biw7ytfelzU/
        # <span id="streamurl">isCWWnlsZLE~1481138074~208.91.0.0~g617lYdo</span>
        echo("phantomjs wait 300 ...")
        p = Popen(["./phantomjs", "dwm.js", "300", url], stdout=PIPE)
        html = p.stdout.read()
        hutf = html.decode('utf8')
        p.wait()
        debug(hutf)
        n = SelStr('h6', hutf)
        if n:
            echo(n[0].text)
            return self.title, None, [], None
        #vid = match1(url, r'haiuken.com/theatre/([^/]+)/')
        m = re.search('''openload.co/embed/([^/]+)/''', url)
        if m:
            uid = m.groups()[0]
            echo(uid)
        m = re.search('''<span id="streamurl">([^<>]+)</span>''', hutf)
        vid = m.groups()[0]
        if not vid.startswith(uid):     # TODO, try to decode it
            vid = uid + "~1497803146~64.180.0.0~eBodZDZa"
        echo(vid)
        url = "https://openload.co/stream/%s?mime=true" % vid

        # "https://openload.co/embed/kUEfGclsU9o/"
        n = SelStr("meta[name=og:title]", hutf)
        if n and self.title == UTITLE:
            self.title = n[0]['content']   # ="skyrim_no-audio_1080.mp4">"

        # https://openload.co/stream/isCWWnlsZLE~1481139117~208.91.0.0~mcLfSy5C?mime=true
        # video/mp4 584989307
        k, tsize = get_kind_size(url)
        k = k.split('/')[-1]
        if self.title.endswith('.' + k):
            self.title = self.title[:-4]
        return self.title, k, [url], tsize

    def query_info(self, url):
        uid = match1(url, '''openload.co/embed/([^/]+)''')
        hutf = self.chrome_hutf(url)
        vid = match1(hutf, r'>(%s[^<]+)<' % uid)
        url = "https://openload.co/stream/%s?mime=true" % vid
        echo(url)
        return self.title, None, [url], None

    def test(self, args):
        #https://openload.co/embed/GN4oyoh2bQY/
        #https://openload.co/stream/GN4oyoh2bQY~1497806882~64.180.0.0~AUcZ8f9j?mime=true
        #https://1fiag6g.oloadcdn.net/dl/l/zxbDTu3BltrypxdY/GN4oyoh2bQY/PoliceUnit38_06.mp4?mime=true

        # https://openload.co/embed/qCpGFs8AOa4/
        #url = 'https://openload.co/embed/9zS9QNUWxZ8/'
        #'https://openload.co/stream/9zS9QNUWxZ8~1497803146~64.180.0.0~eBodZDZa?mime=true'
        #'https://oqt1pl.oloadcdn.net/dl/l/vfG56RBHDh7gErUv/9zS9QNUWxZ8/PoliceUnit38_05.mp4?mime=true'
        #pass
        url = 'https://openload.co/embed/QM5ommgqrG8'
        url = 'https://openload.co/embed/Wx_SaRAFgO4/'
        url = 'https://openload.co/embed/TkRITZPJ0-8'
        #hutf = self.phantom_hutf(url)
        #echo(hutf)
        #hutf = open("/tmp/tmpC6Kwkk").read()
        uid = match1(url, '''openload.co/embed/([^/]+)/''')
        echo("uid =", uid)
        hutf = self.chrome_hutf(url)
        ret = match1(hutf,
                     r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<',
                     r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)',
                     r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<',
                     r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<',
                     r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)',
                     )
        echo(ret)
        ret = match1(hutf, r'>(Wx_SaRAFgO4[^<]+)<')
        echo(ret) class RapidVideo(DWM):     # http://rapidvideo.com/
    handle_list = ['\.rapidvideo\.com/embed/']
    labels = ['auto', '360p', '720p']

    def query_info(self, url):
        hutf = self.get_hutf(url)
        debug(hutf)
        title = SelStr('title', hutf)[0].text
        k = None
        if title.endswith('.mp4'):
            title, k = title[:-4], 'mp4'

        #url = "https://www.rapidvideo.com/embed/FUZ35WDLM7"
        # https://www3731.playercdn.net/187/0/G4i-UJ6bQxIZI6FWc_F5dg/1536365722/180905/692FUZ37O792IXDCUZDFX.mp4
        v = SelStr("video#videojs source", hutf)
        if v:
            u = v[0]["src"]
            return title, k, [u], None

        #url = 'https://www.rapidvideo.com/embed/ZsNSciBj'
        # https://admkis.playercdn.net/85/1/sQ52oTwwZ6vCo3Vk7-RS2g/1482741547/161202/063k10VmKldzoX8.mp4
        hutf = self.get_hutf(url, postdata='block=1')
        data = match1(hutf, 'jwplayer\("home_video"\)\.setup\(([^\(\)]+)\);')
        debug(data)
        data = match1(data, '"sources":\s*(\[[^\[\]]+\])')
        ml, u = 0, ''
        for src in json.loads(data):
            l = src['label']
            if l not in self.labels:
                echo("new label", l)
            i = self.labels.index(l)
            if i > ml:
                ml, u = i, src['file']
        debug(title, u)
        return title, k, [u], None

    def test(self, args):
        url = "https://www.rapidvideo.com/embed/FUZ35WDLM7"
        # https://www3731.playercdn.net/187/0/G4i-UJ6bQxIZI6FWc_F5dg/1536365722/180905/692FUZ37O792IXDCUZDFX.mp4
        #echo(self.query_info(url))
        hutf = self.get_hutf(url)
        #echo(hutf)
        d = SelStr("video#videojs source", hutf)
        u = d[0]["src"] class DWM(object):
    out_dir = './'
    info_only = False
    align_num = 0
    is_playlist = False
    handle_list = []
    get_html_url = ''
    no_proxy = False
    wget_cookie = {}
    no_check_certificate = False
    skim_output = False
    m3u8_slow_merge = False

    def __init__(self, proxy=None):
        global USER_AGENT
        self.redirh = HTTPRedirectHandler()
        self.cookie = HTTPCookieProcessor()
        self.rawopen = build_opener(self.redirh, self.cookie)
        if proxy is None or self.no_proxy:
            self.opener = self.rawopen
        elif proxy == 'auto':
            # proxy.uku.im:8888
            #self.proxyh = ProxyHandler({'http': "http://211.155.86.25:8888"})
            #self.proxyh = ProxyHandler({'http': "proxy.uku.im:8888"})
            self.proxyh = ProxyHandler({'http': "https://secure.uku.im:8443"})
            #self.proxyh = ProxyHandler({'http': "https://proxy.uku.im:443"})
            self.opener = build_opener(self.proxyh, self.redirh, self.cookie)
        else:
            self.proxyh = ProxyHandler(proxy)
            self.opener = build_opener(self.proxyh, self.redirh, self.cookie)
        self.extra_headers = {"User-Agent": USER_AGENT}

    def get_html(self, url, raw=False, postdata=None):
        '''
        Date: Mon, 13 Apr 2015 05:35:05 GMT
        Content-Type: text/html
        Transfer-Encoding: chunked
        Connection: close
        Server: Tengine
        Content-Encoding: gzip
        X-Cache: HIT from us-newyork-ubi.hdslb.com
        '''
        req = Request(url, headers=self.extra_headers)
        if raw:
            rep = self.rawopen.open(req, postdata)
        else:
            rep = self.opener.open(req, postdata)
        self.get_html_url = rep.geturl()
        hds = rep.info()
        # print(repr(info))
        data = rep.read()
        if hds.get("Content-Encoding") == 'gzip':
            echo("It is gzip")
            html = zlib.decompress(data, zlib.MAX_WBITS | 16)
            return html
        elif hds.get("Content-Encoding") == 'deflate':
            echo("It is deflate")
            # decompressobj = zlib.decompressobj(-zlib.MAX_WBITS)
            # return decompressobj.decompress(data) + decompressobj.flush()
            html = zlib.decompress(data, -zlib.MAX_WBITS)
            return html
        else:
            return data

    def get_hutf(self, *param, **dd):
        return self.get_html(*param, **dd).decode('utf8', 'ignore')

    def chrome_hutf(self, url):
        echo("google-chrome wait ...")
        p = subprocess.Popen(["google-chrome", "--headless", "--disable-gpu",
                              "--dump-dom", "--repl", url],
                             stdout=subprocess.PIPE)
        html = p.stdout.read()
        hutf = html.decode('utf8')
        p.wait()
        debug(hutf)
        return hutf

    def phantom_hutf(self, url, timeout=300, refer=None, postdata=None):
        echo('phantomjs wait', timeout, '...')
        cmd = ["./phantomjs", "dwm.js", str(timeout), url]
        if refer:
            cmd.append(refer)
        if postdata:
            cmd.append(postdata)
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
        html = p.stdout.read()
        hutf = html.decode('utf8')
        p.wait()
        return hutf

    def last_m3u8(self, src):
        us = [src]
        while len(us) == 1:
            last = us[0]
            us = self._get_m3u8_urls(last, self.get_html(last))
            t, s = get_kind_size(us[0])
            if t != 'm3u8':
                break
        return last

    def try_m3u8(self, src):
        #url = 'http://www.ttwanda.com/films/us/2091.html?ac'
        # http://www.ttwanda.com/films/us/1881.html?le  mp2t
        us = self._get_m3u8_urls(src, self.get_html(src))
        if len(us) == 1:
            t, s = get_kind_size(us[0])
            if t == 'm3u8':
                us = self.try_m3u8(us[0])
        return us

    def _get_m3u8_urls(self, src, data):
        bu = os.path.dirname(src) + "/"
        rt = "/".join(src.split('/')[:3])
        urls = []
        for line in data.split('\n'):
            line = line.strip()
            if not line or line.startswith("#"):
                continue
            if "://" in line:
                urls.append(line)
            elif line[0] == '/':
                urls.append(rt + line)
            else:
                urls.append(bu + line)
        return urls

    def get_outfn(self, title, ext, unum=0):
        outfn = of = os.path.join(self.out_dir, title + "." + ext)
        if unum > 1 and ext in tss:
            of = os.path.join(self.out_dir, title + ".mp4")
        if os.path.exists(of):
            echo(of + " exists")
            return None
        if not os.path.exists(self.out_dir):
            os.makedirs(self.out_dir)
        #return outfn
        return of

    def align_title_num(self, t):
        t2 = '-'.join(t.split('/'))
        if self.align_num < 2:
            return t2
        ns = re.split("(\d+)", t2, 1)
        return ("%%s%%0%dd%%s" % self.align_num) % (ns[0], int(ns[1]), ns[2])

    def get_total_size(self, urllist):
        echo("total %d" % len(urllist))
        if len(urllist) > 9:
            k, s = get_total_size_mt(urllist)
        else:
            k, s = get_total_size_st(urllist)
        return k, s

    def use_dwm_merge(self, urls, title, ext, clean=True):
        if self.no_merge:
            echo("skip merge")
            return
        merge(os.path.join(self.out_dir, title), ext, len(urls), clean)

    def download_urls_you_get(self, title, ext, urls, totalsize):
        sys.path.insert(1, '../you-get/src')
        from you_get.common import download_urls
        try:
            download_urls(urls, title, ext, totalsize, self.out_dir)
        except subprocess.CalledProcessError as sc:
            if "avconv" not in str(sc):
                raise sc
            # failed to merge because avconv disable concat protocol
            #from merge import merge
            #merge(os.path.join(self.out_dir, title), ext, len(urls), True)
            echo("")
            self.use_dwm_merge(urls, title, ext)
        except RuntimeError as r:
            if "reraise" not in str(r):
                raise r
            self.use_dwm_merge(urls, title, ext)

    def get_real_url(self, bu, rt, uri):
        if "://" in uri:
            u = uri
        elif uri[0] == '/':
            u = rt + uri
        else:
            u = bu + uri
        return u

    def try_key(self, bu, rt, dn, line):
        m = re.search('#EXT-X-KEY:.+URI="([^"]+)"', line)
        if not m:
            #EXT-X-MAP:URI="init-v1-a1.mp4"
            m = re.search('#EXT-X-MAP:.*URI="([^"]+)"', line)
            if not m:
                return line
        u = self.get_real_url(bu, rt, m.group(1))
        n = os.path.basename(u)
        if len(n) > 100:
            n = "long_%s_uri_%d" % (line[1:10], len(n))
        self.wget_one_url(os.path.join(dn, n), u, 0)
        return line.replace(m.group(1), n)

    def wget_m3u8(self, title, url):
        dn = os.path.join(self.out_dir, title + ".dwm")
        if not os.path.exists(dn):
            os.makedirs(dn)
        data = self.get_hutf(url)
        fm3u8 = os.path.join(dn, title + ".m3u8")
        fout = open(fm3u8, "w")
        bu = os.path.dirname(url) + "/"
        rt = "/".join(url.split('/')[:3])
        #lines = [l.strip() for l in data.split('\n')]
        #unum = len([l for l in lines if l and l[0] != '#'])
        #echo("unum =", unum)
        #cnt = 0
        dats = []
        lc = 0
        for line in data.split('\n'):
            lc += 1
            if not line or line.startswith("#"):
                line = self.try_key(bu, rt, dn, line)
                fout.write(line + "\n")
                #fout.flush()
                continue
            u = self.get_real_url(bu, rt, line)
            n = os.path.basename(u)
            if len(n) > 100:
                n = "%04d_%s" % (lc, n.split('?')[0])
            fout.write(n + "\n")
            dats.append((os.path.join(dn, n), u))
            #fout.flush()
            #cnt += 1
            #echo("progress = %.1f%%" % (100.0 * cnt / unum))
            #self.wget_one_url(os.path.join(dn, n), u, unum)
        fout.write("#%s\n" % url)
        fout.close()
        cnt = 0
        unum = len(dats)
        echo("unum =", unum)
        for t, u in dats:
            ret = self.wget_one_url(t, u, unum)
            if ret == 0:
                cnt += 1
                echo("progress = %.1f%%" % (100.0 * cnt / unum))
        if cnt == unum:
            echo("progress = 100%")
            echo("merge")
            self.avconv_m3u8(title, "mp4", fm3u8)
        else:
            echo("some download failed %d != %d, skip merge" % (cnt, unum))

    def download_m3u8(self, title, url):
        outfn = self.get_outfn(title, "mp4")
        if not outfn:
            return    # file exists
        echo("download", outfn)
        #if self.skim_output:
        #    self.wget_m3u8(title, url)
        #else:
        #    self.avconv_m3u8(title, "mp4", url)
        if self.parsed_args.avconv_m3u8:
            self.avconv_m3u8(title, "mp4", url)
        else:
            self.wget_m3u8(title, url)
        return outfn

    def avconv_m3u8(self, title, ext, url):
        outfn = self.get_outfn(title, ext)
        m3u8_merge(url, outfn, self.m3u8_slow_merge)
        ##if not title.endswith(".mp4"):
        ##    title += ".mp4"
        #cmds = ["avconv",
        #        "-allowed_extensions", "ALL",
        #        "-i", url,
        #        "-acodec", "copy",
        #        "-vcodec", "copy",
        #        "-f", "mp4",
        #        outfn + ".dwm",
        #        ]
        #debug(cmds)
        #p = subprocess.Popen(cmds, env={"LANG": "en_CA.UTF-8"})
        #p.wait()
        #if p.returncode == 0:
        #    os.rename(outfn + ".dwm", outfn)
        #    echo(outfn)

    def wget_urls(self, title, ext, urls, tsize):
        if ext is None:
            k, s = get_kind_size(urls[0])
            ext = k
        #echo("ext=%s, tsize=%d" % (ext, tsize))
        title = "_".join(title.split('/'))
        unum = len(urls)
        outfn = self.get_outfn(title, ext, unum)
        if not outfn:
            return    # file exists
        echo("download", outfn)
        if unum == 1:
            self.wget_one_url(outfn, urls[0], 1, tsize)
            return outfn
        #for cnt, url in enumerate(urls[self.dwn_skip:], start=self.dwn_skip):
        for cnt, url in enumerate(urls):
            echo("progress = %.1f%%" % (100.0 * cnt / len(urls)))
            outfn = self.get_outfn("%s[%02d]" % (title, cnt), ext)
            if outfn:
                self.wget_one_url(outfn, url, unum)
        echo("progress = 100%")
        self.use_dwm_merge(urls, title, ext, False)
        return outfn

    def wget_one_url(self, outfn, url, unum, totalsize=0):
        if unum > 1:
            echo("wget", outfn, "/", unum)
        else:
            echo("wget", outfn)
        dwnfn = outfn + ".dwm"
        cmds = ["wget",
                "-U", USER_AGENT,
                #"--wait", "10",
                #"--tries=50",
                #"--read-timeout=60",
                "--timeout=60",
                "--continue",
                "--no-use-server-timestamps",
                #"-S",
                #"--no-verbose",
                ]

        if self.skim_output:
            cmds.append("--quiet")
            #cmds.append("--show-progress")
            #cmds.append("--progress=bar:force")

        if self.no_check_certificate:
            cmds.append("--no-check-certificate")

        # --header='Cookie: FTN5K=af45f935;'
        if self.wget_cookie:
            cmds.append("--header=Cookie: " + self.wget_cookie)

        if isinstance(url, UO):
            cmds += ["-O", dwnfn, "--header=Referer: " + url.referer, url.url]
        else:
            cmds += ["-O", dwnfn, url]
        debug(cmds)
        for i in range(10):
            p = subprocess.Popen(cmds, env={"LANG": "en_CA.UTF-8"})
            if self.skim_output and totalsize > 0:
                p.poll()
                while p.returncode is None:
                    if os.path.exists(dwnfn):
                        echo("progress = %.1f%%" % (
                              100.0 * os.stat(dwnfn).st_size / totalsize))
                    sleep(2)
                    p.poll()
            else:
                p.wait()

            if p.returncode == 0:
                os.rename(dwnfn, outfn)
                break
            sleep(1 + i)
        return p.returncode

    def get_one(self, url, t=UTITLE, n=False):
        title, ext, urls, size = self.query_info(url)
        debug("get_one from query_info title=", title)
        if not urls:
            echo("Empty urls")
            return
        if not title or t != UTITLE:
            if ext and title.endswith("." + ext):
                title = t[:-len("." + ext)]
            else:
                title = t
        nt = norm_title(title)
        debug("norm_title return", nt)
        if n:
            title = nt
        else:
            echo('nt =', nt)
        if self.info_only:
            for url in urls:
                echo(url)
        title = self.align_title_num(title)
        if self.info_only or len(urls) == 1:
            if ext is None or size is None:
                e, s = self.get_total_size(urls)
                if ext is None:
                    ext = e
                if size is None:
                    size = s
            echo("title =", title, ext)
            echo("Size:\t%.2f MiB (%d Bytes)" % (round(size / 1048576.0, 2),
                                                 size))
        if self.info_only:
            return

        if ext == 'm3u8':
            outfn = self.download_m3u8(title, urls)
        else:
            outfn = self.download_urls(title, ext, urls, size)
        debug("post_uri = ", self.parsed_args.post_uri)
        debug("outfn = ", outfn)
        if self.parsed_args.post_uri and outfn:
            #post_file(outfn.encode('utf8'), self.parsed_args.post_uri)
            post_file(outfn, self.parsed_args.post_uri)

    def try_playlist(self, ispl, url):
        if ispl:
            urls = self.get_playlist(url)
            for t, u in urls:
                debug(t, u)
            return urls
        return None

    def get_playlist(self, page_url):
        #raise Exception("Not Implement Yet")
        echo("Playlist Not Implement")
        return []

    def clean_up(self):
        pass

    def test(self, args):
        echo("test Not Implement")

    @classmethod
    def can_handle_it(cls, url):
        for h in cls.handle_list:
            #if h in url:
            if re.search(h, url):
                return True
        return False def match1(text, *patterns):
    """Scans through a string for substrings matched some patterns
       (first-subgroups only).
    Args:
        text: A string to be scanned.
        patterns: Arbitrary number of regex patterns.
    Returns:
        When only one pattern is given, returns a string
        (None if no match found).
        When more than one pattern are given, returns a list of strings
        ([] if no match found).
    """
    if len(patterns) == 1:
        pattern = patterns[0]
        match = re.search(pattern, text)
        if match:
            return match.group(1)
        else:
            return None
    else:
        ret = []
        for pattern in patterns:
            match = re.search(pattern, text)
            if match:
                ret.append(match.group(1))
        return ret def echo(*args):
    sys.stdout.write(" ".join(map(str, args)) + "\n")
    sys.stdout.flush() def start(kls):
    global DEBUG
    p = argparse.ArgumentParser(description='Download Web Movie')
    #, add_help=False)
    p.add_argument('url', metavar='URL', type=str, action='store',
                   help='url of movie')
    p.add_argument('-i', '--info_only', action='store_true',
                   help='show information only')
    p.add_argument('-o', '--output', metavar='dir', action='store',
                   #help='where download file go, dir or url to post',
                   help='where download file go, dir',
                   default='.')
    p.add_argument('-n', '--norm_title', action='store_true',
                   help='normalize title')
    p.add_argument('-p', '--playlist_only', action='store_true',
                   help='try playlist only')
    p.add_argument('-P', '--not_playlist', action='store_true',
                   help='not try playlist')
    p.add_argument('--playlist_top', type=int, metavar='#', action='store',
                   help='only get top # of playlist', default=0)
    p.add_argument('--playlist_skip', type=int, metavar='#', action='store',
                   help='skip # in playlist', default=0)
    p.add_argument('--title', metavar='TITLE', action='store',
                   help='movie name if you want to define it',
                   default=UTITLE)
    p.add_argument('--post_uri', metavar='URI', action='store',
                   help='uri to post downloaded file')
    #p.add_argument('--wget_skip', type=int, metavar='#', action='store',
    #               help='wget skip # urls in list', default=0)
    p.add_argument('--align_num', type=int, metavar='#', action='store',
                   help='align number', default=0)
    p.add_argument('--cookie', metavar='COOKIE_STR', action='store',
                   help='input cookie for login', default='')
    p.add_argument('--user_agent', metavar='USER_AGENT', action='store',
                   help='pair with cookie for login', default='')
    p.add_argument('--no_merge', action='store_true',
                   help='skip merge video pieces')
    p.add_argument('--no_check_certificate', action='store_true',
                   help='not check https certificate')
    p.add_argument('--no_proxy', action='store_true',
                   help='disable auto proxy')
    p.add_argument('--skim_output', action='store_true',
                   help='only output data for WUI')
    p.add_argument('--avconv_m3u8', action='store_true',
                   help='download m3u8 by avconv')
    p.add_argument('--debug', action='store_true',
                   help='display debug message')
    p.add_argument('--testing', action='store_true',
                   help='do testing')
    args = p.parse_args()
    DEBUG = args.debug
    debug(args)

    if not getattr(kls, 'query_info', None):
        kls = kls(args.url)
        if kls is None:
            echo("Not support ", args.url)
            sys.exit(1)
    if args.title == "":
        args.title = UTITLE
    if py3:
        kls.title = args.title
    else:
        kls.title = args.title.decode('utf8')
    kls.out_dir = args.output
    kls.no_merge = args.no_merge
    kls.no_proxy = args.no_proxy
    kls.info_only = args.info_only
    kls.align_num = args.align_num
    kls.login_cookie = args.cookie
    kls.login_agent = args.user_agent
    DWM.no_check_certificate = args.no_check_certificate
    #if args.no_check_certificate:
    if kls.no_check_certificate:
        ssl._create_default_https_context = ssl._create_unverified_context
    kls.download_urls = DWM.wget_urls
    kls.skim_output = args.skim_output
    k = kls()
    k.parsed_args = args
    if args.testing:
        DEBUG = True
        k.test(args)
    else:
        run(k, args) def debug(*args):
    global DEBUG
    if DEBUG:
        echo(*args)
    return DEBUG UTITLE = "UnknownTitle"
import re
import sys
    from urllib import unquote
    from urlparse import urlparse
    from urllib.parse import unquote, urlparse
from qq import QQ
from mybs import SelStr
from youku import YOUKU
from dailymotion import DM
from openload import OpenLoad
from rapidvideo import RapidVideo
from comm import DWM, match1, echo, start, debug, UTITLE
# -*- coding: utf8 -*-

try:
except ImportError:



def h8decode(a, b):
    ss = ""
    l = len(b) + 1
    for i, s in enumerate(a):
        if 0 == i % l:
            ss += s
    return unquote("".join(reversed(ss))).split('+++')


class HAVE8(DWM):     # http://have8.com/
    #handle_list = ['have8tv\.com/v/(drama|movie)/\d+/\d+/'
    #               '(dailymotion|youku|openload|qq|14tv)\.html',
    #               'v\.have8\.tv/drama/\d+/\d+/m3u8\.html']
    handle_list = ['have8\.tv/(drama|movie)/\d+/\d+/'
                   '(dailymotion|youku|openload|qq|14tv|m3u8)\.html',
                   'autocarinsider\.com/(d)/\d+/\d+/'
                   '(rapidvideo|m3u8|openload)',
                   ]

    def get_vsrc(self, hutf):
        m = re.search('var vsource = "([^"]+)";', hutf)
        return m.group(1)

    def get_vid(self, hutf, idx=''):
        m = re.search('adrss\[0\] \= "([^"]+)"', hutf)
        data = m.group(1)
        vids = h8decode(data, self.get_vsrc(hutf))
        vids = [v.split('++') for v in vids]
        if not idx:
            return vids
        for i, v in vids:
            #echo(repr(i), repr(idx))
            if int(i) == int(idx):
                return v
        return None

    def query_info(self, url):
        #if '/v/drama/' in url:
        #    return self.query_info_drama(url)
        #if '/v/movie/' in url:
        #    return self.query_info_movie(url)
        if '.tv/drama/' in url or '.com/d/' in url:
            return self.query_info_drama(url)
        if '.tv/movie/' in url:
            return self.query_info_movie(url)
        return None

    def query_info_drama_m3u8(self, url):
        # http://v.have8.tv/drama/2/25832/m3u8.html?0-29-0
        # https://52dy.hanju2017.com/20180904/BN0R4K7Y/index.m3u8
        hutf = self.get_hutf(url)
        idx = self.get_idx(url)
        debug("idx=", idx)
        m = re.search('var title = "([^"]+)";', hutf)
        debug("m =", m.group(1))
        title = m.group(1) + "_E%02d" % int(idx)
        vid = self.get_vid(hutf, idx)
        vid = unquote(vid)
        debug("vid =", vid)
        urls = self.try_m3u8(vid)
        debug("urls =", len(urls), urls[0])
        return title, None, urls, None

    def get_idx(self, url):
        up = urlparse(url)
        sels = up.query.split('-')
        debug("sels =", sels)
        idx = 1
        if len(sels) > 1:
            idx = sels[1]
        return idx

    def query_info_drama(self, url):
        #url = "http://have8tv.com/v/drama/2/21852/dailymotion.html?0-1-0"
        hutf = self.get_hutf(url)
        #up = urlparse(url)
        #sels = up.query.split('-')
        #debug("sels =", sels)
        #idx = 1
        #if len(sels) > 1:
        #    idx = sels[1]
        source = self.get_vsrc(hutf)
        if source == 'm3u8':
            return self.query_info_drama_m3u8(url)
        idx = self.get_idx(url)
        title = SelStr('meta[name=description]', hutf)[0]['content']
        title = title + "_E%02d" % int(idx)
        vid = self.get_vid(hutf, idx)
        debug('vid =', vid)
        debug('source =', source)
        if source == 'dailymotion':
            u = 'http://www.dailymotion.com/embed/video/' + vid
            return DM().query_info(u)
        elif source == 'youku':
            u = 'http://player.youku.com/embed/' + vid

            return YOUKU().query_info(u)