# Copyright (c) 2018-2022 curoky(<EMAIL>).
#
# This file is part of my-own-x.
# See https://github.com/curoky/my-own-x for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
import re
from pathlib import Path
from typing import List

import pypinyin
import pypinyin.pinyin_dict
import pypinyin.style
import requests
from rime import RimeDict

ThunlpList = map(lambda url: f'https://raw.githubusercontent.com/thunlp/THUOCL/master/data/{url}', [
    'THUOCL_IT.txt', 'THUOCL_animal.txt', 'THUOCL_caijing.txt', 'THUOCL_car.txt',
    'THUOCL_chengyu.txt', 'THUOCL_diming.txt', 'THUOCL_food.txt', 'THUOCL_law.txt',
    'THUOCL_lishimingren.txt', 'THUOCL_medical.txt', 'THUOCL_poem.txt'
])

# https://github.com/qianzhengyang/AllDataPackages
# https://github.com/fighting41love/funNLP
# https://github.com/fkxxyz/rime-cloverpinyin/blob/master/src/thuocl2rime


def getThunlpDict() -> List[RimeDict]:
    initials_set = set(pypinyin.style._constants._INITIALS)  # 声母表
    logging.info(f'initials_set: {initials_set}')

    # 检查每个字都为合法汉字的字符串
    def checkPhrase(phrase):
        for c in phrase:
            if ord(c) not in pypinyin.pinyin_dict.pinyin_dict:
                return False
        return True

    count_no2 = 0
    count_nohanzi = 0
    rime_dicts = []
    for fname, content in map(
            lambda url: [os.path.basename(url), requests.get(url).text], ThunlpList):
        logging.info(f'download {fname}')
        lines = list(
            map(lambda line: re.sub(r'[ \t]+', ' ', line),
                content.replace('\r', '\n').split('\n')))
        Path(f'{fname}').write_text('\n'.join(lines))

        logging.info(f'{fname}: {len(lines)} lines')

        rime_dict = RimeDict(name=fname, words=[])
        for item in map(lambda line: line.split(' '), lines):
            if len(item) != 2:
                count_no2 += 1
                logging.warn(f'format error: len({item})  != 2')
                continue

            # 检查该词组的每个字必须为汉字
            if not checkPhrase(item[0]):
                count_nohanzi += 1
                # logging.warn(f'format error: ({item}) not hanzi')
                continue

            # 获取该词组的拼音
            pinyin = pypinyin.lazy_pinyin(item[0])
            for p in pinyin:
                if p in initials_set:  # 拼音不能只包含声母
                    break
            else:
                pinyin = ' '.join(pinyin)
                rime_dict.words.append(RimeDict.Word(word=item[0], py=pinyin, weight=1))
        rime_dicts.append(rime_dict)

    logging.info(f'count_no2: {count_no2}')
    logging.info(f'count_nohanzi: {count_nohanzi}')
    return rime_dicts

# Copyright (c) 2018-2022 curoky(<EMAIL>).
#
# This file is part of my-own-x.
# See https://github.com/curoky/my-own-x for further info.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import os
import re
from pathlib import Path
from typing import List

import pypinyin
import pypinyin.pinyin_dict
import pypinyin.style
import requests
from rime import RimeDict

ThunlpList = map(lambda url: f'https://raw.githubusercontent.com/thunlp/THUOCL/master/data/{url}', [
    'THUOCL_IT.txt', 'THUOCL_animal.txt', 'THUOCL_caijing.txt', 'THUOCL_car.txt',
    'THUOCL_chengyu.txt', 'THUOCL_diming.txt', 'THUOCL_food.txt', 'THUOCL_law.txt',
    'THUOCL_lishimingren.txt', 'THUOCL_medical.txt', 'THUOCL_poem.txt'
])

# https://github.com/qianzhengyang/AllDataPackages
# https://github.com/fighting41love/funNLP
# https://github.com/fkxxyz/rime-cloverpinyin/blob/master/src/thuocl2rime


def getThunlpDict() -> List[RimeDict]:
    initials_set = set(pypinyin.style._constants._INITIALS)  # 声母表
    logging.info(f'initials_set: {initials_set}')

    # 检查每个字都为合法汉字的字符串
    def checkPhrase(phrase):
        for c in phrase:
            if ord(c) not in pypinyin.pinyin_dict.pinyin_dict:
                return False
        return True

    count_no2 = 0
    count_nohanzi = 0
    rime_dicts = []
    for fname, content in map(
            lambda url: [os.path.basename(url), requests.get(url).text], ThunlpList):
        logging.info(f'download {fname}')
        lines = list(
            map(lambda line: re.sub(r'[ \t]+', ' ', line),
                content.replace('\r', '\n').split('\n')))
        Path(f'{fname}').write_text('\n'.join(lines))

        logging.info(f'{fname}: {len(lines)} lines')

        rime_dict = RimeDict(name=fname, words=[])
        for item in map(lambda line: line.split(' '), lines):
            if len(item) != 2:
                count_no2 += 1
                logging.warn(f'format error: len({item})  != 2')
                continue

            # 检查该词组的每个字必须为汉字
            if not checkPhrase(item[0]):
                count_nohanzi += 1
                # logging.warn(f'format error: ({item}) not hanzi')
                continue

            # 获取该词组的拼音
            pinyin = pypinyin.lazy_pinyin(item[0])
            for p in pinyin:
                if p in initials_set:  # 拼音不能只包含声母
                    break
            else:
                pinyin = ' '.join(pinyin)
                rime_dict.words.append(RimeDict.Word(word=item[0], py=pinyin, weight=1))
        rime_dicts.append(rime_dict)

    logging.info(f'count_no2: {count_no2}')
    logging.info(f'count_nohanzi: {count_nohanzi}')
    return rime_dicts

# Copyright (c) 2018-2022 curoky(<EMAIL>). # # This file is part of my-own-x. # See https://github.com/curoky/my-own-x for further info. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # #     http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # https://github.com/qianzhengyang/AllDataPackages # https://github.com/fighting41love/funNLP # https://github.com/fkxxyz/rime-cloverpinyin/blob/master/src/thuocl2rime # 声母表 # 检查每个字都为合法汉字的字符串 # 检查该词组的每个字必须为汉字 # logging.warn(f'format error: ({item}) not hanzi') # 获取该词组的拼音 # 拼音不能只包含声母