# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# The our class for PDF files
# @Authors:  <NAME> and <NAME>
# @Version: 1.0
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# libraries
import cv2
import os
from pdf2image import convert_from_path
from PyPDF2 import PdfFileReader
from imutils import paths
import numpy as np
import pytesseract
import tempfile
import subprocess
import sys
import PyPDF2 as pyPdf
import random

class createDATA:
   
    __train = ""                              # folder of train files
    __folder_path = ""                        # path to main folder
    __images_path = ""                        # path to folder of images from first page
    # path to folder of JavaScript from pdf file
    __texts_path = ""
    # dictionary for translate PDF language to tessaract language
    __lan_lst = {
        "en-us": "eng",
        "en": "eng",
        "en-za": "eng",
        "en-gb": "eng",
        "en-in": "eng",
        "es-co": "spa",
        "es": "spa",
        "de-de": "deu",
        "fr-fr": "fra",
        "fr-ca": "fra"}

    # dictionary for /Root/Lang 1 - except; 2 - a file have not /Root/Lang; 3
    # - /Root/Lang = ''; 4 - language
    __ans_list = dict()

    # constructor
    def __init__(self, folder_path, train):
        self.__train = train
        self.__folder_path = folder_path
        self.__images_path = folder_path + "/IMAGES"
        self.__texts_path = folder_path + "/TEXTS"

    # this function update ans_list
    def add_ans_list(self, save_dir, base_filename, filename):
        try:
            name = os.path.join(save_dir, base_filename)
            pdfFile = PdfFileReader(open(filename, 'rb'), strict = False)
            catalog = pdfFile.trailer['/Root'].getObject()
            if "/Lang" in catalog:
                lang = catalog['/Lang'].getObject()
                if (lang == ''):
                    self.__ans_list.update({name: [3, 'None']})
                else:
                    lang = lang.lower()
                    language = self.__lan_lst.get(lang)
                    self.__ans_list.update({name: [4, language]})
            else:
                self.__ans_list.update({name: [2, 'None']})
        except Exception as ex:
            #print(filename)
            #print(ex)
            self.__ans_list.update({name: [1, 'None']})

    # this function generate random page
    def get_page_random(self, pdf):
        try:
            page_start = pdf.trailer["/Root"]["/PageLabels"]["/Nums"][1].getObject()["/St"]
        except:
            page_start = 1
        page_count = pdf.getNumPages()
        page_stop = page_start + page_count
        random_page = random.randint(page_start, page_stop)
        return random_page

    # this function convert pdf file to jpg file
    def convert(self, dirpdf):
        # dir of folder and filter for pdf files
        files = [f for f in os.listdir(dirpdf) if os.path.isfile(os.path.join(dirpdf, f))]
        files = list(filter(lambda f: f.endswith(('.pdf', '.PDF')), files))

        # variables for print information
        cnt_files = len(files)
        i = 0
        for filepdf in files:
            try:
                filename = os.path.join(dirpdf, filepdf)
                images_from_path = 0
                with tempfile.TemporaryDirectory() as path:
                    pdf = pyPdf.PdfFileReader(open(filename, "rb"))
                    random_page = self.get_page_random(pdf)
                    images_from_path = convert_from_path(filename, output_folder=path, last_page=random_page, first_page=random_page-1)

                base_filename = os.path.splitext(os.path.basename(filename))[0] + '.jpg'

                # save image
                for page in images_from_path:
                    page.save(os.path.join(self.__images_path, base_filename), 'JPEG')
                i += 1

                # update ans_list
                self.add_ans_list(self.__images_path, base_filename, filename)

                # show an update every 50 images
                if (i > 0 and i % 50 == 0):
                    print("[INFO] processed {}/{}".format(i, cnt_files))
            except Exception:
                # always keep track the error until the code has been clean
                print("[!] Convert PDF to JPEG " + filepdf)
                continue
                return False
        print("[INFO] processed {}/{}".format(cnt_files, cnt_files))
        return True

    # this function convert pdf file to image
    def convertOnePDF(self, filepdf):
        try:
            filename = os.path.join(self.__folder_path, filepdf)
            images_from_path = 0
            with tempfile.TemporaryDirectory() as path:
                pdf = pyPdf.PdfFileReader(open(filename, "rb"))
                random_page = self.get_page_random(pdf)
                images_from_path = convert_from_path(filename, output_folder=path, last_page=random_page, first_page=random_page-1)

            base_filename = os.path.splitext(os.path.basename(filename))[0] + '.jpg'

            # save image
            for page in images_from_path:
                page.save(os.path.join(self.__images_path, base_filename), 'JPEG')
                
            # update ans_list
            self.add_ans_list(self.__images_path, base_filename, filename)
            return True
        except Exception as ex:
            # print(ex)
            # always keep track the error until the code has been clean
            print("[!] Convert PDF to JPEG " + filepdf)
            return False

    # this function extract JavaScript from pdf file to txt
    def extractOnePDF(self, filepdf):
        err_path = os.path.join(self.__folder_path, 'classes/peepdf/errors.txt')
        try:
            filename = os.path.join(self.__folder_path, filepdf)
            self.ex_js(filename)
            # handling the case that previous file failed to parse
            errorfile = os.path.isfile(err_path)                                          # holds boolean value
            if errorfile:
                os.remove(err_path)
            else:
                fi_na = open(self.__texts_path + '/' + str(filepdf) +'.txt', 'w+')        # open text file for current file
                temp_file = open(self.__folder_path + '/classes/JSfromPDF.txt', 'r')
                # copy content from temp file to text file
                for line in temp_file.readlines():
                   try:     
                       fi_na.write(str(line))
                   except:
                       continue
                temp_file.close()
                fi_na.close()
            return True
        except Exception as ex:
            # print(ex)
            # always keep track the error until the code has been clean
            print("[!] Extract JavaScript to TXT " + filepdf)
            return False


    # function extract JS from pdf using peepdf
    def ex_js(self, filename):
        # run peepdf.py
        scriptpath = os.path.join(self.__folder_path + '/classes/peepdf/', 'peepdf.py')               # joins to path
        CommandOfExtract = self.__folder_path + '/classes/extractJS.txt'
        p = subprocess.Popen(['python', scriptpath, '-l', '-f', '-s', CommandOfExtract, filename])    # open subprocess and extract
        p.wait()


    # this function extract JavaScript from pdf file to txt
    def extract(self, dirpdf):
        # dir of folder and filter for pdf files
        files = [f for f in os.listdir(dirpdf) if os.path.isfile(os.path.join(dirpdf, f))]
        files = list(filter(lambda f: f.endswith(('.pdf', '.PDF')), files))

        # variables for print information
        cnt_files = len(files)
        i = 0
        err_path = os.path.join(self.__folder_path, 'classes/peepdf/errors.txt')
        for filepdf in files:
            i += 1
            try:
                filename = os.path.join(dirpdf, filepdf)
                self.ex_js(filename)
                # handling the case that previous file failed to parse
                errorfile = os.path.isfile(err_path)                                          # holds boolean value
                if errorfile:
                    os.remove(err_path)
                else:
                    fi_na = open(self.__texts_path + '/' + str(filepdf) +'.txt', 'w+')        # open text file for current file
                    temp_file = open(self.__folder_path + '/classes/JSfromPDF.txt', 'r')
                    # copy content from temp file to text file
                    try:
                        for line in temp_file.readlines():
                            fi_na.write(str(line))
                    except:
                        continue
                    temp_file.close()
                    fi_na.close()
                # show an update every 50 pdf files
                if (i > 0 and i % 50 == 0):
                    print("[INFO] processed {}/{}".format(i, cnt_files))
            except Exception as ex:
                #print(ex)
                # always keep track the error until the code has been clean
                print("[!] Extract JavaScript to TXT " + filepdf)
                return False
        print("[INFO] processed {}/{}".format(cnt_files, cnt_files))
        return True

    # this function return dictionary
    def getDict(self):
        return self.__ans_list

# -------------------------------------------------------------------------------------------------------------------------------------------------------------
# The our class for PDF files
# @Authors:  <NAME> and <NAME>
# @Version: 1.0
# -------------------------------------------------------------------------------------------------------------------------------------------------------------

# libraries
import cv2
import os
from pdf2image import convert_from_path
from PyPDF2 import PdfFileReader
from imutils import paths
import numpy as np
import pytesseract
import tempfile
import subprocess
import sys
import PyPDF2 as pyPdf
import random

class createDATA:
   
    __train = ""                              # folder of train files
    __folder_path = ""                        # path to main folder
    __images_path = ""                        # path to folder of images from first page
    # path to folder of JavaScript from pdf file
    __texts_path = ""
    # dictionary for translate PDF language to tessaract language
    __lan_lst = {
        "en-us": "eng",
        "en": "eng",
        "en-za": "eng",
        "en-gb": "eng",
        "en-in": "eng",
        "es-co": "spa",
        "es": "spa",
        "de-de": "deu",
        "fr-fr": "fra",
        "fr-ca": "fra"}

    # dictionary for /Root/Lang 1 - except; 2 - a file have not /Root/Lang; 3
    # - /Root/Lang = ''; 4 - language
    __ans_list = dict()

    # constructor
    def __init__(self, folder_path, train):
        self.__train = train
        self.__folder_path = folder_path
        self.__images_path = folder_path + "/IMAGES"
        self.__texts_path = folder_path + "/TEXTS"

    # this function update ans_list
    def add_ans_list(self, save_dir, base_filename, filename):
        try:
            name = os.path.join(save_dir, base_filename)
            pdfFile = PdfFileReader(open(filename, 'rb'), strict = False)
            catalog = pdfFile.trailer['/Root'].getObject()
            if "/Lang" in catalog:
                lang = catalog['/Lang'].getObject()
                if (lang == ''):
                    self.__ans_list.update({name: [3, 'None']})
                else:
                    lang = lang.lower()
                    language = self.__lan_lst.get(lang)
                    self.__ans_list.update({name: [4, language]})
            else:
                self.__ans_list.update({name: [2, 'None']})
        except Exception as ex:
            #print(filename)
            #print(ex)
            self.__ans_list.update({name: [1, 'None']})

    # this function generate random page
    def get_page_random(self, pdf):
        try:
            page_start = pdf.trailer["/Root"]["/PageLabels"]["/Nums"][1].getObject()["/St"]
        except:
            page_start = 1
        page_count = pdf.getNumPages()
        page_stop = page_start + page_count
        random_page = random.randint(page_start, page_stop)
        return random_page

    # this function convert pdf file to jpg file
    def convert(self, dirpdf):
        # dir of folder and filter for pdf files
        files = [f for f in os.listdir(dirpdf) if os.path.isfile(os.path.join(dirpdf, f))]
        files = list(filter(lambda f: f.endswith(('.pdf', '.PDF')), files))

        # variables for print information
        cnt_files = len(files)
        i = 0
        for filepdf in files:
            try:
                filename = os.path.join(dirpdf, filepdf)
                images_from_path = 0
                with tempfile.TemporaryDirectory() as path:
                    pdf = pyPdf.PdfFileReader(open(filename, "rb"))
                    random_page = self.get_page_random(pdf)
                    images_from_path = convert_from_path(filename, output_folder=path, last_page=random_page, first_page=random_page-1)

                base_filename = os.path.splitext(os.path.basename(filename))[0] + '.jpg'

                # save image
                for page in images_from_path:
                    page.save(os.path.join(self.__images_path, base_filename), 'JPEG')
                i += 1

                # update ans_list
                self.add_ans_list(self.__images_path, base_filename, filename)

                # show an update every 50 images
                if (i > 0 and i % 50 == 0):
                    print("[INFO] processed {}/{}".format(i, cnt_files))
            except Exception:
                # always keep track the error until the code has been clean
                print("[!] Convert PDF to JPEG " + filepdf)
                continue
                return False
        print("[INFO] processed {}/{}".format(cnt_files, cnt_files))
        return True

    # this function convert pdf file to image
    def convertOnePDF(self, filepdf):
        try:
            filename = os.path.join(self.__folder_path, filepdf)
            images_from_path = 0
            with tempfile.TemporaryDirectory() as path:
                pdf = pyPdf.PdfFileReader(open(filename, "rb"))
                random_page = self.get_page_random(pdf)
                images_from_path = convert_from_path(filename, output_folder=path, last_page=random_page, first_page=random_page-1)

            base_filename = os.path.splitext(os.path.basename(filename))[0] + '.jpg'

            # save image
            for page in images_from_path:
                page.save(os.path.join(self.__images_path, base_filename), 'JPEG')
                
            # update ans_list
            self.add_ans_list(self.__images_path, base_filename, filename)
            return True
        except Exception as ex:
            # print(ex)
            # always keep track the error until the code has been clean
            print("[!] Convert PDF to JPEG " + filepdf)
            return False

    # this function extract JavaScript from pdf file to txt
    def extractOnePDF(self, filepdf):
        err_path = os.path.join(self.__folder_path, 'classes/peepdf/errors.txt')
        try:
            filename = os.path.join(self.__folder_path, filepdf)
            self.ex_js(filename)
            # handling the case that previous file failed to parse
            errorfile = os.path.isfile(err_path)                                          # holds boolean value
            if errorfile:
                os.remove(err_path)
            else:
                fi_na = open(self.__texts_path + '/' + str(filepdf) +'.txt', 'w+')        # open text file for current file
                temp_file = open(self.__folder_path + '/classes/JSfromPDF.txt', 'r')
                # copy content from temp file to text file
                for line in temp_file.readlines():
                   try:     
                       fi_na.write(str(line))
                   except:
                       continue
                temp_file.close()
                fi_na.close()
            return True
        except Exception as ex:
            # print(ex)
            # always keep track the error until the code has been clean
            print("[!] Extract JavaScript to TXT " + filepdf)
            return False


    # function extract JS from pdf using peepdf
    def ex_js(self, filename):
        # run peepdf.py
        scriptpath = os.path.join(self.__folder_path + '/classes/peepdf/', 'peepdf.py')               # joins to path
        CommandOfExtract = self.__folder_path + '/classes/extractJS.txt'
        p = subprocess.Popen(['python', scriptpath, '-l', '-f', '-s', CommandOfExtract, filename])    # open subprocess and extract
        p.wait()


    # this function extract JavaScript from pdf file to txt
    def extract(self, dirpdf):
        # dir of folder and filter for pdf files
        files = [f for f in os.listdir(dirpdf) if os.path.isfile(os.path.join(dirpdf, f))]
        files = list(filter(lambda f: f.endswith(('.pdf', '.PDF')), files))

        # variables for print information
        cnt_files = len(files)
        i = 0
        err_path = os.path.join(self.__folder_path, 'classes/peepdf/errors.txt')
        for filepdf in files:
            i += 1
            try:
                filename = os.path.join(dirpdf, filepdf)
                self.ex_js(filename)
                # handling the case that previous file failed to parse
                errorfile = os.path.isfile(err_path)                                          # holds boolean value
                if errorfile:
                    os.remove(err_path)
                else:
                    fi_na = open(self.__texts_path + '/' + str(filepdf) +'.txt', 'w+')        # open text file for current file
                    temp_file = open(self.__folder_path + '/classes/JSfromPDF.txt', 'r')
                    # copy content from temp file to text file
                    try:
                        for line in temp_file.readlines():
                            fi_na.write(str(line))
                    except:
                        continue
                    temp_file.close()
                    fi_na.close()
                # show an update every 50 pdf files
                if (i > 0 and i % 50 == 0):
                    print("[INFO] processed {}/{}".format(i, cnt_files))
            except Exception as ex:
                #print(ex)
                # always keep track the error until the code has been clean
                print("[!] Extract JavaScript to TXT " + filepdf)
                return False
        print("[INFO] processed {}/{}".format(cnt_files, cnt_files))
        return True

    # this function return dictionary
    def getDict(self):
        return self.__ans_list

# ------------------------------------------------------------------------------------------------------------------------------------------------------------- # The our class for PDF files # @Authors:  <NAME> and <NAME> # @Version: 1.0 # ------------------------------------------------------------------------------------------------------------------------------------------------------------- # libraries # folder of train files # path to main folder # path to folder of images from first page # path to folder of JavaScript from pdf file # dictionary for translate PDF language to tessaract language # dictionary for /Root/Lang 1 - except; 2 - a file have not /Root/Lang; 3 # - /Root/Lang = ''; 4 - language # constructor # this function update ans_list #print(filename) #print(ex) # this function generate random page # this function convert pdf file to jpg file # dir of folder and filter for pdf files # variables for print information # save image # update ans_list # show an update every 50 images # always keep track the error until the code has been clean # this function convert pdf file to image # save image # update ans_list # print(ex) # always keep track the error until the code has been clean # this function extract JavaScript from pdf file to txt # handling the case that previous file failed to parse # holds boolean value # open text file for current file # copy content from temp file to text file # print(ex) # always keep track the error until the code has been clean # function extract JS from pdf using peepdf # run peepdf.py # joins to path # open subprocess and extract # this function extract JavaScript from pdf file to txt # dir of folder and filter for pdf files # variables for print information # handling the case that previous file failed to parse # holds boolean value # open text file for current file # copy content from temp file to text file # show an update every 50 pdf files #print(ex) # always keep track the error until the code has been clean # this function return dictionary