# AUTOGENERATED! DO NOT EDIT! File to edit: dev/01_imgtools.ipynb (unless otherwise specified).

__all__ = ['download_image', 'download_images', 'verify_image_tf', 'verify_images']

# Cell
from .core import *
from multiprocessing.pool import ThreadPool
from functools import partial
import pathlib
import urllib
import tqdm
import PIL
import warnings
import uuid
import requests

# Cell
def download_image(dest, url, timeout=5):
    """
    Downloads an image from url to destination file path (pathlib.Path object).
    Tries to detect and preserve original suffix and defaults to '.jpg' if none
    Assigns a randomly generated filename to avoid name clashes.
    """
    tmp = urllib.parse.urlparse(url)
    suffix = pathlib.Path(tmp.path).suffix
    if suffix == '':
        suffix = '.jpg'
    stem = str(uuid.uuid4())
    local_file = dest/f'{stem + suffix}'
    result = ''
    user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0'
    try:
        r = requests.get(url, stream=True, timeout=5, headers={"User-Agent": user_agent})
        if r.status_code == 200:
            with open(local_file, 'wb') as f:
                for chunk in r:
                    f.write(chunk)
            result = f"Success: {local_file}. Downloaded {url}"
        else:
            result = f"Failed: Can't download {url}"
    except Exception as e:
        result = f"Failed: Can't download {url}. Exception: {str(e)}"

    return result

# Cell
def download_images(url_file_path, dest, n_threads=4, timeout=5):
    """
    Downloads images from list of urls.
    Makes use of multiprocessing library to support concurrent downloads if supported by CPU.
    Visualizes progress using tqdm.
    """
    urls = pathlib.Path(url_file_path).read_text().strip().split("\n")
    dest = pathlib.Path(dest)
    dest.mkdir(exist_ok=True, parents=True)
    results = []
    for x in tqdm.tqdm_notebook(
        ThreadPool(n_threads).imap_unordered(
            partial(download_image, dest, timeout=timeout),
            urls), total=len(urls)):
        results.append(x)
    return results

# Cell
def verify_image_tf(img_file):
    """
    Verifies if an image can be opened by tf.image module
    tf.image currently supports only BMP, JPEG, GIF and PNG.
    Function attempts to convert non-supported formats to JPEG.
    If the image cannot be opened or converted, it is deleted.
    """
    try:
        import tensorflow as tf
        import PIL
        warnings.simplefilter("ignore")
        suffix = img_file.suffix
        if suffix.lower() not in ['.bmp', '.jpeg', '.jpg', '.gif', '.png']:
            renamed = img_file.parent/f'{img_file.name+".jpg"}'
            PIL.Image.open(img_file).save(renamed, "JPEG")
            img_file.unlink()
            print(f"File converted: {renamed}")
            img_file = renamed
        img = tf.io.read_file(str(img_file))
        img = tf.image.decode_image(img, channels=3, expand_animations=False)
    except Exception as e:
        print(f'Failed: Deleting {str(img_file)}: {e}')
        img_file.unlink()

# Cell
def verify_images(file_path, n_threads=4, recurse=False):
    """
    Verifies images from file path.
    Makes use of multiprocessing library to support concurrent verification if supported by CPU.
    Visualizes progress using tqdm.
    """
    path = pathlib.Path(file_path)
    files = get_all_files(file_path, recurse=recurse)
    for x in tqdm.tqdm_notebook(
        ThreadPool(n_threads).imap_unordered(
            verify_image_tf,
            files), total=len(files)):
        pass
# AUTOGENERATED! DO NOT EDIT! File to edit: dev/01_imgtools.ipynb (unless otherwise specified).

__all__ = ['download_image', 'download_images', 'verify_image_tf', 'verify_images']

# Cell
from .core import *
from multiprocessing.pool import ThreadPool
from functools import partial
import pathlib
import urllib
import tqdm
import PIL
import warnings
import uuid
import requests

# Cell
def download_image(dest, url, timeout=5):
    """
    Downloads an image from url to destination file path (pathlib.Path object).
    Tries to detect and preserve original suffix and defaults to '.jpg' if none
    Assigns a randomly generated filename to avoid name clashes.
    """
    tmp = urllib.parse.urlparse(url)
    suffix = pathlib.Path(tmp.path).suffix
    if suffix == '':
        suffix = '.jpg'
    stem = str(uuid.uuid4())
    local_file = dest/f'{stem + suffix}'
    result = ''
    user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:71.0) Gecko/20100101 Firefox/71.0'
    try:
        r = requests.get(url, stream=True, timeout=5, headers={"User-Agent": user_agent})
        if r.status_code == 200:
            with open(local_file, 'wb') as f:
                for chunk in r:
                    f.write(chunk)
            result = f"Success: {local_file}. Downloaded {url}"
        else:
            result = f"Failed: Can't download {url}"
    except Exception as e:
        result = f"Failed: Can't download {url}. Exception: {str(e)}"

    return result

# Cell
def download_images(url_file_path, dest, n_threads=4, timeout=5):
    """
    Downloads images from list of urls.
    Makes use of multiprocessing library to support concurrent downloads if supported by CPU.
    Visualizes progress using tqdm.
    """
    urls = pathlib.Path(url_file_path).read_text().strip().split("\n")
    dest = pathlib.Path(dest)
    dest.mkdir(exist_ok=True, parents=True)
    results = []
    for x in tqdm.tqdm_notebook(
        ThreadPool(n_threads).imap_unordered(
            partial(download_image, dest, timeout=timeout),
            urls), total=len(urls)):
        results.append(x)
    return results

# Cell
def verify_image_tf(img_file):
    """
    Verifies if an image can be opened by tf.image module
    tf.image currently supports only BMP, JPEG, GIF and PNG.
    Function attempts to convert non-supported formats to JPEG.
    If the image cannot be opened or converted, it is deleted.
    """
    try:
        import tensorflow as tf
        import PIL
        warnings.simplefilter("ignore")
        suffix = img_file.suffix
        if suffix.lower() not in ['.bmp', '.jpeg', '.jpg', '.gif', '.png']:
            renamed = img_file.parent/f'{img_file.name+".jpg"}'
            PIL.Image.open(img_file).save(renamed, "JPEG")
            img_file.unlink()
            print(f"File converted: {renamed}")
            img_file = renamed
        img = tf.io.read_file(str(img_file))
        img = tf.image.decode_image(img, channels=3, expand_animations=False)
    except Exception as e:
        print(f'Failed: Deleting {str(img_file)}: {e}')
        img_file.unlink()

# Cell
def verify_images(file_path, n_threads=4, recurse=False):
    """
    Verifies images from file path.
    Makes use of multiprocessing library to support concurrent verification if supported by CPU.
    Visualizes progress using tqdm.
    """
    path = pathlib.Path(file_path)
    files = get_all_files(file_path, recurse=recurse)
    for x in tqdm.tqdm_notebook(
        ThreadPool(n_threads).imap_unordered(
            verify_image_tf,
            files), total=len(files)):
        pass
# AUTOGENERATED! DO NOT EDIT! File to edit: dev/01_imgtools.ipynb (unless otherwise specified). # Cell # Cell Downloads an image from url to destination file path (pathlib.Path object).     Tries to detect and preserve original suffix and defaults to '.jpg' if none     Assigns a randomly generated filename to avoid name clashes. # Cell Downloads images from list of urls.     Makes use of multiprocessing library to support concurrent downloads if supported by CPU.     Visualizes progress using tqdm. # Cell Verifies if an image can be opened by tf.image module     tf.image currently supports only BMP, JPEG, GIF and PNG.     Function attempts to convert non-supported formats to JPEG.     If the image cannot be opened or converted, it is deleted. # Cell Verifies images from file path.     Makes use of multiprocessing library to support concurrent verification if supported by CPU.     Visualizes progress using tqdm.