Upload 7 files

README.md

@@ -4,7 +4,7 @@ emoji: 🤗
 colorFrom: yellow
 colorTo: blue
 sdk: gradio
-sdk_version: 5.12.0
+sdk_version: 5.17.1
 app_file: app.py
 pinned: false
 license: mit

civitai_to_hf.py

@@ -1,11 +1,13 @@
 import gradio as gr
 from huggingface_hub import HfApi, hf_hub_url
+from huggingface_hub.hf_api import RepoFile
 import os
 from pathlib import Path
 import gc
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util import Retry
+import urllib
 from utils import (get_token, set_token, is_repo_exists, get_user_agent, get_download_file,
                    list_uniq, list_sub, duplicate_hf_repo, HF_SUBFOLDER_NAME, get_state, set_state)
 import re
@@ -65,6 +67,55 @@ def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, progres
     return url
 
 
+def get_repo_hashes(repo_id: str, repo_type: str="model"):
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
+    hashes = []
+    try:
+        if not api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token): return hashes
+        tree = api.list_repo_tree(repo_id=repo_id, repo_type=repo_type, token=hf_token)
+        for f in tree:
+            if not isinstance(f, RepoFile) or f.lfs is None: continue
+            hashes.append(f.lfs["sha256"])
+    except Exception as e:
+        print(e)
+    finally:
+        return hashes
+
+
+def get_civitai_sha256(dl_url: str, api_key=""):
+    def is_invalid_file(qs: dict, json: dict, k: str):
+        return k in qs.keys() and qs[k][0] != json.get(k, None) and json.get(k, None) is not None
+
+    if "https://civitai.com/api/download/models/" not in dl_url: return None
+    user_agent = get_user_agent()
+    headers = {'User-Agent': user_agent, 'content-type': 'application/json'}
+    if api_key: headers['Authorization'] = f'Bearer {{{api_key}}}'
+    base_url = 'https://civitai.com/api/v1/model-versions/'
+    params = {}
+    session = requests.Session()
+    retries = Retry(total=5, backoff_factor=1, status_forcelist=[500, 502, 503, 504])
+    session.mount("https://", HTTPAdapter(max_retries=retries))
+    m = re.match(r'https://civitai.com/api/download/models/(\d+)\??(.+)?', dl_url)
+    if m is None: return None
+    url = base_url + m.group(1)
+    qs = urllib.parse.parse_qs(m.group(2))
+    if "type" not in qs.keys(): qs["type"] = ["Model"]
+    try:
+        r = session.get(url, params=params, headers=headers, stream=True, timeout=(5.0, 15))
+        if not r.ok: return None
+        json = dict(r.json())
+        if "files" not in json.keys() or not isinstance(json["files"], list): return None
+        hash = None
+        for d in json["files"]:
+            if is_invalid_file(qs, d, "type") or is_invalid_file(qs, d, "format") or is_invalid_file(qs, d, "size") or is_invalid_file(qs, d, "fp"): continue
+            hash = d["hashes"]["SHA256"].lower()
+        return hash
+    except Exception as e:
+        print(e)
+        return None
+
+
 def is_same_file(filename: str, cmp_sha256: str, cmp_size: int):
     if cmp_sha256:
         sha256_hash = hashlib.sha256()
@@ -152,16 +203,22 @@ def upload_info_to_repo(dl_url, filename, repo_id, repo_type, is_private, civita
 def download_civitai(dl_url, civitai_key, hf_token, urls, 
                      newrepo_id, repo_type="model", is_private=True, is_info=False, is_rename=True, progress=gr.Progress(track_tqdm=True)):
     if hf_token: set_token(hf_token)
-    else: set_token(os.environ.get("HF_TOKEN")) # default huggingface write token
+    else: set_token(os.getenv("HF_TOKEN", False)) # default huggingface write token
     if not civitai_key: civitai_key = os.environ.get("CIVITAI_API_KEY") # default Civitai API key
     if not newrepo_id: newrepo_id = os.environ.get("HF_REPO") # default repo to upload
     if not get_token() or not civitai_key: raise gr.Error("HF write token and Civitai API key is required.")
     if not urls: urls = []
     dl_urls = parse_urls(dl_url)
     remain_urls = dl_urls.copy()
+    hashes = set(get_repo_hashes(newrepo_id, repo_type))
     try:
         md = f'### Your repo: [{newrepo_id}]({"https://huggingface.co/datasets/" if repo_type == "dataset" else "https://huggingface.co/"}{newrepo_id})\n'
         for u in dl_urls:
+            if get_civitai_sha256(u, civitai_key) in hashes:
+                print(f"{u} is already exitsts. skipping.")
+                remain_urls.remove(u)
+                md += f"- Skipped [{str(u)}]({str(u)})\n"
+                continue
             file = download_file(u, civitai_key)
             if not Path(file).exists() or not Path(file).is_file(): continue
             if is_rename: file = get_safe_filename(file, newrepo_id, repo_type)

utils.py

@@ -15,7 +15,7 @@ def get_token():
     try:
         token = HfFolder.get_token()
     except Exception:
-        token = ""
+        token = False
     return token
 
 
@@ -170,6 +170,10 @@ def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqd
         elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
             print(f"File to download alreday exists: {url}")
             new_file = f"{temp_dir}/{url.split('/')[-1]}"
+        elif "huggingface.co" in url:
+            url = url.replace("?download=true", "")
+            if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
+            new_file = download_hf_file(temp_dir, url)
         else:
             print(f"Start downloading: {url}")
             recursive = False if "huggingface.co" in url else True