Upload 4 files
Browse files- Dockerfile +6 -0
- README.md +62 -6
- data-sync.sh +139 -0
- gitattributes +35 -0
Dockerfile
ADDED
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
FROM ghcr.io/open-webui/open-webui:dev
|
2 |
+
|
3 |
+
COPY data-sync.sh data-sync.sh
|
4 |
+
|
5 |
+
RUN chmod -R 777 ./data && \
|
6 |
+
sed -i "1r data-sync.sh" ./start.sh
|
README.md
CHANGED
@@ -1,10 +1,66 @@
|
|
1 |
---
|
2 |
-
title:
|
3 |
-
emoji:
|
4 |
-
colorFrom:
|
5 |
-
colorTo:
|
6 |
sdk: docker
|
7 |
-
pinned:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
8 |
---
|
9 |
|
10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
---
|
2 |
+
title: OpenWebUI with Data Sync
|
3 |
+
emoji: 🔄
|
4 |
+
colorFrom: blue
|
5 |
+
colorTo: purple
|
6 |
sdk: docker
|
7 |
+
pinned: true
|
8 |
+
app_port: 8080
|
9 |
+
suggested_hardware: cpu-basic
|
10 |
+
suggested_storage: medium
|
11 |
+
short_description: OpenWebUI with automated GitHub data sync
|
12 |
+
tags:
|
13 |
+
- openwebui
|
14 |
+
- chat
|
15 |
+
- llm
|
16 |
+
- ai
|
17 |
+
- docker
|
18 |
+
- data-sync
|
19 |
+
- backup
|
20 |
+
models:
|
21 |
+
- openai-community/gpt2
|
22 |
+
datasets:
|
23 |
+
- openwebui/openwebui
|
24 |
---
|
25 |
|
26 |
+
# OpenWebUI with Data Synchronization
|
27 |
+
|
28 |
+
A self-hosted OpenWebUI instance with automated data backup and synchronization to GitHub repositories.
|
29 |
+
|
30 |
+
## Features
|
31 |
+
|
32 |
+
- 🤖 **OpenWebUI Interface**: Full-featured chat interface for AI models
|
33 |
+
- 🔄 **Automated Data Sync**: Automatic backup to GitHub repository every hour
|
34 |
+
- 🔒 **Private Storage**: Secure data storage in your private GitHub repo
|
35 |
+
- 📊 **Persistent Data**: Never lose your conversations and settings
|
36 |
+
- 🐳 **Docker Ready**: Easy deployment on Hugging Face Spaces
|
37 |
+
|
38 |
+
## Environment Variables
|
39 |
+
|
40 |
+
Set these in your Space's Repository secrets:
|
41 |
+
|
42 |
+
### Required
|
43 |
+
- `G_NAME`: Your GitHub repository (e.g., `username/repo-name`)
|
44 |
+
- `G_TOKEN`: GitHub Personal Access Token with repo access
|
45 |
+
|
46 |
+
### Optional
|
47 |
+
- `WEBDAV_URL`: WebDAV server URL for additional backup
|
48 |
+
- `WEBDAV_USERNAME`: WebDAV username
|
49 |
+
- `WEBDAV_PASSWORD`: WebDAV password
|
50 |
+
|
51 |
+
## Setup Instructions
|
52 |
+
|
53 |
+
1. Create a private GitHub repository for data backup
|
54 |
+
2. Generate a GitHub Personal Access Token with repo permissions
|
55 |
+
3. Add the environment variables to your Space's Repository secrets
|
56 |
+
4. Deploy and enjoy persistent OpenWebUI!
|
57 |
+
|
58 |
+
## Data Synchronization
|
59 |
+
|
60 |
+
The system automatically:
|
61 |
+
- Syncs every hour (08:00-24:00 Beijing time)
|
62 |
+
- Backs up `webui.db` to your GitHub repository
|
63 |
+
- Creates versioned commits with timestamps
|
64 |
+
- Handles conflicts and retries on failures
|
65 |
+
|
66 |
+
Check out the [configuration reference](https://huggingface.co/docs/hub/spaces-config-reference) for more details.
|
data-sync.sh
ADDED
@@ -0,0 +1,139 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/bin/bash
|
2 |
+
# Note: Ensure that date, git, curl and other tools are installed in the system when the script is executed, and set the TZ timezone environment (can be temporarily specified in each date command)
|
3 |
+
|
4 |
+
# Check necessary environment variables
|
5 |
+
if [ -z "$G_NAME" ] || [ -z "$G_TOKEN" ]; then
|
6 |
+
echo "Missing required environment variables G_NAME or G_TOKEN"
|
7 |
+
exit 1
|
8 |
+
fi
|
9 |
+
|
10 |
+
# Parse repository name and username
|
11 |
+
IFS='/' read -r GITHUB_USER GITHUB_REPO <<< "$G_NAME"
|
12 |
+
|
13 |
+
# Build GitHub repository clone URL with token
|
14 |
+
REPO_URL="https://${G_TOKEN}@github.com/${G_NAME}.git"
|
15 |
+
mkdir -p ./data/github_data
|
16 |
+
|
17 |
+
# Clone repository
|
18 |
+
echo "Cloning repository..."
|
19 |
+
git clone "$REPO_URL" ./data/github_data || {
|
20 |
+
echo "Clone failed, please check if G_NAME and G_TOKEN are correct."
|
21 |
+
exit 1
|
22 |
+
}
|
23 |
+
|
24 |
+
if [ -f ./data/github_data/webui.db ]; then
|
25 |
+
cp ./data/github_data/webui.db ./data/webui.db
|
26 |
+
echo "Successfully pulled from GitHub repository"
|
27 |
+
else
|
28 |
+
echo "webui.db not found in GitHub repository, will push during sync"
|
29 |
+
fi
|
30 |
+
|
31 |
+
# Define sync function, according to Beijing time 08:00~24:00 (including hourly sync) requirements
|
32 |
+
sync_data() {
|
33 |
+
while true; do
|
34 |
+
# Use Asia/Shanghai timezone to get current time and its components
|
35 |
+
CURRENT_TS=$(TZ=Asia/Shanghai date +%s)
|
36 |
+
CURRENT_DATE=$(TZ=Asia/Shanghai date '+%Y-%m-%d')
|
37 |
+
CURRENT_HOUR=$(TZ=Asia/Shanghai date +%H) # 00~23
|
38 |
+
CURRENT_MIN=$(TZ=Asia/Shanghai date +%M)
|
39 |
+
CURRENT_SEC=$(TZ=Asia/Shanghai date +%S)
|
40 |
+
|
41 |
+
# Calculate next sync target timestamp (Beijing time)
|
42 |
+
# If current time is before 08:00, target is today 08:00
|
43 |
+
if [ "$CURRENT_HOUR" -lt 8 ]; then
|
44 |
+
TARGET_TS=$(TZ=Asia/Shanghai date -d "${CURRENT_DATE} 08:00:00" +%s)
|
45 |
+
# If between 08:00 to 22:59, next hour is on the same day
|
46 |
+
elif [ "$CURRENT_HOUR" -ge 8 ] && [ "$CURRENT_HOUR" -lt 23 ]; then
|
47 |
+
# If exactly at the hour (seconds and minutes are 0) then sync immediately
|
48 |
+
if [ "$CURRENT_MIN" -eq 0 ] && [ "$CURRENT_SEC" -eq 0 ]; then
|
49 |
+
TARGET_TS=$CURRENT_TS
|
50 |
+
else
|
51 |
+
NEXT_HOUR=$((10#$CURRENT_HOUR + 1))
|
52 |
+
TARGET_TS=$(TZ=Asia/Shanghai date -d "${CURRENT_DATE} ${NEXT_HOUR}:00:00" +%s)
|
53 |
+
fi
|
54 |
+
# If current time is between 23:00~23:59, next target is next day 00:00 (which is 24:00 sync)
|
55 |
+
else # CURRENT_HOUR == 23
|
56 |
+
if [ "$CURRENT_MIN" -eq 0 ] && [ "$CURRENT_SEC" -eq 0 ]; then
|
57 |
+
TARGET_TS=$CURRENT_TS
|
58 |
+
else
|
59 |
+
TOMORROW=$(TZ=Asia/Shanghai date -d "tomorrow" '+%Y-%m-%d')
|
60 |
+
TARGET_TS=$(TZ=Asia/Shanghai date -d "${TOMORROW} 00:00:00" +%s)
|
61 |
+
fi
|
62 |
+
fi
|
63 |
+
|
64 |
+
# Calculate wait time (if exactly at sync time then sleep_time is 0)
|
65 |
+
SLEEP_TIME=$(( TARGET_TS - CURRENT_TS ))
|
66 |
+
if [ "$SLEEP_TIME" -gt 0 ]; then
|
67 |
+
echo "Time until next sync: ${SLEEP_TIME} seconds (Beijing time next sync: $(TZ=Asia/Shanghai date -d "@$TARGET_TS" '+%Y-%m-%d %H:%M:%S'))"
|
68 |
+
sleep "$SLEEP_TIME"
|
69 |
+
fi
|
70 |
+
|
71 |
+
# Output current Beijing time during sync
|
72 |
+
CURRENT_TIME=$(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')
|
73 |
+
echo "Current time $CURRENT_TIME"
|
74 |
+
|
75 |
+
# ---- Start sync process ----
|
76 |
+
|
77 |
+
# 1. Sync to GitHub
|
78 |
+
echo "Starting GitHub sync..."
|
79 |
+
cd ./data/github_data || { echo "Failed to change directory"; exit 1; }
|
80 |
+
git config user.name "AutoSync Bot"
|
81 |
+
git config user.email "[email protected]"
|
82 |
+
|
83 |
+
# Ensure on main branch, if switch fails try master branch
|
84 |
+
git checkout main 2>/dev/null || git checkout master
|
85 |
+
|
86 |
+
# Copy latest database file to repository directory
|
87 |
+
if [ -f "../webui.db" ]; then
|
88 |
+
cp ../webui.db ./webui.db
|
89 |
+
else
|
90 |
+
echo "Database not yet initialized"
|
91 |
+
fi
|
92 |
+
|
93 |
+
# Check if there are changes
|
94 |
+
if [[ -n $(git status -s) ]]; then
|
95 |
+
git add webui.db
|
96 |
+
git commit -m "Auto sync webui.db $(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')"
|
97 |
+
git push origin HEAD && {
|
98 |
+
echo "GitHub push successful"
|
99 |
+
} || {
|
100 |
+
echo "Push failed, waiting for retry..."
|
101 |
+
sleep 10
|
102 |
+
git push origin HEAD || {
|
103 |
+
echo "Retry failed, abandoning GitHub push."
|
104 |
+
}
|
105 |
+
}
|
106 |
+
else
|
107 |
+
echo "GitHub: No database changes detected"
|
108 |
+
fi
|
109 |
+
# Return to main directory
|
110 |
+
cd ../..
|
111 |
+
|
112 |
+
# 2. Sync to WebDAV (if environment variables are complete)
|
113 |
+
if [ -z "$WEBDAV_URL" ] || [ -z "$WEBDAV_USERNAME" ] || [ -z "$WEBDAV_PASSWORD" ]; then
|
114 |
+
echo "WebDAV environment variables missing, skipping WebDAV sync."
|
115 |
+
else
|
116 |
+
echo "Starting WebDAV sync..."
|
117 |
+
FILENAME="webui_$(TZ=Asia/Shanghai date +'%m_%d').db"
|
118 |
+
if [ -f ./data/webui.db ]; then
|
119 |
+
curl -T ./data/webui.db --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$WEBDAV_URL/$FILENAME" && {
|
120 |
+
echo "WebDAV upload successful"
|
121 |
+
} || {
|
122 |
+
echo "WebDAV upload failed, waiting for retry..."
|
123 |
+
sleep 10
|
124 |
+
curl -T ./data/webui.db --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$WEBDAV_URL/$FILENAME" || {
|
125 |
+
echo "Retry failed, abandoning WebDAV upload."
|
126 |
+
}
|
127 |
+
}
|
128 |
+
else
|
129 |
+
echo "webui.db file not found, skipping WebDAV sync."
|
130 |
+
fi
|
131 |
+
fi
|
132 |
+
|
133 |
+
# ---- Sync process complete, next loop will automatically calculate wait time based on current Beijing time ----
|
134 |
+
|
135 |
+
done
|
136 |
+
}
|
137 |
+
|
138 |
+
# Start sync process in background
|
139 |
+
sync_data &
|
gitattributes
ADDED
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
11 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
12 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
13 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
14 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
15 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
16 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
17 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
18 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
19 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
20 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
21 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
22 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
23 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
24 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
25 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
26 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
27 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
28 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
29 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
30 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
31 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
32 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
33 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
34 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
35 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|