owlninjam commited on
Commit
7c1a9d1
·
verified ·
1 Parent(s): 0d85f15

Upload 4 files

Browse files
Files changed (4) hide show
  1. Dockerfile +6 -0
  2. README.md +62 -6
  3. data-sync.sh +139 -0
  4. gitattributes +35 -0
Dockerfile ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ FROM ghcr.io/open-webui/open-webui:dev
2
+
3
+ COPY data-sync.sh data-sync.sh
4
+
5
+ RUN chmod -R 777 ./data && \
6
+ sed -i "1r data-sync.sh" ./start.sh
README.md CHANGED
@@ -1,10 +1,66 @@
1
  ---
2
- title: Openwebui
3
- emoji: 🏢
4
- colorFrom: red
5
- colorTo: red
6
  sdk: docker
7
- pinned: false
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: OpenWebUI with Data Sync
3
+ emoji: 🔄
4
+ colorFrom: blue
5
+ colorTo: purple
6
  sdk: docker
7
+ pinned: true
8
+ app_port: 8080
9
+ suggested_hardware: cpu-basic
10
+ suggested_storage: medium
11
+ short_description: OpenWebUI with automated GitHub data sync
12
+ tags:
13
+ - openwebui
14
+ - chat
15
+ - llm
16
+ - ai
17
+ - docker
18
+ - data-sync
19
+ - backup
20
+ models:
21
+ - openai-community/gpt2
22
+ datasets:
23
+ - openwebui/openwebui
24
  ---
25
 
26
+ # OpenWebUI with Data Synchronization
27
+
28
+ A self-hosted OpenWebUI instance with automated data backup and synchronization to GitHub repositories.
29
+
30
+ ## Features
31
+
32
+ - 🤖 **OpenWebUI Interface**: Full-featured chat interface for AI models
33
+ - 🔄 **Automated Data Sync**: Automatic backup to GitHub repository every hour
34
+ - 🔒 **Private Storage**: Secure data storage in your private GitHub repo
35
+ - 📊 **Persistent Data**: Never lose your conversations and settings
36
+ - 🐳 **Docker Ready**: Easy deployment on Hugging Face Spaces
37
+
38
+ ## Environment Variables
39
+
40
+ Set these in your Space's Repository secrets:
41
+
42
+ ### Required
43
+ - `G_NAME`: Your GitHub repository (e.g., `username/repo-name`)
44
+ - `G_TOKEN`: GitHub Personal Access Token with repo access
45
+
46
+ ### Optional
47
+ - `WEBDAV_URL`: WebDAV server URL for additional backup
48
+ - `WEBDAV_USERNAME`: WebDAV username
49
+ - `WEBDAV_PASSWORD`: WebDAV password
50
+
51
+ ## Setup Instructions
52
+
53
+ 1. Create a private GitHub repository for data backup
54
+ 2. Generate a GitHub Personal Access Token with repo permissions
55
+ 3. Add the environment variables to your Space's Repository secrets
56
+ 4. Deploy and enjoy persistent OpenWebUI!
57
+
58
+ ## Data Synchronization
59
+
60
+ The system automatically:
61
+ - Syncs every hour (08:00-24:00 Beijing time)
62
+ - Backs up `webui.db` to your GitHub repository
63
+ - Creates versioned commits with timestamps
64
+ - Handles conflicts and retries on failures
65
+
66
+ Check out the [configuration reference](https://huggingface.co/docs/hub/spaces-config-reference) for more details.
data-sync.sh ADDED
@@ -0,0 +1,139 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+ # Note: Ensure that date, git, curl and other tools are installed in the system when the script is executed, and set the TZ timezone environment (can be temporarily specified in each date command)
3
+
4
+ # Check necessary environment variables
5
+ if [ -z "$G_NAME" ] || [ -z "$G_TOKEN" ]; then
6
+ echo "Missing required environment variables G_NAME or G_TOKEN"
7
+ exit 1
8
+ fi
9
+
10
+ # Parse repository name and username
11
+ IFS='/' read -r GITHUB_USER GITHUB_REPO <<< "$G_NAME"
12
+
13
+ # Build GitHub repository clone URL with token
14
+ REPO_URL="https://${G_TOKEN}@github.com/${G_NAME}.git"
15
+ mkdir -p ./data/github_data
16
+
17
+ # Clone repository
18
+ echo "Cloning repository..."
19
+ git clone "$REPO_URL" ./data/github_data || {
20
+ echo "Clone failed, please check if G_NAME and G_TOKEN are correct."
21
+ exit 1
22
+ }
23
+
24
+ if [ -f ./data/github_data/webui.db ]; then
25
+ cp ./data/github_data/webui.db ./data/webui.db
26
+ echo "Successfully pulled from GitHub repository"
27
+ else
28
+ echo "webui.db not found in GitHub repository, will push during sync"
29
+ fi
30
+
31
+ # Define sync function, according to Beijing time 08:00~24:00 (including hourly sync) requirements
32
+ sync_data() {
33
+ while true; do
34
+ # Use Asia/Shanghai timezone to get current time and its components
35
+ CURRENT_TS=$(TZ=Asia/Shanghai date +%s)
36
+ CURRENT_DATE=$(TZ=Asia/Shanghai date '+%Y-%m-%d')
37
+ CURRENT_HOUR=$(TZ=Asia/Shanghai date +%H) # 00~23
38
+ CURRENT_MIN=$(TZ=Asia/Shanghai date +%M)
39
+ CURRENT_SEC=$(TZ=Asia/Shanghai date +%S)
40
+
41
+ # Calculate next sync target timestamp (Beijing time)
42
+ # If current time is before 08:00, target is today 08:00
43
+ if [ "$CURRENT_HOUR" -lt 8 ]; then
44
+ TARGET_TS=$(TZ=Asia/Shanghai date -d "${CURRENT_DATE} 08:00:00" +%s)
45
+ # If between 08:00 to 22:59, next hour is on the same day
46
+ elif [ "$CURRENT_HOUR" -ge 8 ] && [ "$CURRENT_HOUR" -lt 23 ]; then
47
+ # If exactly at the hour (seconds and minutes are 0) then sync immediately
48
+ if [ "$CURRENT_MIN" -eq 0 ] && [ "$CURRENT_SEC" -eq 0 ]; then
49
+ TARGET_TS=$CURRENT_TS
50
+ else
51
+ NEXT_HOUR=$((10#$CURRENT_HOUR + 1))
52
+ TARGET_TS=$(TZ=Asia/Shanghai date -d "${CURRENT_DATE} ${NEXT_HOUR}:00:00" +%s)
53
+ fi
54
+ # If current time is between 23:00~23:59, next target is next day 00:00 (which is 24:00 sync)
55
+ else # CURRENT_HOUR == 23
56
+ if [ "$CURRENT_MIN" -eq 0 ] && [ "$CURRENT_SEC" -eq 0 ]; then
57
+ TARGET_TS=$CURRENT_TS
58
+ else
59
+ TOMORROW=$(TZ=Asia/Shanghai date -d "tomorrow" '+%Y-%m-%d')
60
+ TARGET_TS=$(TZ=Asia/Shanghai date -d "${TOMORROW} 00:00:00" +%s)
61
+ fi
62
+ fi
63
+
64
+ # Calculate wait time (if exactly at sync time then sleep_time is 0)
65
+ SLEEP_TIME=$(( TARGET_TS - CURRENT_TS ))
66
+ if [ "$SLEEP_TIME" -gt 0 ]; then
67
+ echo "Time until next sync: ${SLEEP_TIME} seconds (Beijing time next sync: $(TZ=Asia/Shanghai date -d "@$TARGET_TS" '+%Y-%m-%d %H:%M:%S'))"
68
+ sleep "$SLEEP_TIME"
69
+ fi
70
+
71
+ # Output current Beijing time during sync
72
+ CURRENT_TIME=$(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')
73
+ echo "Current time $CURRENT_TIME"
74
+
75
+ # ---- Start sync process ----
76
+
77
+ # 1. Sync to GitHub
78
+ echo "Starting GitHub sync..."
79
+ cd ./data/github_data || { echo "Failed to change directory"; exit 1; }
80
+ git config user.name "AutoSync Bot"
81
+ git config user.email "[email protected]"
82
+
83
+ # Ensure on main branch, if switch fails try master branch
84
+ git checkout main 2>/dev/null || git checkout master
85
+
86
+ # Copy latest database file to repository directory
87
+ if [ -f "../webui.db" ]; then
88
+ cp ../webui.db ./webui.db
89
+ else
90
+ echo "Database not yet initialized"
91
+ fi
92
+
93
+ # Check if there are changes
94
+ if [[ -n $(git status -s) ]]; then
95
+ git add webui.db
96
+ git commit -m "Auto sync webui.db $(TZ=Asia/Shanghai date '+%Y-%m-%d %H:%M:%S')"
97
+ git push origin HEAD && {
98
+ echo "GitHub push successful"
99
+ } || {
100
+ echo "Push failed, waiting for retry..."
101
+ sleep 10
102
+ git push origin HEAD || {
103
+ echo "Retry failed, abandoning GitHub push."
104
+ }
105
+ }
106
+ else
107
+ echo "GitHub: No database changes detected"
108
+ fi
109
+ # Return to main directory
110
+ cd ../..
111
+
112
+ # 2. Sync to WebDAV (if environment variables are complete)
113
+ if [ -z "$WEBDAV_URL" ] || [ -z "$WEBDAV_USERNAME" ] || [ -z "$WEBDAV_PASSWORD" ]; then
114
+ echo "WebDAV environment variables missing, skipping WebDAV sync."
115
+ else
116
+ echo "Starting WebDAV sync..."
117
+ FILENAME="webui_$(TZ=Asia/Shanghai date +'%m_%d').db"
118
+ if [ -f ./data/webui.db ]; then
119
+ curl -T ./data/webui.db --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$WEBDAV_URL/$FILENAME" && {
120
+ echo "WebDAV upload successful"
121
+ } || {
122
+ echo "WebDAV upload failed, waiting for retry..."
123
+ sleep 10
124
+ curl -T ./data/webui.db --user "$WEBDAV_USERNAME:$WEBDAV_PASSWORD" "$WEBDAV_URL/$FILENAME" || {
125
+ echo "Retry failed, abandoning WebDAV upload."
126
+ }
127
+ }
128
+ else
129
+ echo "webui.db file not found, skipping WebDAV sync."
130
+ fi
131
+ fi
132
+
133
+ # ---- Sync process complete, next loop will automatically calculate wait time based on current Beijing time ----
134
+
135
+ done
136
+ }
137
+
138
+ # Start sync process in background
139
+ sync_data &
gitattributes ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
12
+ *.model filter=lfs diff=lfs merge=lfs -text
13
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
14
+ *.npy filter=lfs diff=lfs merge=lfs -text
15
+ *.npz filter=lfs diff=lfs merge=lfs -text
16
+ *.onnx filter=lfs diff=lfs merge=lfs -text
17
+ *.ot filter=lfs diff=lfs merge=lfs -text
18
+ *.parquet filter=lfs diff=lfs merge=lfs -text
19
+ *.pb filter=lfs diff=lfs merge=lfs -text
20
+ *.pickle filter=lfs diff=lfs merge=lfs -text
21
+ *.pkl filter=lfs diff=lfs merge=lfs -text
22
+ *.pt filter=lfs diff=lfs merge=lfs -text
23
+ *.pth filter=lfs diff=lfs merge=lfs -text
24
+ *.rar filter=lfs diff=lfs merge=lfs -text
25
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
26
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
27
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
28
+ *.tar filter=lfs diff=lfs merge=lfs -text
29
+ *.tflite filter=lfs diff=lfs merge=lfs -text
30
+ *.tgz filter=lfs diff=lfs merge=lfs -text
31
+ *.wasm filter=lfs diff=lfs merge=lfs -text
32
+ *.xz filter=lfs diff=lfs merge=lfs -text
33
+ *.zip filter=lfs diff=lfs merge=lfs -text
34
+ *.zst filter=lfs diff=lfs merge=lfs -text
35
+ *tfevents* filter=lfs diff=lfs merge=lfs -text