Spaces:
Running
on
Zero
Running
on
Zero
File size: 1,877 Bytes
4893ce0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 |
#!/bin/sh
# Variables
SOURCE_DIR=$1
DEST_DIR=$2
MAX_SIZE=$(awk "BEGIN {printf \"%d\", $3 * 1024 * 1024}") # Convert GB to KB as an integer
# Get the base name of the source directory to use as TAR_NAME
TAR_NAME=$(basename "$SOURCE_DIR")
# Create destination directory if it doesn't exist
mkdir -p "$DEST_DIR"
# Function to create a new tar file
create_tar() {
tar_number=$1
file_list=$2
tar_name=$(printf "%s/${TAR_NAME}_%0${width}d.tar.gz" "$DEST_DIR" "$tar_number")
tar -zcvf "$tar_name" -C "$SOURCE_DIR" -T "$file_list"
}
# Initialize
tar_number=1
current_size=0
temp_dir=$(mktemp -d)
file_list="$temp_dir/file_list_$tar_number"
echo Start indexing "file_list_$tar_number"
cd "$SOURCE_DIR" || exit 1
# Iterate over all files in the source directory
find . -type f | while IFS= read -r file; do
file_size=$(du -k "$file" | cut -f1)
if [ $(( current_size + file_size )) -gt $MAX_SIZE ]; then
tar_number=$((tar_number + 1))
file_list="$temp_dir/file_list_$tar_number"
echo Start indexing "file_list_$tar_number"
current_size=0
fi
echo "$file" >> "$file_list"
current_size=$((current_size + file_size))
done
# Determine the width for the tar file numbers
total_files=$(find "$temp_dir" -name 'file_list_*' | wc -l)
width=${#total_files}
# Set PARALLEL_PROCESSES to the number of file lists if not provided
PARALLEL_PROCESSES=${4:-$total_files}
# Debug information
echo "Total files: $total_files"
echo "Width: $width"
echo "Parallel processes: $PARALLEL_PROCESSES"
# Run tar creation in parallel
find "$temp_dir" -name 'file_list_*' | xargs -P "$PARALLEL_PROCESSES" -I {} sh -c '
file_list={}
tar_number=$(basename "$file_list" | cut -d_ -f3)
tar_name=$(printf "%s/'"$TAR_NAME"'_%0'"$width"'d.tar.gz" "'"$DEST_DIR"'" "$tar_number")
tar -zcvf "$tar_name" -C "'"$SOURCE_DIR"'" -T "$file_list"
'
# Clean up
rm -rf "$temp_dir" |