File size: 609 Bytes
6123c5a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 |
#!/bin/bash
GPU_ID=5
THRESHOLD=50000
function get_free_memory() {
nvidia-smi --query-gpu=memory.free --format=csv,nounits,noheader | sed -n "$((GPU_ID+1))p"
}
while true; do
FREE_MEM=$(get_free_memory)
echo "GPU $GPU_ID free memory: ${FREE_MEM}MB"
if [ "$FREE_MEM" -gt "$THRESHOLD" ]; then
echo "Free memory is greater than 40GB. Running your script...."
# nohup bash train_demo_3456.sh > nohup_all_coco_v2_phi35_lre4_e1_0913.out &
nohup python target_all.py --size 400 --gpus 2 --interval 0.01 &
break
fi
# 设定检查间隔，例如每隔5分钟检查一次
sleep 10
done
|