#!/bin/bash | |
GPU_ID=5 | |
THRESHOLD=50000 | |
function get_free_memory() { | |
nvidia-smi --query-gpu=memory.free --format=csv,nounits,noheader | sed -n "$((GPU_ID+1))p" | |
} | |
while true; do | |
FREE_MEM=$(get_free_memory) | |
echo "GPU $GPU_ID free memory: ${FREE_MEM}MB" | |
if [ "$FREE_MEM" -gt "$THRESHOLD" ]; then | |
echo "Free memory is greater than 40GB. Running your script...." | |
# nohup bash train_demo_3456.sh > nohup_all_coco_v2_phi35_lre4_e1_0913.out & | |
nohup python target_all.py --size 400 --gpus 2 --interval 0.01 & | |
break | |
fi | |
# 设定检查间隔,例如每隔5分钟检查一次 | |
sleep 10 | |
done | |