USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 8202 JOB_NAME = FS-LOW QUEUE = wp10 NSLOTS = 28 NHOSTS = 5 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW EXE_HOST = n161 NFS_HOST = caip02s EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/8202.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 23202 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/8201jtomczak: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/ancont: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Tue Jun 21 11:56:11 EDT 2011 on host n161 in /mnt/p06/8202jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile ']' + machines=/tmp/8202.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile + cat /opt/sge6/spool/n161/active_jobs/8202.1/pe_hostfile + read line ++ echo n161 3 wp10@n161 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n161 ++ echo n161 3 wp10@n161 UNDEFINED ++ cut -f2 '-d ' + nslots=3 + i=1 + '[' 1 -le 3 ']' + echo n161 ++ expr 1 + 1 + i=2 + '[' 2 -le 3 ']' + echo n161 ++ expr 2 + 1 + i=3 + '[' 3 -le 3 ']' + echo n161 ++ expr 3 + 1 + i=4 + '[' 4 -le 3 ']' + read line ++ echo n151 9 wp10@n151 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n151 ++ echo n151 9 wp10@n151 UNDEFINED ++ cut -f2 '-d ' + nslots=9 + i=1 + '[' 1 -le 9 ']' + echo n151 ++ expr 1 + 1 + i=2 + '[' 2 -le 9 ']' + echo n151 ++ expr 2 + 1 + i=3 + '[' 3 -le 9 ']' + echo n151 ++ expr 3 + 1 + i=4 + '[' 4 -le 9 ']' + echo n151 ++ expr 4 + 1 + i=5 + '[' 5 -le 9 ']' + echo n151 ++ expr 5 + 1 + i=6 + '[' 6 -le 9 ']' + echo n151 ++ expr 6 + 1 + i=7 + '[' 7 -le 9 ']' + echo n151 ++ expr 7 + 1 + i=8 + '[' 8 -le 9 ']' + echo n151 ++ expr 8 + 1 + i=9 + '[' 9 -le 9 ']' + echo n151 ++ expr 9 + 1 + i=10 + '[' 10 -le 9 ']' + read line ++ echo n145 1 wp10@n145 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n145 ++ echo n145 1 wp10@n145 UNDEFINED ++ cut -f2 '-d ' + nslots=1 + i=1 + '[' 1 -le 1 ']' + echo n145 ++ expr 1 + 1 + i=2 + '[' 2 -le 1 ']' + read line ++ echo n143 8 wp10@n143 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n143 ++ echo n143 8 wp10@n143 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n143 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n143 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n143 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n143 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n143 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n143 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n143 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n143 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line ++ echo n163 7 wp10@n163 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n163 ++ echo n163 7 wp10@n163 UNDEFINED ++ cut -f2 '-d ' + nslots=7 + i=1 + '[' 1 -le 7 ']' + echo n163 ++ expr 1 + 1 + i=2 + '[' 2 -le 7 ']' + echo n163 ++ expr 2 + 1 + i=3 + '[' 3 -le 7 ']' + echo n163 ++ expr 3 + 1 + i=4 + '[' 4 -le 7 ']' + echo n163 ++ expr 4 + 1 + i=5 + '[' 5 -le 7 ']' + echo n163 ++ expr 5 + 1 + i=6 + '[' 6 -le 7 ']' + echo n163 ++ expr 6 + 1 + i=7 + '[' 7 -le 7 ']' + echo n163 ++ expr 7 + 1 + i=8 + '[' 8 -le 7 ']' + read line + cat /tmp/8202.1.wp10/machines n161 n161 n161 n151 n151 n151 n151 n151 n151 n151 n151 n151 n145 n143 n143 n143 n143 n143 n143 n143 n143 n163 n163 n163 n163 n163 n163 n163 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/8202.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=23202 ++ cat /tmp/8202.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 23202 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 23202 rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 23202 -d 0 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 23202 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 23202 rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 23202 -d 0 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n163 /opt/mpich2/intel/11.1//bin/smpd 23202 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/8202.1.wp10/machines ++ uniq rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 23202 -d 0 rsh n163 /opt/mpich2/intel/11.1//bin/smpd -port 23202 -d 0 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 23202 -status n161 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 23202 -d 0 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n161' startmpich2.sh: found running smpd on n161 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 23202 -status n151 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n151' startmpich2.sh: found running smpd on n151 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 23202 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n145' startmpich2.sh: found running smpd on n145 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 23202 -status n143 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n143' startmpich2.sh: found running smpd on n143 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 23202 -status n163 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n163' startmpich2.sh: found running smpd on n163 + '[' 5 -eq 5 ']' + echo 'startmpich2.sh: got all 5 of 5 nodes' startmpich2.sh: got all 5 of 5 nodes + exit 0 START END Sat Jun 25 07:42:40 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Sat Jun 25 07:42:45 EDT 2011