USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 6354 JOB_NAME = F100 QUEUE = wp10 NSLOTS = 60 NHOSTS = 5 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW EXE_HOST = n145 NFS_HOST = caip02s EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/6354.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 21354 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/FIX_mu: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/GO-small: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/SPEC: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/ancont: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/cdos: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Tue May 17 09:44:12 EDT 2011 on host n145 in /mnt/p06/6354jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile ']' + machines=/tmp/6354.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile + cat /opt/sge6/spool/n145/active_jobs/6354.1/pe_hostfile + read line ++ echo n145 12 wp10@n145 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n145 ++ echo n145 12 wp10@n145 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n145 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n145 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n145 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n145 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n145 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n145 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n145 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n145 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n145 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n145 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n145 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n145 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n146 12 wp10@n146 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n146 ++ echo n146 12 wp10@n146 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n146 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n146 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n146 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n146 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n146 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n146 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n146 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n146 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n146 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n146 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n146 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n146 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n147 12 wp10@n147 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n147 ++ echo n147 12 wp10@n147 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n147 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n147 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n147 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n147 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n147 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n147 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n147 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n147 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n147 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n147 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n147 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n147 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n148 12 wp10@n148 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n148 ++ echo n148 12 wp10@n148 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n148 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n148 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n148 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n148 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n148 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n148 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n148 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n148 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n148 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n148 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n148 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n148 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n149 12 wp10@n149 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n149 ++ echo n149 12 wp10@n149 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n149 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n149 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n149 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n149 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n149 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n149 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n149 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n149 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n149 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n149 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n149 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n149 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line + cat /tmp/6354.1.wp10/machines n145 n145 n145 n145 n145 n145 n145 n145 n145 n145 n145 n145 n146 n146 n146 n146 n146 n146 n146 n146 n146 n146 n146 n146 n147 n147 n147 n147 n147 n147 n147 n147 n147 n147 n147 n147 n148 n148 n148 n148 n148 n148 n148 n148 n148 n148 n148 n148 n149 n149 n149 n149 n149 n149 n149 n149 n149 n149 n149 n149 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/6354.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=21354 ++ cat /tmp/6354.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 21354 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 21354 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 21354 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 21354 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 21354 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/6354.1.wp10/machines rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 21354 -d 0 ++ uniq + for host in '`cat $machines | uniq`' rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 21354 -d 0 ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n145 rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 21354 -d 0 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 21354 -d 0 rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 21354 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n145' startmpich2.sh: missing smpd on n145 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n146 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n146' startmpich2.sh: missing smpd on n146 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n147 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n147' startmpich2.sh: missing smpd on n147 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n148 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n148' startmpich2.sh: missing smpd on n148 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n149 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n149' startmpich2.sh: missing smpd on n149 + '[' 5 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/6354.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n145' startmpich2.sh: found running smpd on n145 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n146 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n146' startmpich2.sh: found running smpd on n146 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n147 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n147' startmpich2.sh: found running smpd on n147 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n148 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n148' startmpich2.sh: found running smpd on n148 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 21354 -status n149 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n149' startmpich2.sh: found running smpd on n149 + '[' 5 -eq 5 ']' + echo 'startmpich2.sh: got all 5 of 5 nodes' startmpich2.sh: got all 5 of 5 nodes + exit 0 START END Fri May 20 17:23:41 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Fri May 20 17:23:46 EDT 2011