USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 5291 JOB_NAME = FAST-F100 QUEUE = wp10 NSLOTS = 57 NHOSTS = 6 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW EXE_HOST = n164 NFS_HOST = caip02s EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/5291.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20291 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Sat Apr 23 19:32:55 EDT 2011 on host n164 in /mnt/p06/5291jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile ']' + machines=/tmp/5291.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile + read line + cat /opt/sge6/spool/n164/active_jobs/5291.1/pe_hostfile ++ cut -f1 -d. ++ echo n164 1 wp10@n164 UNDEFINED ++ cut -f1 '-d ' + host=n164 ++ echo n164 1 wp10@n164 UNDEFINED ++ cut -f2 '-d ' + nslots=1 + i=1 + '[' 1 -le 1 ']' + echo n164 ++ expr 1 + 1 + i=2 + '[' 2 -le 1 ']' + read line ++ cut -f1 -d. ++ cut -f1 '-d ' ++ echo n156 8 wp10@n156 UNDEFINED + host=n156 ++ echo n156 8 wp10@n156 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n156 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n156 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n156 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n156 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n156 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n156 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n156 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n156 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line ++ cut -f1 -d. ++ echo n157 12 wp10@n157 UNDEFINED ++ cut -f1 '-d ' + host=n157 ++ cut -f2 '-d ' ++ echo n157 12 wp10@n157 UNDEFINED + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n157 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n157 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n157 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n157 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n157 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n157 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n157 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n157 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n157 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n157 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n157 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n157 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ cut -f1 -d. ++ echo n158 12 wp10@n158 UNDEFINED ++ cut -f1 '-d ' + host=n158 ++ echo n158 12 wp10@n158 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n158 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n158 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n158 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n158 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n158 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n158 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n158 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n158 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n158 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n158 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n158 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n158 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ cut -f1 -d. ++ cut -f1 '-d ' ++ echo n143 12 wp10@n143 UNDEFINED + host=n143 ++ cut -f2 '-d ' ++ echo n143 12 wp10@n143 UNDEFINED + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n143 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n143 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n143 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n143 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n143 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n143 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n143 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n143 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n143 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n143 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n143 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n143 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ cut -f1 -d. ++ echo n144 12 wp10@n144 UNDEFINED ++ cut -f1 '-d ' + host=n144 ++ cut -f2 '-d ' ++ echo n144 12 wp10@n144 UNDEFINED + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n144 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n144 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n144 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n144 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n144 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n144 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n144 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n144 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n144 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n144 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n144 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n144 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line + cat /tmp/5291.1.wp10/machines n164 n156 n156 n156 n156 n156 n156 n156 n156 n157 n157 n157 n157 n157 n157 n157 n157 n157 n157 n157 n157 n158 n158 n158 n158 n158 n158 n158 n158 n158 n158 n158 n158 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5291.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=20291 ++ cat /tmp/5291.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 20291 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n156 /opt/mpich2/intel/11.1//bin/smpd 20291 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 20291 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 20291 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 20291 rsh n156 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 20291 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/5291.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n164 ++ cut -f 1 -d ' ' ++ head -n 1 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n164' startmpich2.sh: missing smpd on n164 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n156 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n156' startmpich2.sh: found running smpd on n156 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n157 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n157' startmpich2.sh: missing smpd on n157 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n158 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n158' startmpich2.sh: missing smpd on n158 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n143 ++ head -n 1 ++ cut -f 1 -d ' ' rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 20291 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n143' startmpich2.sh: missing smpd on n143 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n144 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n144' startmpich2.sh: missing smpd on n144 + '[' 6 -eq 1 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/5291.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ cut -f 1 -d ' ' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n164 ++ head -n 1 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n164' startmpich2.sh: found running smpd on n164 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n156 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n156' startmpich2.sh: found running smpd on n156 + for host in '`cat $machines | uniq`' ++ cut -f 1 -d ' ' ++ head -n 1 ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n157 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n157' startmpich2.sh: found running smpd on n157 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n158 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n158' startmpich2.sh: found running smpd on n158 + for host in '`cat $machines | uniq`' ++ cut -f 1 -d ' ' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n143 ++ head -n 1 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n143' startmpich2.sh: found running smpd on n143 + for host in '`cat $machines | uniq`' ++ cut -f 1 -d ' ' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20291 -status n144 ++ head -n 1 + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n144' startmpich2.sh: found running smpd on n144 + '[' 6 -eq 6 ']' + echo 'startmpich2.sh: got all 6 of 6 nodes' startmpich2.sh: got all 6 of 6 nodes + exit 0 START END Sun Apr 24 00:51:20 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Sun Apr 24 00:51:28 EDT 2011