USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 5287 JOB_NAME = FAST-F100 QUEUE = wp10 NSLOTS = 57 NHOSTS = 6 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW EXE_HOST = n143 NFS_HOST = caip02s EXE_DIR = /mnt/p06 OUT_DIR = /tmp TMP_DIR = /tmp/5287.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 20287 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Sat Apr 23 16:46:24 EDT 2011 on host n143 in /mnt/p06/5287jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile ']' + machines=/tmp/5287.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile + cat /opt/sge6/spool/n143/active_jobs/5287.1/pe_hostfile + read line ++ echo n143 12 wp10@n143 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n143 ++ echo n143 12 wp10@n143 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n143 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n143 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n143 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n143 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n143 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n143 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n143 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n143 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n143 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n143 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n143 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n143 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n144 12 wp10@n144 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n144 ++ echo n144 12 wp10@n144 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n144 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n144 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n144 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n144 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n144 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n144 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n144 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n144 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n144 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n144 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n144 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n144 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n145 8 wp10@n145 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n145 ++ echo n145 8 wp10@n145 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n145 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n145 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n145 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n145 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n145 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n145 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n145 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n145 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line ++ echo n142 11 wp10@n142 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n142 ++ echo n142 11 wp10@n142 UNDEFINED ++ cut -f2 '-d ' + nslots=11 + i=1 + '[' 1 -le 11 ']' + echo n142 ++ expr 1 + 1 + i=2 + '[' 2 -le 11 ']' + echo n142 ++ expr 2 + 1 + i=3 + '[' 3 -le 11 ']' + echo n142 ++ expr 3 + 1 + i=4 + '[' 4 -le 11 ']' + echo n142 ++ expr 4 + 1 + i=5 + '[' 5 -le 11 ']' + echo n142 ++ expr 5 + 1 + i=6 + '[' 6 -le 11 ']' + echo n142 ++ expr 6 + 1 + i=7 + '[' 7 -le 11 ']' + echo n142 ++ expr 7 + 1 + i=8 + '[' 8 -le 11 ']' + echo n142 ++ expr 8 + 1 + i=9 + '[' 9 -le 11 ']' + echo n142 ++ expr 9 + 1 + i=10 + '[' 10 -le 11 ']' + echo n142 ++ expr 10 + 1 + i=11 + '[' 11 -le 11 ']' + echo n142 ++ expr 11 + 1 + i=12 + '[' 12 -le 11 ']' + read line ++ echo n159 8 wp10@n159 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n159 ++ echo n159 8 wp10@n159 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n159 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n159 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n159 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n159 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n159 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n159 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n159 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n159 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line ++ echo n160 6 wp10@n160 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n160 ++ echo n160 6 wp10@n160 UNDEFINED ++ cut -f2 '-d ' + nslots=6 + i=1 + '[' 1 -le 6 ']' + echo n160 ++ expr 1 + 1 + i=2 + '[' 2 -le 6 ']' + echo n160 ++ expr 2 + 1 + i=3 + '[' 3 -le 6 ']' + echo n160 ++ expr 3 + 1 + i=4 + '[' 4 -le 6 ']' + echo n160 ++ expr 4 + 1 + i=5 + '[' 5 -le 6 ']' + echo n160 ++ expr 5 + 1 + i=6 + '[' 6 -le 6 ']' + echo n160 ++ expr 6 + 1 + i=7 + '[' 7 -le 6 ']' + read line + cat /tmp/5287.1.wp10/machines n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n143 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n144 n145 n145 n145 n145 n145 n145 n145 n145 n142 n142 n142 n142 n142 n142 n142 n142 n142 n142 n142 n159 n159 n159 n159 n159 n159 n159 n159 n160 n160 n160 n160 n160 n160 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5287.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=20287 ++ cat /tmp/5287.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 20287 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 20287 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 20287 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n142 /opt/mpich2/intel/11.1//bin/smpd 20287 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 20287 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 20287 rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 ++ cat /tmp/5287.1.wp10/machines ++ uniq rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n143 ++ head -n 1 rsh n142 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 ++ cut -f 1 -d ' ' rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 20287 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n143' startmpich2.sh: missing smpd on n143 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n144 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n144' startmpich2.sh: missing smpd on n144 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n145' startmpich2.sh: missing smpd on n145 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n142 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n142' startmpich2.sh: missing smpd on n142 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n159 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n159' startmpich2.sh: missing smpd on n159 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n160 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n160' startmpich2.sh: missing smpd on n160 + '[' 6 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/5287.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n143 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n143' startmpich2.sh: found running smpd on n143 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n144 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n144' startmpich2.sh: found running smpd on n144 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n145 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n145' startmpich2.sh: found running smpd on n145 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n142 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n142' startmpich2.sh: found running smpd on n142 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n159 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n159' startmpich2.sh: found running smpd on n159 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 20287 -status n160 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n160' startmpich2.sh: found running smpd on n160 + '[' 6 -eq 6 ']' + echo 'startmpich2.sh: got all 6 of 6 nodes' startmpich2.sh: got all 6 of 6 nodes + exit 0 START END Sat Apr 23 19:25:49 EDT 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Sat Apr 23 19:25:55 EDT 2011