USER = jtomczak HOMEDIR = /home/users/jtomczak JOB_ID = 17748 JOB_NAME = FS-b150-6 QUEUE = wp10 NSLOTS = 31 NHOSTS = 3 SUB_HOST = rupc04 SUB_DIR = /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW EXE_HOST = n164 NFS_HOST = caip02s EXE_DIR = /mnt/p05 OUT_DIR = /tmp TMP_DIR = /tmp/17748.1.wp10 INFO = Files larger than 100MB will be compressed to reduce the network load. INFO2 = Do NOT remove this file while job is running! PORT = 22748 rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu_2: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/FIX_mu_3: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/GO-small: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/ancont: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/cdos: not a plain file rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW/imp.0: not a plain file Failure to rcp files! Your job started at Sun Nov 6 16:23:48 EST 2011 on host n164 in /mnt/p05/17748jtomczak Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/NEW on host rupc04 ----------------------------------------------------------------------------------------- + echo -catch_rsh /opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile /opt/mpich2/intel/11.1/ -catch_rsh /opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile /opt/mpich2/intel/11.1/ + SLEEPTIME=5 + RETRIES=10 + catch_rsh=0 + catch_hostname=0 + '[' 3 -gt 0 ']' + case "$1" in + catch_rsh=1 + shift + '[' 2 -gt 0 ']' + case "$1" in + break ++ basename /opt/sge6/mpich2_smpd/startmpich2.sh + me=startmpich2.sh + '[' 2 -lt 2 ']' + pe_hostfile=/opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile + MPICH2_ROOT=/opt/mpich2/intel/11.1/ + export MPICH2_ROOT + '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']' + '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']' + '[' '!' -r /opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile ']' + machines=/tmp/17748.1.wp10/machines + PeHostfile2MachineFile /opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile + cat /opt/sge6/spool/n164/active_jobs/17748.1/pe_hostfile + read line ++ echo n164 11 wp10@n164 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n164 ++ echo n164 11 wp10@n164 UNDEFINED ++ cut -f2 '-d ' + nslots=11 + i=1 + '[' 1 -le 11 ']' + echo n164 ++ expr 1 + 1 + i=2 + '[' 2 -le 11 ']' + echo n164 ++ expr 2 + 1 + i=3 + '[' 3 -le 11 ']' + echo n164 ++ expr 3 + 1 + i=4 + '[' 4 -le 11 ']' + echo n164 ++ expr 4 + 1 + i=5 + '[' 5 -le 11 ']' + echo n164 ++ expr 5 + 1 + i=6 + '[' 6 -le 11 ']' + echo n164 ++ expr 6 + 1 + i=7 + '[' 7 -le 11 ']' + echo n164 ++ expr 7 + 1 + i=8 + '[' 8 -le 11 ']' + echo n164 ++ expr 8 + 1 + i=9 + '[' 9 -le 11 ']' + echo n164 ++ expr 9 + 1 + i=10 + '[' 10 -le 11 ']' + echo n164 ++ expr 10 + 1 + i=11 + '[' 11 -le 11 ']' + echo n164 ++ expr 11 + 1 + i=12 + '[' 12 -le 11 ']' + read line ++ echo n155 12 wp10@n155 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n155 ++ echo n155 12 wp10@n155 UNDEFINED ++ cut -f2 '-d ' + nslots=12 + i=1 + '[' 1 -le 12 ']' + echo n155 ++ expr 1 + 1 + i=2 + '[' 2 -le 12 ']' + echo n155 ++ expr 2 + 1 + i=3 + '[' 3 -le 12 ']' + echo n155 ++ expr 3 + 1 + i=4 + '[' 4 -le 12 ']' + echo n155 ++ expr 4 + 1 + i=5 + '[' 5 -le 12 ']' + echo n155 ++ expr 5 + 1 + i=6 + '[' 6 -le 12 ']' + echo n155 ++ expr 6 + 1 + i=7 + '[' 7 -le 12 ']' + echo n155 ++ expr 7 + 1 + i=8 + '[' 8 -le 12 ']' + echo n155 ++ expr 8 + 1 + i=9 + '[' 9 -le 12 ']' + echo n155 ++ expr 9 + 1 + i=10 + '[' 10 -le 12 ']' + echo n155 ++ expr 10 + 1 + i=11 + '[' 11 -le 12 ']' + echo n155 ++ expr 11 + 1 + i=12 + '[' 12 -le 12 ']' + echo n155 ++ expr 12 + 1 + i=13 + '[' 13 -le 12 ']' + read line ++ echo n157 8 wp10@n157 UNDEFINED ++ cut -f1 '-d ' ++ cut -f1 -d. + host=n157 ++ echo n157 8 wp10@n157 UNDEFINED ++ cut -f2 '-d ' + nslots=8 + i=1 + '[' 1 -le 8 ']' + echo n157 ++ expr 1 + 1 + i=2 + '[' 2 -le 8 ']' + echo n157 ++ expr 2 + 1 + i=3 + '[' 3 -le 8 ']' + echo n157 ++ expr 3 + 1 + i=4 + '[' 4 -le 8 ']' + echo n157 ++ expr 4 + 1 + i=5 + '[' 5 -le 8 ']' + echo n157 ++ expr 5 + 1 + i=6 + '[' 6 -le 8 ']' + echo n157 ++ expr 6 + 1 + i=7 + '[' 7 -le 8 ']' + echo n157 ++ expr 7 + 1 + i=8 + '[' 8 -le 8 ']' + echo n157 ++ expr 8 + 1 + i=9 + '[' 9 -le 8 ']' + read line + cat /tmp/17748.1.wp10/machines n164 n164 n164 n164 n164 n164 n164 n164 n164 n164 n164 n155 n155 n155 n155 n155 n155 n155 n155 n155 n155 n155 n155 n157 n157 n157 n157 n157 n157 n157 n157 + '[' 1 = 1 ']' + rsh_wrapper=/opt/sge6/mpich2_smpd/rsh + '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']' + rshcmd=rsh + case "$ARC" in + ln -s /opt/sge6/mpich2_smpd/rsh /tmp/17748.1.wp10/rsh + hash -r + '[' 0 = 1 ']' + port=22748 ++ cat /tmp/17748.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 22748 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 22748 + for host in '`cat $machines | uniq`' + /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 22748 + actual_retry=1 + '[' 1 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (1 of 10)' startmpich2.sh: check for smpd daemons (1 of 10) + numhosts=0 ++ cat /tmp/17748.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n164 ++ head -n 1 rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 22748 -d 0 rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 22748 -d 0 ++ cut -f 1 -d ' ' rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 22748 -d 0 Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n164' startmpich2.sh: missing smpd on n164 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n155 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n155' startmpich2.sh: missing smpd on n155 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n157 ++ head -n 1 ++ cut -f 1 -d ' ' Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1 internal ABORT - process 0 + '[' '' == smpd ']' + echo 'startmpich2.sh: missing smpd on n157' startmpich2.sh: missing smpd on n157 + '[' 3 -eq 0 ']' + sleep 5 + (( actual_retry += 1 )) + '[' 2 -le 10 ']' + echo 'startmpich2.sh: check for smpd daemons (2 of 10)' startmpich2.sh: check for smpd daemons (2 of 10) + numhosts=0 ++ cat /tmp/17748.1.wp10/machines ++ uniq + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n164 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n164' startmpich2.sh: found running smpd on n164 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n155 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n155' startmpich2.sh: found running smpd on n155 + for host in '`cat $machines | uniq`' ++ /opt/mpich2/intel/11.1//bin/smpd -port 22748 -status n157 ++ head -n 1 ++ cut -f 1 -d ' ' + '[' smpd == smpd ']' + (( numhosts += 1 )) + echo 'startmpich2.sh: found running smpd on n157' startmpich2.sh: found running smpd on n157 + '[' 3 -eq 3 ']' + echo 'startmpich2.sh: got all 3 of 3 nodes' startmpich2.sh: got all 3 of 3 nodes + exit 0 START END Fri Nov 18 19:51:25 EST 2011 -catch_rsh /opt/mpich2/intel/11.1/ Your job finished at Fri Nov 18 19:51:31 EST 2011