USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  16991
JOB_NAME  =  FS-LOW
QUEUE     =  wp12
NSLOTS    =  49
NHOSTS    =  6

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW

EXE_HOST  =  n173
EXE_DIR   =  /mnt/n173
OUT_DIR   =  /scratch
TMP_DIR   =  /tmp/16991.1.wp12

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  21991
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/FIX_MU: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Thu Oct 27 16:48:09 EDT 2011 on host n173 in /mnt/n173/16991jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile ']'
+ machines=/tmp/16991.1.wp12/machines
+ PeHostfile2MachineFile /opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile
+ cat /opt/sge6/core/spool/n173/active_jobs/16991.1/pe_hostfile
+ read line
++ echo n173 8 wp12@n173 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n173
++ echo n173 8 wp12@n173 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n173
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n173
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n173
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n173
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n173
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n173
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n173
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n173
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n176 1 wp12@n176 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n176
++ echo n176 1 wp12@n176 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n176
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n172
++ echo n172 12 wp12@n172 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n172
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n172
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n172
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n172
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n172
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n172
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n172
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n172
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n172
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n172
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n172
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n172
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n175 4 wp12@n175 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n175
++ echo n175 4 wp12@n175 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n175
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n175
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n175
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n175
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n181 12 wp12@n181 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n181
++ echo n181 12 wp12@n181 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n181
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n181
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n181
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n181
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n181
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n181
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n181
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n181
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n181
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n181
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n181
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n181
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n182
++ echo n182 12 wp12@n182 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n182
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n182
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n182
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n182
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n182
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n182
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n182
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n182
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n182
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n182
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n182
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n182
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
+ cat /tmp/16991.1.wp12/machines
n173
n173
n173
n173
n173
n173
n173
n173
n176
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n172
n175
n175
n175
n175
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n181
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
n182
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/16991.1.wp12/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=21991
++ cat /tmp/16991.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n173 /opt/mpich2/intel/11.1//bin/smpd 21991
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n176 /opt/mpich2/intel/11.1//bin/smpd 21991
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n172 /opt/mpich2/intel/11.1//bin/smpd 21991
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n175 /opt/mpich2/intel/11.1//bin/smpd 21991
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n181 /opt/mpich2/intel/11.1//bin/smpd 21991
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n182 /opt/mpich2/intel/11.1//bin/smpd 21991
rsh n173 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/16991.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n173
++ head -n 1
rsh n176 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
++ cut -f 1 -d ' '
rsh n172 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
rsh n181 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
rsh n175 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
rsh n182 /opt/mpich2/intel/11.1//bin/smpd -port 21991 -d 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n173'
startmpich2.sh: missing smpd on n173
+ for host in '`cat $machines | uniq`'
++ head -n 1
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n176
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n176'
startmpich2.sh: missing smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n172
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n172'
startmpich2.sh: missing smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n175
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n175'
startmpich2.sh: missing smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n181
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n181'
startmpich2.sh: missing smpd on n181
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n182
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n182'
startmpich2.sh: missing smpd on n182
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/16991.1.wp12/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n173
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n173'
startmpich2.sh: found running smpd on n173
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n176
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n176'
startmpich2.sh: found running smpd on n176
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n172
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n172'
startmpich2.sh: found running smpd on n172
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n175
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n175'
startmpich2.sh: found running smpd on n175
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n181
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n181'
startmpich2.sh: found running smpd on n181
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 21991 -status n182
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n182'
startmpich2.sh: found running smpd on n182
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
START
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Fri Oct 28 23:24:39 EDT 2011