USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  20176
JOB_NAME  =  F40
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  4

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0

EXE_HOST  =  n141
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/20176.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20176
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0/VERTEX: not a plain file
Failure to rcp files!

Your job started at Fri Dec 9 12:30:43 EST 2011 on host n141 in /mnt/p05/20176jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile ']'
+ machines=/tmp/20176.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile
+ cat /opt/sge6/spool/n141/active_jobs/20176.1/pe_hostfile
+ read line
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 12 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n141
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n141
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n141
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n141
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n141
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n141
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 8 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n159
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n159
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n159
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n165 10 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 10 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n151 6 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 6 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
+ cat /tmp/20176.1.wp10/machines
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n141
n159
n159
n159
n159
n159
n159
n159
n159
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n151
n151
n151
n151
n151
n151
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/20176.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20176
++ cat /tmp/20176.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 20176
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 20176
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 20176
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 20176
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/20176.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 20176 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n141
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 20176 -d 0
++ head -n 1
++ cut -f 1 -d ' '
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 20176 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 20176 -d 0
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ '[' 4 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/20176.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20176 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ '[' 4 -eq 4 ']'
+ echo 'startmpich2.sh: got all 4 of 4 nodes'
startmpich2.sh: got all 4 of 4 nodes
+ exit 0
rm: cannot remove `mpi_prefix.dat': No such file or directory
gzip: *.gz: No such file or directory
rm: cannot remove `wnohu*': No such file or directory
START
rm: cannot remove `vertex*': No such file or directory
END
Sun Dec 11 19:54:16 EST 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sun Dec 11 19:54:21 EST 2011