USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  20623
JOB_NAME  =  F40
QUEUE     =  wp10
NSLOTS    =  64
NHOSTS    =  6

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0

EXE_HOST  =  n147
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/20623.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20623
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0/20176jtomczak: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0/VERTEX: not a plain file
Failure to rcp files!

Your job started at Fri Dec 16 14:33:32 EST 2011 on host n147 in /mnt/p06/20623jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta40/nf6/NEW/imp.0 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile ']'
+ machines=/tmp/20623.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile
+ cat /opt/sge6/spool/n147/active_jobs/20623.1/pe_hostfile
+ read line
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n147
++ echo n147 12 wp10@n147 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n147
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n147
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n147
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n147
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n147
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n147
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n147
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n147
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n147
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n147
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n147
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n147
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 12 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n148
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n148
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n149 12 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 12 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n149
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n149
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n149
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n149
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n149
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n149
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n149
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n149
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 12 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n151
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n151
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n151
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n151
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n151
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n151
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n151
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n151
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n151
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n152
++ echo n152 4 wp10@n152 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n152
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n152
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n152
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n152
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/20623.1.wp10/machines
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n147
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n149
n149
n149
n149
n149
n149
n149
n149
n149
n149
n149
n149
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n151
n152
n152
n152
n152
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/20623.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20623
++ cat /tmp/20623.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n147 /opt/mpich2/intel/11.1//bin/smpd 20623
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 20623
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 20623
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 20623
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 20623
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n152 /opt/mpich2/intel/11.1//bin/smpd 20623
rsh n147 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/20623.1.wp10/machines
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
++ uniq
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n147
++ head -n 1
++ cut -f 1 -d ' '
rsh n152 /opt/mpich2/intel/11.1//bin/smpd -port 20623 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n147'
startmpich2.sh: missing smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n148
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n152
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n152'
startmpich2.sh: missing smpd on n152
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/20623.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n147
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n147'
startmpich2.sh: found running smpd on n147
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20623 -status n152
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n152'
startmpich2.sh: found running smpd on n152
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
rm: cannot remove `mpi_prefix.dat': No such file or directory
gzip: *.gz: No such file or directory
rm: cannot remove `wnohu*': No such file or directory
rm: cannot remove `stat*': No such file or directory
START
END
Sat Dec 31 06:07:56 EST 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Dec 31 06:08:01 EST 2011