USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  8340
JOB_NAME  =  FS-LOW
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  6

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW

EXE_HOST  =  n148
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/8340.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23340
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Jun 25 17:36:09 EDT 2011 on host n148 in /mnt/p05/8340jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta200/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile ']'
+ machines=/tmp/8340.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile
+ cat /opt/sge6/spool/n148/active_jobs/8340.1/pe_hostfile
+ read line
++ echo n148 5 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 5 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n158 3 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 3 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n159 1 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 1 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n157 11 wp10@n157 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n157
++ echo n157 11 wp10@n157 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n157
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n157
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n157
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n157
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n157
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n157
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n157
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n157
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n157
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n157
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n157
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n150 6 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 6 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n165 10 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 10 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
+ cat /tmp/8340.1.wp10/machines
n148
n148
n148
n148
n148
n158
n158
n158
n159
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n150
n150
n150
n150
n150
n150
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/8340.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23340
++ cat /tmp/8340.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 23340
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 23340
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 23340
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 23340
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 23340
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 23340
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/8340.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n148
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
++ cut -f 1 -d ' '
++ head -n 1
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 23340 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n158
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n157
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n157'
startmpich2.sh: missing smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ '[' 6 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/8340.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n157
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23340 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ '[' 6 -eq 6 ']'
+ echo 'startmpich2.sh: got all 6 of 6 nodes'
startmpich2.sh: got all 6 of 6 nodes
+ exit 0
START
END
Wed Jun 29 13:34:41 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Wed Jun 29 13:34:48 EDT 2011