USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  8103
JOB_NAME  =  FS6-b60
QUEUE     =  wp10
NSLOTS    =  36
NHOSTS    =  5

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW

EXE_HOST  =  n148
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/8103.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  23103
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/SPEC: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Sat Jun 18 15:13:31 EDT 2011 on host n148 in /mnt/p06/8103jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta60/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile ']'
+ machines=/tmp/8103.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile
+ cat /opt/sge6/spool/n148/active_jobs/8103.1/pe_hostfile
+ read line
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n148
++ echo n148 10 wp10@n148 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n148
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n148
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n148
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n148
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n148
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n148
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n148
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n148
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n148
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n148
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
++ echo n149 1 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 1 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n160 3 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 3 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n160
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n160
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n159 11 wp10@n159 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n159
++ echo n159 11 wp10@n159 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n159
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n159
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n159
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n159
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n159
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n159
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n159
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n159
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n159
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n159
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n159
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 11 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n158
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n158
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n158
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n158
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n158
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n158
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n158
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n158
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
+ cat /tmp/8103.1.wp10/machines
n148
n148
n148
n148
n148
n148
n148
n148
n148
n148
n149
n160
n160
n160
n159
n159
n159
n159
n159
n159
n159
n159
n159
n159
n159
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/8103.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=23103
++ cat /tmp/8103.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n148 /opt/mpich2/intel/11.1//bin/smpd 23103
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 23103
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 23103
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n159 /opt/mpich2/intel/11.1//bin/smpd 23103
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 23103
rsh n148 /opt/mpich2/intel/11.1//bin/smpd -port 23103 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/8103.1.wp10/machines
++ uniq
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 23103 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n148
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 23103 -d 0
++ head -n 1
++ cut -f 1 -d ' '
rsh n159 /opt/mpich2/intel/11.1//bin/smpd -port 23103 -d 0
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 23103 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n148'
startmpich2.sh: missing smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n159
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n159'
startmpich2.sh: missing smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n158
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ '[' 5 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/8103.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n148
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n148'
startmpich2.sh: found running smpd on n148
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n159
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n159'
startmpich2.sh: found running smpd on n159
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 23103 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
START
END
Tue Jun 21 22:38:15 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Jun 21 22:38:20 EDT 2011