USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  4867
JOB_NAME  =  FS-b150-6
QUEUE     =  wp10
NSLOTS    =  33
NHOSTS    =  7

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6

EXE_HOST  =  n146
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/4867.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  24867
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/GO: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/GO-small: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/ancont: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/cdos: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/cdos_fine: not a plain file
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/imp.0: not a plain file
Failure to rcp files!

Your job started at Wed Apr 13 14:08:31 EDT 2011 on host n146 in /mnt/p06/4867jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile ']'
+ machines=/tmp/4867.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile
+ cat /opt/sge6/spool/n146/active_jobs/4867.1/pe_hostfile
+ read line
++ echo n146 1 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 1 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n154 2 wp10@n154 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n154
++ echo n154 2 wp10@n154 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n154
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n154
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n142
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n142
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n142
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n142
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n142
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n142
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n142
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n142
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n142
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n142
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n142
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n142
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n142
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n141 1 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 1 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n160 2 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 2 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n160
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n157 11 wp10@n157 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n157
++ echo n157 11 wp10@n157 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n157
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n157
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n157
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n157
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n157
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n157
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n157
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n157
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n157
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n157
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n157
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
+ cat /tmp/4867.1.wp10/machines
n146
n154
n154
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n141
n160
n160
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n157
n149
n149
n149
n149
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/4867.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=24867
++ cat /tmp/4867.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 24867
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n154 /opt/mpich2/intel/11.1//bin/smpd 24867
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
rsh n154 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n142 /opt/mpich2/intel/11.1//bin/smpd 24867
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 24867
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 24867
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 24867
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 24867
rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/4867.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n146
++ head -n 1
++ cut -f 1 -d ' '
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n154
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n154'
startmpich2.sh: missing smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n142
++ head -n 1
++ cut -f 1 -d ' '
rsh n142 /opt/mpich2/intel/11.1//bin/smpd -port 24867 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n142'
startmpich2.sh: missing smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ cut -f 1 -d ' '
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n160
++ head -n 1
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n157
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ '[' 7 -eq 3 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/4867.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n154
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n154'
startmpich2.sh: found running smpd on n154
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n142
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n142'
startmpich2.sh: found running smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n157
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24867 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
START
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Tue Apr 19 08:17:56 EDT 2011