USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  4667
JOB_NAME  =  FS-b150-6
QUEUE     =  wp10
NSLOTS    =  25
NHOSTS    =  7

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6

EXE_HOST  =  n162
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/4667.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  24667
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6/imp.0: not a plain file
Failure to rcp files!

Your job started at Wed Apr 6 08:41:38 EDT 2011 on host n162 in /mnt/p06/4667jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta150/nf6 on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile ']'
+ machines=/tmp/4667.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile
+ cat /opt/sge6/spool/n162/active_jobs/4667.1/pe_hostfile
+ read line
++ echo n162 3 wp10@n162 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n162
++ echo n162 3 wp10@n162 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n162
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n162
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n162
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n165 2 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 2 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=2
+ i=1
+ '[' 1 -le 2 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 2 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 2 ']'
+ read line
++ echo n161 1 wp10@n161 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n161
++ echo n161 1 wp10@n161 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n161
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n155 4 wp10@n155 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n155
++ echo n155 4 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n155
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n155
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n155
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n146 1 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 1 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n149
++ echo n149 4 wp10@n149 UNDEFINED
++ cut -f2 '-d '
+ nslots=4
+ i=1
+ '[' 1 -le 4 ']'
+ echo n149
++ expr 1 + 1
+ i=2
+ '[' 2 -le 4 ']'
+ echo n149
++ expr 2 + 1
+ i=3
+ '[' 3 -le 4 ']'
+ echo n149
++ expr 3 + 1
+ i=4
+ '[' 4 -le 4 ']'
+ echo n149
++ expr 4 + 1
+ i=5
+ '[' 5 -le 4 ']'
+ read line
++ echo n143 10 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 10 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=10
+ i=1
+ '[' 1 -le 10 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 10 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 10 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 10 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 10 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 10 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 10 ']'
+ echo n143
++ expr 7 + 1
+ i=8
+ '[' 8 -le 10 ']'
+ echo n143
++ expr 8 + 1
+ i=9
+ '[' 9 -le 10 ']'
+ echo n143
++ expr 9 + 1
+ i=10
+ '[' 10 -le 10 ']'
+ echo n143
++ expr 10 + 1
+ i=11
+ '[' 11 -le 10 ']'
+ read line
+ cat /tmp/4667.1.wp10/machines
n162
n162
n162
n165
n165
n161
n155
n155
n155
n155
n146
n149
n149
n149
n149
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/4667.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=24667
++ cat /tmp/4667.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n162 /opt/mpich2/intel/11.1//bin/smpd 24667
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 24667
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n161 /opt/mpich2/intel/11.1//bin/smpd 24667
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 24667
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 24667
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n149 /opt/mpich2/intel/11.1//bin/smpd 24667
rsh n162 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
rsh n161 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 24667
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
rsh n149 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
++ cat /tmp/4667.1.wp10/machines
++ uniq
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n162
++ head -n 1
++ cut -f 1 -d ' '
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 24667 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n162'
startmpich2.sh: missing smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n161
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n161'
startmpich2.sh: missing smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n155
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n155'
startmpich2.sh: missing smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n146
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n149
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n149'
startmpich2.sh: missing smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n143
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ '[' 7 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/4667.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n162
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n162'
startmpich2.sh: found running smpd on n162
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n161
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n161'
startmpich2.sh: found running smpd on n161
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n149
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n149'
startmpich2.sh: found running smpd on n149
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 24667 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
START
END
Sat Apr  9 09:21:28 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sat Apr 9 09:21:33 EDT 2011