USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  5296
JOB_NAME  =  SUNDAY_ONLY-F100
QUEUE     =  wp10
NSLOTS    =  57
NHOSTS    =  5

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW

EXE_HOST  =  n158
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/5296.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20296
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Sun Apr 24 14:25:19 EDT 2011 on host n158 in /mnt/p05/5296jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile ']'
+ machines=/tmp/5296.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile
+ cat /opt/sge6/spool/n158/active_jobs/5296.1/pe_hostfile
+ read line
++ echo n158 12 wp10@n158 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n158
++ echo n158 12 wp10@n158 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n158
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n158
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n158
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n158
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n158
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n158
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n158
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n158
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n158
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n158
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n158
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n158
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n143 12 wp10@n143 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n143
++ echo n143 12 wp10@n143 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n143
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n143
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n143
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n143
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n143
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n143
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n143
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n143
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n143
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n143
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n143
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n143
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n144
++ echo n144 12 wp10@n144 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n144
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n144
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n144
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n144
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n144
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n144
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n144
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n144
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n144
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n144
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n144
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n144
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n145
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n145
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n145
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n145
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n145
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n145
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n145
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n145
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n145
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n145
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n145
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n145
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n165 9 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 9 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=9
+ i=1
+ '[' 1 -le 9 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 9 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 9 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 9 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 9 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 9 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 9 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 9 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 9 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 9 ']'
+ read line
+ cat /tmp/5296.1.wp10/machines
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n158
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n143
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n144
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n165
n165
n165
n165
n165
n165
n165
n165
n165
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5296.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20296
++ cat /tmp/5296.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n158 /opt/mpich2/intel/11.1//bin/smpd 20296
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n143 /opt/mpich2/intel/11.1//bin/smpd 20296
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 20296
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 20296
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 20296
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/5296.1.wp10/machines
rsh n158 /opt/mpich2/intel/11.1//bin/smpd -port 20296 -d 0
++ uniq
rsh n143 /opt/mpich2/intel/11.1//bin/smpd -port 20296 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n158
++ cut -f 1 -d ' '
rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 20296 -d 0
++ head -n 1
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 20296 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 20296 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n158'
startmpich2.sh: missing smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n143
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n143'
startmpich2.sh: missing smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n144
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n144'
startmpich2.sh: missing smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n145
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ '[' 5 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/5296.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n158
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n158'
startmpich2.sh: found running smpd on n158
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n143
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n143'
startmpich2.sh: found running smpd on n143
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n145
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20296 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ '[' 5 -eq 5 ']'
+ echo 'startmpich2.sh: got all 5 of 5 nodes'
startmpich2.sh: got all 5 of 5 nodes
+ exit 0
START
END
Sun Apr 24 23:02:03 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sun Apr 24 23:02:08 EDT 2011