USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  5295
JOB_NAME  =  SUNDAY_ONLY-F100
QUEUE     =  wp10
NSLOTS    =  57
NHOSTS    =  7

SUB_HOST  =  rupc06
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW

EXE_HOST  =  n145
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p06
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/5295.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  20295
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW/imp.0: not a plain file
Failure to rcp files!

Your job started at Sun Apr 24 01:00:11 EDT 2011 on host n145 in /mnt/p06/5295jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta100/nf6/NEW on host rupc06
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile ']'
+ machines=/tmp/5295.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile
+ cat /opt/sge6/spool/n145/active_jobs/5295.1/pe_hostfile
+ read line
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n145
++ echo n145 12 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n145
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n145
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n145
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n145
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n145
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n145
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n145
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n145
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n145
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n145
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n145
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n165
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n165
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 12 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n150
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n160 7 wp10@n160 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n160
++ echo n160 7 wp10@n160 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n160
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n160
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n160
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n160
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n160
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n160
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n160
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n164 1 wp10@n164 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n164
++ echo n164 1 wp10@n164 UNDEFINED
++ cut -f2 '-d '
+ nslots=1
+ i=1
+ '[' 1 -le 1 ']'
+ echo n164
++ expr 1 + 1
+ i=2
+ '[' 2 -le 1 ']'
+ read line
++ echo n156 8 wp10@n156 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n156
++ echo n156 8 wp10@n156 UNDEFINED
++ cut -f2 '-d '
+ nslots=8
+ i=1
+ '[' 1 -le 8 ']'
+ echo n156
++ expr 1 + 1
+ i=2
+ '[' 2 -le 8 ']'
+ echo n156
++ expr 2 + 1
+ i=3
+ '[' 3 -le 8 ']'
+ echo n156
++ expr 3 + 1
+ i=4
+ '[' 4 -le 8 ']'
+ echo n156
++ expr 4 + 1
+ i=5
+ '[' 5 -le 8 ']'
+ echo n156
++ expr 5 + 1
+ i=6
+ '[' 6 -le 8 ']'
+ echo n156
++ expr 6 + 1
+ i=7
+ '[' 7 -le 8 ']'
+ echo n156
++ expr 7 + 1
+ i=8
+ '[' 8 -le 8 ']'
+ echo n156
++ expr 8 + 1
+ i=9
+ '[' 9 -le 8 ']'
+ read line
++ echo n157 5 wp10@n157 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n157
++ echo n157 5 wp10@n157 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n157
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n157
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n157
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n157
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n157
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
+ cat /tmp/5295.1.wp10/machines
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n160
n160
n160
n160
n160
n160
n160
n164
n156
n156
n156
n156
n156
n156
n156
n156
n157
n157
n157
n157
n157
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/5295.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=20295
++ cat /tmp/5295.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 20295
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 20295
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 20295
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n160 /opt/mpich2/intel/11.1//bin/smpd 20295
+ for host in '`cat $machines | uniq`'
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n164 /opt/mpich2/intel/11.1//bin/smpd 20295
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n156 /opt/mpich2/intel/11.1//bin/smpd 20295
rsh n160 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n157 /opt/mpich2/intel/11.1//bin/smpd 20295
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/5295.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n145
rsh n164 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
++ head -n 1
rsh n157 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
rsh n156 /opt/mpich2/intel/11.1//bin/smpd -port 20295 -d 0
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n160
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n160'
startmpich2.sh: missing smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n164
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n164'
startmpich2.sh: missing smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n156
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n156'
startmpich2.sh: missing smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n157
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n157'
startmpich2.sh: missing smpd on n157
+ '[' 7 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/5295.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n145
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n160
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n160'
startmpich2.sh: found running smpd on n160
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n164
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n164'
startmpich2.sh: found running smpd on n164
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n156
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n156'
startmpich2.sh: found running smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 20295 -status n157
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n157'
startmpich2.sh: found running smpd on n157
+ '[' 7 -eq 7 ']'
+ echo 'startmpich2.sh: got all 7 of 7 nodes'
startmpich2.sh: got all 7 of 7 nodes
+ exit 0
START
END
Sun Apr 24 11:05:08 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Sun Apr 24 11:05:14 EDT 2011