USER      =  jtomczak
HOMEDIR   =  /home/users/jtomczak
JOB_ID    =  7856
JOB_NAME  =  FS-LOW
QUEUE     =  wp10
NSLOTS    =  84
NHOSTS    =  10

SUB_HOST  =  rupc04
SUB_DIR   =  /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250

EXE_HOST  =  n142
NFS_HOST  =  caip02s
EXE_DIR   =  /mnt/p05
OUT_DIR   =  /tmp
TMP_DIR   =  /tmp/7856.1.wp10

INFO      =  Files larger than 100MB will be compressed to reduce the network load.
INFO2     =  Do NOT remove this file while job is running!

PORT      =  22856
rcp: /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250/imp.0: not a plain file
Failure to rcp files!

Your job started at Mon Jun 13 12:19:07 EDT 2011 on host n142 in /mnt/p05/7856jtomczak
Submission directory is /mnt/wk07/jtomczak/DATA/MATERIALS/FeSi/DMFT/QMC/U5.0/FAST/beta250 on host rupc04
-----------------------------------------------------------------------------------------
+ echo -catch_rsh /opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile /opt/mpich2/intel/11.1/
-catch_rsh /opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile /opt/mpich2/intel/11.1/
+ SLEEPTIME=5
+ RETRIES=10
+ catch_rsh=0
+ catch_hostname=0
+ '[' 3 -gt 0 ']'
+ case "$1" in
+ catch_rsh=1
+ shift
+ '[' 2 -gt 0 ']'
+ case "$1" in
+ break
++ basename /opt/sge6/mpich2_smpd/startmpich2.sh
+ me=startmpich2.sh
+ '[' 2 -lt 2 ']'
+ pe_hostfile=/opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile
+ MPICH2_ROOT=/opt/mpich2/intel/11.1/
+ export MPICH2_ROOT
+ '[' '!' -x /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 ']'
+ '[' '!' -x /opt/mpich2/intel/11.1//bin/smpd ']'
+ '[' '!' -r /opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile ']'
+ machines=/tmp/7856.1.wp10/machines
+ PeHostfile2MachineFile /opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile
+ cat /opt/sge6/spool/n142/active_jobs/7856.1/pe_hostfile
+ read line
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n142
++ echo n142 12 wp10@n142 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n142
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n142
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n142
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n142
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n142
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n142
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n142
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n142
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n142
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n142
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n142
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n142
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n141 6 wp10@n141 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n141
++ echo n141 6 wp10@n141 UNDEFINED
++ cut -f2 '-d '
+ nslots=6
+ i=1
+ '[' 1 -le 6 ']'
+ echo n141
++ expr 1 + 1
+ i=2
+ '[' 2 -le 6 ']'
+ echo n141
++ expr 2 + 1
+ i=3
+ '[' 3 -le 6 ']'
+ echo n141
++ expr 3 + 1
+ i=4
+ '[' 4 -le 6 ']'
+ echo n141
++ expr 4 + 1
+ i=5
+ '[' 5 -le 6 ']'
+ echo n141
++ expr 5 + 1
+ i=6
+ '[' 6 -le 6 ']'
+ echo n141
++ expr 6 + 1
+ i=7
+ '[' 7 -le 6 ']'
+ read line
++ echo n155 7 wp10@n155 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n155
++ echo n155 7 wp10@n155 UNDEFINED
++ cut -f2 '-d '
+ nslots=7
+ i=1
+ '[' 1 -le 7 ']'
+ echo n155
++ expr 1 + 1
+ i=2
+ '[' 2 -le 7 ']'
+ echo n155
++ expr 2 + 1
+ i=3
+ '[' 3 -le 7 ']'
+ echo n155
++ expr 3 + 1
+ i=4
+ '[' 4 -le 7 ']'
+ echo n155
++ expr 4 + 1
+ i=5
+ '[' 5 -le 7 ']'
+ echo n155
++ expr 5 + 1
+ i=6
+ '[' 6 -le 7 ']'
+ echo n155
++ expr 6 + 1
+ i=7
+ '[' 7 -le 7 ']'
+ echo n155
++ expr 7 + 1
+ i=8
+ '[' 8 -le 7 ']'
+ read line
++ echo n146 5 wp10@n146 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n146
++ echo n146 5 wp10@n146 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n146
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n146
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n146
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n146
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n146
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n145 11 wp10@n145 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n145
++ echo n145 11 wp10@n145 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n145
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n145
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n145
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n145
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n145
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n145
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n145
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n145
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n145
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n145
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n145
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n165
++ echo n165 12 wp10@n165 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n165
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n165
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n165
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n165
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n165
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n165
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n165
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n165
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n165
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n165
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n165
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n165
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n151 3 wp10@n151 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n151
++ echo n151 3 wp10@n151 UNDEFINED
++ cut -f2 '-d '
+ nslots=3
+ i=1
+ '[' 1 -le 3 ']'
+ echo n151
++ expr 1 + 1
+ i=2
+ '[' 2 -le 3 ']'
+ echo n151
++ expr 2 + 1
+ i=3
+ '[' 3 -le 3 ']'
+ echo n151
++ expr 3 + 1
+ i=4
+ '[' 4 -le 3 ']'
+ read line
++ echo n156 12 wp10@n156 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n156
++ echo n156 12 wp10@n156 UNDEFINED
++ cut -f2 '-d '
+ nslots=12
+ i=1
+ '[' 1 -le 12 ']'
+ echo n156
++ expr 1 + 1
+ i=2
+ '[' 2 -le 12 ']'
+ echo n156
++ expr 2 + 1
+ i=3
+ '[' 3 -le 12 ']'
+ echo n156
++ expr 3 + 1
+ i=4
+ '[' 4 -le 12 ']'
+ echo n156
++ expr 4 + 1
+ i=5
+ '[' 5 -le 12 ']'
+ echo n156
++ expr 5 + 1
+ i=6
+ '[' 6 -le 12 ']'
+ echo n156
++ expr 6 + 1
+ i=7
+ '[' 7 -le 12 ']'
+ echo n156
++ expr 7 + 1
+ i=8
+ '[' 8 -le 12 ']'
+ echo n156
++ expr 8 + 1
+ i=9
+ '[' 9 -le 12 ']'
+ echo n156
++ expr 9 + 1
+ i=10
+ '[' 10 -le 12 ']'
+ echo n156
++ expr 10 + 1
+ i=11
+ '[' 11 -le 12 ']'
+ echo n156
++ expr 11 + 1
+ i=12
+ '[' 12 -le 12 ']'
+ echo n156
++ expr 12 + 1
+ i=13
+ '[' 13 -le 12 ']'
+ read line
++ echo n144 5 wp10@n144 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n144
++ echo n144 5 wp10@n144 UNDEFINED
++ cut -f2 '-d '
+ nslots=5
+ i=1
+ '[' 1 -le 5 ']'
+ echo n144
++ expr 1 + 1
+ i=2
+ '[' 2 -le 5 ']'
+ echo n144
++ expr 2 + 1
+ i=3
+ '[' 3 -le 5 ']'
+ echo n144
++ expr 3 + 1
+ i=4
+ '[' 4 -le 5 ']'
+ echo n144
++ expr 4 + 1
+ i=5
+ '[' 5 -le 5 ']'
+ echo n144
++ expr 5 + 1
+ i=6
+ '[' 6 -le 5 ']'
+ read line
++ echo n150 11 wp10@n150 UNDEFINED
++ cut -f1 '-d '
++ cut -f1 -d.
+ host=n150
++ echo n150 11 wp10@n150 UNDEFINED
++ cut -f2 '-d '
+ nslots=11
+ i=1
+ '[' 1 -le 11 ']'
+ echo n150
++ expr 1 + 1
+ i=2
+ '[' 2 -le 11 ']'
+ echo n150
++ expr 2 + 1
+ i=3
+ '[' 3 -le 11 ']'
+ echo n150
++ expr 3 + 1
+ i=4
+ '[' 4 -le 11 ']'
+ echo n150
++ expr 4 + 1
+ i=5
+ '[' 5 -le 11 ']'
+ echo n150
++ expr 5 + 1
+ i=6
+ '[' 6 -le 11 ']'
+ echo n150
++ expr 6 + 1
+ i=7
+ '[' 7 -le 11 ']'
+ echo n150
++ expr 7 + 1
+ i=8
+ '[' 8 -le 11 ']'
+ echo n150
++ expr 8 + 1
+ i=9
+ '[' 9 -le 11 ']'
+ echo n150
++ expr 9 + 1
+ i=10
+ '[' 10 -le 11 ']'
+ echo n150
++ expr 10 + 1
+ i=11
+ '[' 11 -le 11 ']'
+ echo n150
++ expr 11 + 1
+ i=12
+ '[' 12 -le 11 ']'
+ read line
+ cat /tmp/7856.1.wp10/machines
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n142
n141
n141
n141
n141
n141
n141
n155
n155
n155
n155
n155
n155
n155
n146
n146
n146
n146
n146
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n145
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n165
n151
n151
n151
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n156
n144
n144
n144
n144
n144
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
n150
+ '[' 1 = 1 ']'
+ rsh_wrapper=/opt/sge6/mpich2_smpd/rsh
+ '[' '!' -x /opt/sge6/mpich2_smpd/rsh ']'
+ rshcmd=rsh
+ case "$ARC" in
+ ln -s /opt/sge6/mpich2_smpd/rsh /tmp/7856.1.wp10/rsh
+ hash -r
+ '[' 0 = 1 ']'
+ port=22856
++ cat /tmp/7856.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n142 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n141 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n155 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n146 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n145 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n165 /opt/mpich2/intel/11.1//bin/smpd 22856
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n151 /opt/mpich2/intel/11.1//bin/smpd 22856
rsh n142 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
rsh n141 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n156 /opt/mpich2/intel/11.1//bin/smpd 22856
rsh n146 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
rsh n145 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n144 /opt/mpich2/intel/11.1//bin/smpd 22856
rsh n155 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
rsh n165 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
+ for host in '`cat $machines | uniq`'
+ /opt/sge6/mpich2_smpd/bin/lx26-amd64/start_mpich2 -n n150 /opt/mpich2/intel/11.1//bin/smpd 22856
+ actual_retry=1
+ '[' 1 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (1 of 10)'
startmpich2.sh: check for smpd daemons (1 of 10)
+ numhosts=0
++ cat /tmp/7856.1.wp10/machines
++ uniq
rsh n151 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
rsh n156 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n142
++ head -n 1
++ cut -f 1 -d ' '
rsh n144 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
rsh n150 /opt/mpich2/intel/11.1//bin/smpd -port 22856 -d 0
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n142'
startmpich2.sh: missing smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n141
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n141'
startmpich2.sh: missing smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n155
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n155'
startmpich2.sh: missing smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n146
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n146'
startmpich2.sh: missing smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n145
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n145'
startmpich2.sh: missing smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n165
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n165'
startmpich2.sh: missing smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n151
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n151'
startmpich2.sh: missing smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n156
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n156'
startmpich2.sh: missing smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n144
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n144'
startmpich2.sh: missing smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n150
++ head -n 1
++ cut -f 1 -d ' '
Assertion failed in file smpd_sock_wait.i at line 337: pollinfo->state == SMPDU_SOCKI_STATE_DISCONNECTED && pollfd->fd == -1
internal ABORT - process 0
+ '[' '' == smpd ']'
+ echo 'startmpich2.sh: missing smpd on n150'
startmpich2.sh: missing smpd on n150
+ '[' 10 -eq 0 ']'
+ sleep 5
+ (( actual_retry += 1 ))
+ '[' 2 -le 10 ']'
+ echo 'startmpich2.sh: check for smpd daemons (2 of 10)'
startmpich2.sh: check for smpd daemons (2 of 10)
+ numhosts=0
++ cat /tmp/7856.1.wp10/machines
++ uniq
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n142
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n142'
startmpich2.sh: found running smpd on n142
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n141
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n141'
startmpich2.sh: found running smpd on n141
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n155
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n155'
startmpich2.sh: found running smpd on n155
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n146
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n146'
startmpich2.sh: found running smpd on n146
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n145
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n145'
startmpich2.sh: found running smpd on n145
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n165
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n165'
startmpich2.sh: found running smpd on n165
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n151
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n151'
startmpich2.sh: found running smpd on n151
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n156
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n156'
startmpich2.sh: found running smpd on n156
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n144
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n144'
startmpich2.sh: found running smpd on n144
+ for host in '`cat $machines | uniq`'
++ /opt/mpich2/intel/11.1//bin/smpd -port 22856 -status n150
++ head -n 1
++ cut -f 1 -d ' '
+ '[' smpd == smpd ']'
+ (( numhosts += 1 ))
+ echo 'startmpich2.sh: found running smpd on n150'
startmpich2.sh: found running smpd on n150
+ '[' 10 -eq 10 ']'
+ echo 'startmpich2.sh: got all 10 of 10 nodes'
startmpich2.sh: got all 10 of 10 nodes
+ exit 0
rm: cannot remove `dmft?.error*': No such file or directory
rm: cannot remove `*.outputdmf?.*': No such file or directory
START
END
Mon Jun 13 23:53:40 EDT 2011
-catch_rsh /opt/mpich2/intel/11.1/
Your job finished at Mon Jun 13 23:53:45 EDT 2011