bbs.cpp

Go to the documentation of this file.

#include <../../nrnconf.h>
#include "nrnmpi.h"
#include "bbsconf.h"
#include <stdio.h>
#include <stdlib.h>
#include <InterViews/resource.h>
#include "oc2iv.h"
#include "bbs.h"
#include "bbslocal.h"
#if defined(NRNMPI)
#include "bbsdirect.h"
#include "bbsrcli.h"
#endif
 
#if defined(HAVE_TMS) && !NRNMPI
#include <time.h>
#include <sys/times.h>
#include <limits.h>
static struct tms tmsbuf, tms_start_;
static clock_t starttime;
#endif
 
extern int nrn_global_argc;
extern char** nrn_global_argv;
 
bool BBSImpl::is_master_ = false;
bool BBSImpl::started_ = false;
bool BBSImpl::done_ = false;
bool BBSImpl::master_works_ = true;
 
#undef debug
#define debug BBSImpl::debug_
 
int BBSImpl::debug_ = 0;
int BBSImpl::mytid_;
 
static int etaskcnt;
static double total_exec_time;
static double worker_take_time;
 
BBS::BBS() {
    init(-1);
}
 
BBS::BBS(int n) {
    init(n);
}
 
#if NRNMPI
void BBS::init(int) {
    if (nrnmpi_use == 0) {
        BBSImpl::is_master_ = true;
        impl_ = new BBSLocal();
        return;
    }
    if (!BBSImpl::started_) {
        BBSImpl::is_master_ = (nrnmpi_myid_bbs == 0) ? true : false;
        BBSImpl::master_works_ = true;
        // printf("%d BBS::init is_master=%d\n", nrnmpi_myid_bbs, BBSImpl::is_master_);
    }
    // Just as with PVM which stored buffers on the bulletin board
    // so we have the following files to store MPI_PACKED buffers
    // on the bulletin board. It would be possible to factor out
    // the pvm stuff and we may do that later but for now we
    // start with copies of the four files that worked for PVM
    // and convert to the nrnmpi functions implemented in
    // ../nrnmpi
    // The four files are
    // bbsclimpi.cpp - mpi remote client BBSClient from bbsrcli.cpp
    // bbsdirectmpi.cpp - mpi master client BBSDirect from bbsdirect.cpp
    // bbssrvmpi.cpp - mpi server portion to remote client from master bbs
    //      BBSDirectServer derived from bbssrv.cpp
    // bbslsrvmpi.cpp - mpi master bbs portion of BBSDirectServer
    //      derived from bbslsrv2.cpp
    // We reuse the .h files of these.
    if (BBSImpl::is_master_) {
        impl_ = new BBSDirect();
    } else {
        impl_ = new BBSClient();
    }
}
#else   // !NRNMPI
void BBS::init(int) {
    if (!BBSImpl::started_) {
        BBSImpl::is_master_ = true;
        BBSImpl::master_works_ = true;
    }
    impl_ = new BBSLocal();
}
#endif  // !NRNMPI
 
BBSImpl::BBSImpl() {
    runworker_called_ = 0;
    wait_time_ = 0.;
    send_time_ = 0.;
    integ_time_ = 0.;
    working_id_ = 0;
    n_ = 0;
    pickle_ret_ = 0;
    pickle_ret_size_ = 0;
}
 
BBS::~BBS() {
    delete impl_;
}
 
BBSImpl::~BBSImpl() {
    if (pickle_ret_) {
        delete[] pickle_ret_;
    }
}
 
bool BBS::is_master() {
    return BBSImpl::is_master_;
}
 
int BBS::nhost() {
    return nrnmpi_numprocs;
}
 
int BBS::myid() {
    return nrnmpi_myid;
}
 
bool BBSImpl::is_master() {
    return is_master_;
}
 
double BBS::time() {
    return impl_->time();
}
 
double BBSImpl::time() {
#if NRNMPI
    return nrnmpi_wtime();
#else
#ifdef HAVE_TMS
    return double(times(&tmsbuf)) / 100.;
#else
    return 0.;
#endif
#endif
}
 
double BBS::wait_time() {
    return impl_->wait_time_;
}
double BBS::integ_time() {
    return impl_->integ_time_;
}
double BBS::send_time() {
    return impl_->send_time_;
}
void BBS::add_wait_time(double st) {
    impl_->wait_time_ += impl_->time() - st;
}
 
void BBS::perror(const char* s) {
    impl_->perror(s);
}
 
void BBSImpl::perror(const char*) {}
 
int BBS::upkint() {
    int i = impl_->upkint();
    if (debug) {
        printf("upkint %d\n", i);
    }
    return i;
}
 
double BBS::upkdouble() {
    double d = impl_->upkdouble();
    if (debug) {
        printf("upkdouble %g\n", d);
    }
    return d;
}
 
void BBS::upkvec(int n, double* px) {
    impl_->upkvec(n, px);
    if (debug) {
        printf("upkvec %d\n", n);
    }
}
 
char* BBS::upkstr() {
    char* s = impl_->upkstr();
    if (debug) {
        printf("upkstr |%s|\n", s);
    }
    return s;
}
 
char* BBS::upkpickle(size_t* n) {
    char* s = impl_->upkpickle(n);
    if (debug) {
        printf("upkpickle %lu |%s|\n", *n, s);
    }
    return s;
}
 
void BBS::pkbegin() {
    if (debug) {
        printf("pkbegin\n");
    }
    impl_->pkbegin();
}
 
void BBS::pkint(int i) {
    if (debug) {
        printf("pkint %d\n", i);
    }
    impl_->pkint(i);
}
 
void BBS::pkdouble(double x) {
    if (debug) {
        printf("pkdouble %g\n", x);
    }
    impl_->pkdouble(x);
}
 
void BBS::pkvec(int n, double* px) {
    if (debug) {
        printf("pkdouble %d\n", n);
    }
    impl_->pkvec(n, px);
}
 
void BBS::pkstr(const char* s) {
    if (debug) {
        printf("pkstr |%s|\n", s);
    }
    impl_->pkstr(s);
}
 
void BBS::pkpickle(const char* s, size_t n) {
    if (debug) {
        printf("pkpickle %lu |%s|\n", n, s);
    }
    impl_->pkpickle(s, n);
}
 
#if 0
// for now all todo messages are of the three item form
// tid
// gid
// id
// "stmt"
// the latter should set hoc_ac_ if the return value is important
// right now every arg must be literal, we'll handle variables later.
// eg the following should work.
// n = 1000  x = 0  for i=1,n { x += i }  hoc_ac_ = x
// although it may makes sense to send the result directly to the
// tid for now we just put it back onto the mailbox in the form
// message: "result tid gid" with two items, i.e. id, hoc_ac_
// Modified 11/30/09. To allow a return of a (pickled) PythonObject
// for the case when the execution is for a Python Callable the return
// message result now has three items, i.e. id, rtype, hoc_ac or pickled
// PyObject. rtype = 0 means hoc_ac, rtype = 1 means pickled PyObject.
// execute_helper returns the pickle string or (char*)0.
#endif
#if 0
// the todo management has been considerably modified to support
// a priority queue style for the order in which tasks are executed.
// Now the server manages the task id. Given a task id, it knows the
// parent task id.
// The working_id is always non-trivial on worker machines, and is
// only 0 on the master when dealing with a submission at the top level
// post_todo(working_id_) message is the statement, here the working_id is
//  the parent of the future stmt task.
// take_todo: message is the statement. return is the id of this task
// post_result(id) message is return value.
// the result will be retrieved relative to the submitting working_id_
// look_take_result(working_id_) message is the return value. the return
// value of this call is the id of the task that computed the return.
#endif
 
// BBSImpl::execute_helper() in ocbbs.cpp
 
void BBSImpl::execute(int id) {  // assumes a "_todo" message in receive buffer
    ++etaskcnt;
    double st, et;
    int userid;
    char* rs;
    char* s;
    size_t n;
    int i;
    int save_id = working_id_;
    int save_n = n_;
    working_id_ = id;
    n_ = 0;
    st = time();
    if (debug_) {
        printf("execute begin %g: working_id_=%d\n", st, working_id_);
    }
    userid = upkint();
    int wid = upkint();
    hoc_ac_ = double(id);
    rs = execute_helper(&n, id);  // builds and execute hoc statement
    et = time() - st;
    total_exec_time += et;
    if (debug) {
        printf("execute end elapsed %g: working_id_=%d hoc_ac_=%g\n", et, working_id_, hoc_ac_);
    }
    pkbegin();
    pkint(userid);
    pkint(wid);
    pkint(rs ? 1 : 0);
    if (!rs) {
        pkdouble(hoc_ac_);
    } else {
        pkpickle(rs, n);
        delete[] rs;
    }
    working_id_ = save_id;
    n_ = save_n;
    post_result(id);
}
 
int BBS::submit(int userid) {
    return impl_->submit(userid);
}
 
int BBSImpl::submit(int userid) {
    // userid was the first item packed
    ++n_;
    if (debug) {
        printf("submit n_= %d for working_id=%d userid=%d\n", n_, working_id_, userid);
    }
    if (userid < 0) {
        save_args(userid);
    } else {
        post_todo(working_id_);
    }
    return userid;
}
 
void BBS::context() {
    impl_->context();
}
 
void BBSImpl::context() {
    printf("can't execute BBS::context on a worker\n");
    exit(1);
}
 
bool BBS::working(int& id, double& x, int& userid) {
    return impl_->working(id, x, userid);
}
 
void BBS::master_works(int flag) {
    if (impl_->is_master() && nrnmpi_numprocs_bbs > 1) {
        impl_->master_works_ = flag ? true : false;
    }
}
 
int BBSImpl::master_take_result(int pid) {
    assert(0);
    return 0;
}
 
bool BBSImpl::working(int& id, double& x, int& userid) {
    int cnt = 0;
    int rtype;
    double t;
    if (n_ <= 0) {
        if (debug) {
            printf("working n_=%d: return false\n", n_);
        }
        return false;
    }
    if (debug) {
        t = time();
    }
    for (;;) {
        ++cnt;
        if (master_works_) {
            id = look_take_result(working_id_);
        } else {
            id = master_take_result(working_id_);
        }
        if (id != 0) {
            userid = upkint();
            int wid = upkint();
            rtype = upkint();
            if (rtype == 0) {
                x = upkdouble();
            } else {
                assert(rtype == 1);
                x = 0.0;
                if (pickle_ret_) {
                    delete[] pickle_ret_;
                }
                pickle_ret_ = upkpickle(&pickle_ret_size_);
            }
            --n_;
            if (debug) {
                printf("working n_=%d: after %d try elapsed %g sec got result for %d id=%d x=%g\n",
                       n_,
                       cnt,
                       time() - t,
                       working_id_,
                       id,
                       x);
            }
            if (userid < 0) {
                return_args(userid);
            }
            return true;
        } else if ((id = look_take_todo()) != 0) {
            if (debug) {
                printf("working: no result for %d but did get _todo id=%d\n", working_id_, id);
            }
            execute(id);
        }
    }
};
 
void BBS::worker() {
    impl_->runworker_called_ = 1;
    impl_->worker();
}
 
void BBSImpl::worker() {
    // forever request and execute commands
    double st, et;
    int id;
    if (!is_master()) {
        if (nrnmpi_myid_bbs == -1) {  // wait for message from
            for (;;) {                // the proper nrnmpi_myid == 0
                subworld_worker_execute();
            }
        }
        for (;;) {
            st = time();
            id = take_todo();
            et = time() - st;
            worker_take_time += et;
            execute(id);
        }
    }
}
 
void BBS::post(const char* key) {
    if (debug) {
        printf("post: |%s|\n", key);
    }
    impl_->post(key);
}
 
bool BBS::look_take(const char* key) {
    bool b = impl_->look_take(key);
    if (debug) {
        printf("look_take |%s| return %d\n", key, b);
    }
    return b;
}
 
bool BBS::look(const char* key) {
    bool b = impl_->look(key);
    if (debug) {
        printf("look |%s| return %d\n", key, b);
    }
    return b;
}
 
void BBS::take(const char* key) {  // blocking
    double t;
    if (debug) {
        t = time();
        printf("begin take |%s| at %g\n", key, t);
    }
    impl_->take(key);
    if (debug) {
        printf("end take |%s| elapsed %g from %g\n", key, time() - t, t);
    }
}
 
void BBS::done() {
    if (impl_->runworker_called_) {
        impl_->done();
    }
}
 
void BBSImpl::done() {
    if (done_) {
        return;
    }
    done_ = true;
#ifdef HAVE_TMS
    clock_t elapsed = times(&tmsbuf) - starttime;
    printf("%d tasks in %g seconds. %g seconds waiting for tasks\n",
           etaskcnt,
           total_exec_time,
           worker_take_time);
    printf("user=%g sys=%g elapsed=%g %g%%\n",
           (double) (tmsbuf.tms_utime - tms_start_.tms_utime) / 100,
           (double) (tmsbuf.tms_stime - tms_start_.tms_stime) / 100,
           (double) (elapsed) / 100,
           100. * (double) (tmsbuf.tms_utime - tms_start_.tms_utime) / (double) elapsed);
#endif
}
 
void BBSImpl::start() {
    if (started_) {
        return;
    }
    started_ = 1;
#ifdef HAVE_TMS
    starttime = times(&tms_start_);
#endif
}