/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/* vim: set ts=8 sts=4 sw=4 noexpandtab : */
/*
 *
 */



#include "mpid_nem_impl.h"
#include "llc_impl.h"

//#define MPID_NEM_LLC_DEBUG_POLL
#ifdef MPID_NEM_LLC_DEBUG_POLL
#define dprintf printf
#else
#define dprintf(...)
#endif

/* function prototypes */

static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid);
static void MPID_nem_llc_recv_handler(void *vp_vc, uint64_t raddr, void *buf, size_t bsz);

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_poll
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPID_nem_llc_poll(int in_blocking_progress)
{
    int mpi_errno = MPI_SUCCESS;
    MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_POLL);

    MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_POLL);

    {
        int rc;
        rc = llc_poll(in_blocking_progress, MPID_nem_llc_send_handler, MPID_nem_llc_recv_handler);
        if (rc != 0) {
            mpi_errno = MPI_ERR_OTHER;
            MPIR_ERR_POP(mpi_errno);
        }
    }

  fn_exit:
    MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_POLL);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_send_handler
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid)
{
    /* int mpi_errno = 0; */
    MPIR_Request *sreq = cba;   /* from llc_writev(,,,,cbarg,) */
    MPIR_Request_kind_t kind;
    /* MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */

    /* MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */

    MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "llc_send_handler");

    MPIR_Assert(sreq != NULL);

    if (sreq == (void *) 0xdeadbeefUL) {
        MPIDI_VC_t *vc = (void *) p_reqid[0];
        MPID_nem_llc_vc_area *vc_llc;

        MPIR_Assert(vc != NULL);
        /* printf("from credit %p (pg_rank %d)\n", vc, vc->pg_rank); */

        vc_llc = VC_LLC(vc);
        MPID_nem_llc_send_queued(vc, &vc_llc->send_queue);

        p_reqid[0] = !MPIDI_CH3I_Sendq_empty(vc_llc->send_queue);
        MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "llc_send_handler");
        MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "send queue %d", (unsigned int) p_reqid[0]);

        goto fn_exit;
    }

    kind = sreq->kind;
    switch (kind) {
        unsigned int reqtype;
    case MPIR_REQUEST_KIND__SEND:
    case MPIR_REQUEST_KIND__PREQUEST_SEND:{
            reqtype = MPIDI_Request_get_type(sreq);

            /* Free temporal buffer for non-contiguous data.
             * MPIDI_Request_create_sreq (in mpid_isend.c) sets req->dev.datatype.
             * A control message has a req_type of MPIDI_REQUEST_TYPE_RECV and
             * msg_type of MPIDI_REQUEST_EAGER_MSG because
             * control message send follows
             * MPIDI_CH3_iStartMsg/v-->MPID_nem_llc_iStartContigMsg-->MPID_nem_llc_iSendContig
             * and MPID_nem_llc_iSendContig set req->dev.state to zero
             * because MPIR_Request_create
             * sets it to zero. In addition, eager-short message has req->comm of zero. */
#ifndef	notdef_leak_0001_hack
            /* See also MPIDI_CH3_Request_create and _destory() */
            /*     in src/mpid/ch3/src/ch3u_request.c */
#endif /* notdef_leak_0001_hack */
            if (reqtype != MPIDI_REQUEST_TYPE_RECV && sreq->comm) {
                /* Exclude control messages which have MPIDI_REQUEST_TYPE_RECV.
                 * Note that RMA messages should be included.
                 * Exclude eager-short by requiring req->comm != 0. */
                int is_contig;
                MPIR_Datatype_is_contig(sreq->dev.datatype, &is_contig);
                if (!is_contig && REQ_FIELD(sreq, pack_buf)) {
                    dprintf("llc_send_handler,non-contiguous,free pack_buf\n");
                    MPL_free(REQ_FIELD(sreq, pack_buf));
                }
            }

            if ((REQ_FIELD(sreq, rma_buf) != NULL && sreq->dev.datatype_ptr &&
                 sreq->dev.segment_size > 0)) {
                MPL_free(REQ_FIELD(sreq, rma_buf));    // allocated in MPID_nem_llc_SendNoncontig
                REQ_FIELD(sreq, rma_buf) = NULL;
            }

            /* sreq: src/mpid/ch3/include/mpidpre.h */
            {
                MPIDI_VC_t *vc;
                int (*reqFn) (MPIDI_VC_t * vc, MPIR_Request * sreq, int *complete);
                int complete;
                int r_mpi_errno;

                p_reqid[0] = 0 /* REQ_LLC(sreq)->woff */ ;

                vc = sreq->ch.vc;       /* before callback */
                reqFn = sreq->dev.OnDataAvail;
                if (reqFn == 0) {
                    MPIR_Assert(reqtype != MPIDI_REQUEST_TYPE_GET_RESP);

                    r_mpi_errno = MPID_Request_complete(sreq);
                    if (r_mpi_errno != MPI_SUCCESS) {
                        MPIR_ERR_POP(r_mpi_errno);
                    }
                    MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete");
                }
                else {
                    complete = 0;
                    r_mpi_errno = reqFn(vc, sreq, &complete);
                    if (r_mpi_errno)
                        MPIR_ERR_POP(r_mpi_errno);
                    if (complete == 0) {
                        MPIR_Assert(complete == TRUE);
                    }
                    MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, ".... complete2");
                }

                /* push queued messages */
                {
                    MPID_nem_llc_vc_area *vc_llc = VC_LLC(vc);

                    MPID_nem_llc_send_queued(vc, &vc_llc->send_queue);
                }
            }
            break;
        }
    default:
        printf("send_handler,unknown kind=%08x\n", sreq->kind);
        MPID_nem_llc_segv;
        break;
    }

  fn_exit:
    /* MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */
    return /* mpi_errno */ ;
  fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_recv_handler
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static void MPID_nem_llc_recv_handler(void *vp_vc, uint64_t raddr, void *buf, size_t bsz)
{
    int mpi_errno = 0;
    MPIDI_VC_t *vc = vp_vc;
    /* MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */

    /* MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */

    MPL_DBG_MSG(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "llc_recv_handler");

    {
        int pg_rank = (int) raddr;
        MPIDI_PG_t *pg = MPIDI_Process.my_pg;
        MPIDI_VC_t *vc_from_pg = 0;

        if ((pg != 0)
            && ((pg_rank >= 0) && (pg_rank < MPIDI_PG_Get_size(pg)))
) {
            /*
             * MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
             */
            MPIDI_PG_Get_vc_set_active(pg, pg_rank, &vc_from_pg);
        }
        else {
            MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "bad vc %p or", pg);
            MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "bad pg_rank %d", pg_rank);
            MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "bad pg_rank < %d", MPIDI_PG_Get_size(pg));
            vc_from_pg = vc;    /* XXX */
        }
        if (vc != vc_from_pg) {
            MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "bad vc for pg_rank %d", pg_rank);
        }
        if (vc == 0) {
            vc = vc_from_pg;
        }
    }
    if (vc != 0) {
        mpi_errno = MPID_nem_handle_pkt(vc, buf, bsz);
        if (mpi_errno != 0) {
            MPL_DBG_MSG_D(MPIDI_CH3_DBG_CHANNEL, VERBOSE, "MPID_nem_handle_pkt() = %d", mpi_errno);
        }
    }

  fn_exit:
    /* MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */
    return;
    //fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_recv_posted
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPID_nem_llc_recv_posted(struct MPIDI_VC *vc, struct MPIR_Request *req)
{
    int mpi_errno = MPI_SUCCESS, llc_errno;
    int dt_contig;
    intptr_t data_sz;
    MPIR_Datatype*dt_ptr;
    MPI_Aint dt_true_lb;
    int i;

    MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);
    MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);

    /* req->dev.datatype is set in MPID_irecv --> MPIDI_CH3U_Recvq_FDU_or_AEP */
    MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr,
                            dt_true_lb);

    /* Don't save VC because it's not used in llc_poll */

    /* Save data size for llc_poll */
    req->dev.recv_data_sz = data_sz;

#if 0   /* FIXME : vc is NULL when rank is MPI_ANY_SOURCE */
    dprintf
        ("llc_recv_posted,%d<-%d,vc=%p,req=%p,user_buf=%p,data_sz=%ld,datatype=%08x,dt_contig=%d\n",
         MPIDI_Process.my_pg_rank, vc->pg_rank, vc, req, req->dev.user_buf, req->dev.recv_data_sz,
         req->dev.datatype, dt_contig);
#endif

    void *write_to_buf;
    if (dt_contig) {
        write_to_buf = (void *) ((char *) req->dev.user_buf + dt_true_lb);
    }
    else {
        REQ_FIELD(req, pack_buf) = MPL_malloc(data_sz, MPL_MEM_BUFFER);
        MPIR_ERR_CHKANDJUMP(!REQ_FIELD(req, pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory");
        write_to_buf = REQ_FIELD(req, pack_buf);
    }

    int LLC_my_rank;
    LLC_comm_rank(LLC_COMM_MPICH, &LLC_my_rank);
    dprintf("llc_isend,LLC_my_rank=%d\n", LLC_my_rank);

#if 0   /* FIXME : vc is NULL when rank is MPI_ANY_SOURCE */
    dprintf("llc_recv_posted,remote_endpoint_addr=%ld\n", VC_FIELD(vc, remote_endpoint_addr));
#endif

    LLC_cmd_t *cmd = LLC_cmd_alloc2(1, 1, 1);

    cmd[0].opcode = LLC_OPCODE_RECV;
    cmd[0].comm = LLC_COMM_MPICH;
    cmd[0].req_id = (uint64_t) cmd;

    if (req->dev.match.parts.rank == MPI_ANY_SOURCE) {
        cmd[0].rank = LLC_ANY_SOURCE;
        cmd[0].mask.rank = 0;
    }
    else {
        cmd[0].rank = VC_FIELD(vc, remote_endpoint_addr);
    }

    /* req->comm is set in MPID_irecv --> MPIDI_CH3U_Recvq_FDU_or_AEP */
    if (req->dev.match.parts.tag == MPI_ANY_TAG) {
        *(int32_t *) ((uint8_t *) & cmd[0].tag) = LLC_ANY_TAG;
        *(int32_t *) ((uint8_t *) & cmd[0].mask.tag) = 0;
    }
    else {
        *(int32_t *) ((uint8_t *) & cmd[0].tag) = req->dev.match.parts.tag;
    }

    *(MPIR_Context_id_t *) ((uint8_t *) & cmd[0].tag + sizeof(int32_t)) =
        req->dev.match.parts.context_id;
    MPIR_Assert(sizeof(LLC_tag_t) >= sizeof(int32_t) + sizeof(MPIR_Context_id_t));
    memset((uint8_t *) & cmd[0].tag + sizeof(int32_t) + sizeof(MPIR_Context_id_t),
           0, sizeof(LLC_tag_t) - sizeof(int32_t) - sizeof(MPIR_Context_id_t));


    dprintf("llc_recv_posted,tag=");
    for (i = 0; i < sizeof(LLC_tag_t); i++) {
        dprintf("%02x", (int) *((uint8_t *) & cmd[0].tag + i));
    }
    dprintf("\n");


    cmd[0].iov_local[0].addr = (uint64_t) write_to_buf;
    cmd[0].iov_local[0].length = data_sz;
    cmd[0].niov_local = 1;

    cmd[0].iov_remote[0].addr = 0;
    cmd[0].iov_remote[0].length = data_sz;;
    cmd[0].niov_remote = 1;

    ((struct llc_cmd_area *) cmd[0].usr_area)->cbarg = req;
    if (req->dev.match.parts.rank == MPI_ANY_SOURCE) {
        ((struct llc_cmd_area *) cmd[0].usr_area)->raddr = MPI_ANY_SOURCE;      /* FIXME : should 0 ? */
    }
    else {
        ((struct llc_cmd_area *) cmd[0].usr_area)->raddr = VC_FIELD(vc, remote_endpoint_addr);
    }

    REQ_FIELD(req, cmds) = cmd;

    llc_errno = LLC_post(cmd, 1);
    MPIR_ERR_CHKANDJUMP(llc_errno != LLC_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**LLC_post");

  fn_exit:
    MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);
    return mpi_errno;
  fn_fail:
    goto fn_exit;
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_anysource_posted
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
void MPID_nem_llc_anysource_posted(MPIR_Request * req)
{
    int mpi_errno = MPI_SUCCESS;
    MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);
    MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);

    mpi_errno = MPID_nem_llc_recv_posted(NULL, req);
    MPIR_Assert(mpi_errno == MPI_SUCCESS);

    MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);
}

#undef FUNCNAME
#define FUNCNAME MPID_nem_llc_anysource_matched
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPID_nem_llc_anysource_matched(MPIR_Request * req)
{
    int matched = FALSE;

    MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);
    MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);

    /* FIXME : How to call a cancel_recv function */
    /* If LLC_postedq is still having this request, delete it.
     * Ohterwise, return TURE */
    matched = LLC_req_approve_recv((LLC_cmd_t *) REQ_FIELD(req, cmds));

    MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);

    return matched;
}
