lib/nvmf/vfio_user.c
/*
 * Creates a completion or submission I/O queue. Returns 0 on success, -errno
 * on error.
 */
Creates a completion or submission I/O queue. Returns 0 on success, -errno
on error.
Creates a completion or submission I / O queue . Returns 0 on success - errno on error .
static int
handle_create_io_q(struct nvmf_vfio_user_ctrlr *ctrlr,
		   struct spdk_nvme_cmd *cmd, const bool is_cq)
{
	uint16_t qid;
	uint32_t qsize;
	uint16_t sc = SPDK_NVME_SC_SUCCESS;
	uint16_t sct = SPDK_NVME_SCT_GENERIC;
	int err = 0;
	struct nvmf_vfio_user_qpair *vu_qpair;
	struct nvme_q *io_q;
	assert(ctrlr != NULL);
	assert(cmd != NULL);
	qid = cmd->cdw10_bits.create_io_q.qid;
	if (qid == 0 || qid >= NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR) {
		SPDK_ERRLOG("%s: invalid QID=%d, max=%d\n", ctrlr_id(ctrlr),
			    qid, NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR);
		sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
		sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;
		goto out;
	}
	if (io_q_exists(ctrlr, qid, is_cq)) {
		SPDK_ERRLOG("%s: %cQ%d already exists\n", ctrlr_id(ctrlr),
			    is_cq ? 'C' : 'S', qid);
		sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
		sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;
		goto out;
	}
	qsize = cmd->cdw10_bits.create_io_q.qsize + 1;
	if (qsize == 1 || qsize > max_queue_size(ctrlr)) {
		SPDK_ERRLOG("%s: invalid I/O queue size %u\n", ctrlr_id(ctrlr), qsize);
		sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
		sc = SPDK_NVME_SC_INVALID_QUEUE_SIZE;
		goto out;
	}
	SPDK_DEBUGLOG(nvmf_vfio,
		      "%s: create I/O %cQ%d: QSIZE=%#x\n", ctrlr_id(ctrlr),
		      is_cq ? 'C' : 'S', qid, qsize);
	if (is_cq) {
		err = init_qp(ctrlr, ctrlr->qp[0]->qpair.transport, qsize, qid);
		if (err != 0) {
			sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
			goto out;
		}
		io_q = &ctrlr->qp[qid]->cq;
		if (cmd->cdw11_bits.create_io_cq.pc != 0x1) {
			SPDK_ERRLOG("%s: non-PC CQ not supporred\n", ctrlr_id(ctrlr));
			sc = SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF;
			goto out;
		}
		io_q->ien = cmd->cdw11_bits.create_io_cq.ien;
		io_q->iv = cmd->cdw11_bits.create_io_cq.iv;
		io_q->phase = true;
	} else {
		if (cmd->cdw11_bits.create_io_sq.cqid == 0) {
			SPDK_ERRLOG("%s: invalid CQID 0\n", ctrlr_id(ctrlr));
			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
			sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;
			goto out;
		}
		if (!io_q_exists(ctrlr, cmd->cdw11_bits.create_io_sq.cqid, true)) {
			SPDK_ERRLOG("%s: CQ%d does not exist\n", ctrlr_id(ctrlr),
				    cmd->cdw11_bits.create_io_sq.cqid);
			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
			sc = SPDK_NVME_SC_COMPLETION_QUEUE_INVALID;
			goto out;
		}
		if (cmd->cdw11_bits.create_io_sq.pc != 0x1) {
			SPDK_ERRLOG("%s: non-PC SQ not supported\n", ctrlr_id(ctrlr));
			sc = SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF;
			goto out;
		}
		if (qid != cmd->cdw11_bits.create_io_sq.cqid) {
			SPDK_ERRLOG("%s: doesn't support shared CQ now\n", ctrlr_id(ctrlr));
			sct = SPDK_NVME_SCT_COMMAND_SPECIFIC;
			sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER;
		}
		io_q = &ctrlr->qp[qid]->sq;
		io_q->cqid = cmd->cdw11_bits.create_io_sq.cqid;
		SPDK_DEBUGLOG(nvmf_vfio, "%s: SQ%d CQID=%d\n", ctrlr_id(ctrlr),
			      qid, io_q->cqid);
	}
	io_q->is_cq = is_cq;
	io_q->size = qsize;
	io_q->prp1 = cmd->dptr.prp.prp1;
	err = map_q(ctrlr, io_q, is_cq, true);
	if (err) {
		sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR;
		SPDK_ERRLOG("%s: failed to map I/O queue: %m\n", ctrlr_id(ctrlr));
		goto out;
	}
	SPDK_DEBUGLOG(nvmf_vfio, "%s: mapped %cQ%d IOVA=%#lx vaddr=%#llx\n",
		      ctrlr_id(ctrlr), is_cq ? 'C' : 'S',
		      qid, cmd->dptr.prp.prp1, (unsigned long long)io_q->addr);
	if (is_cq) {
		*hdbl(ctrlr, io_q) = 0;
	} else {
		vu_qpair = ctrlr->qp[qid];
		*tdbl(ctrlr, io_q) = 0;
		vu_qpair->sq.head = 0;
		if (vu_qpair->state == VFIO_USER_QPAIR_SQ_DELETED) {
			vu_qpair->state = VFIO_USER_QPAIR_ACTIVE;
		} else {
			vu_qpair->create_io_sq_cmd = *cmd;
			spdk_nvmf_tgt_new_qpair(ctrlr->transport->transport.tgt,
						&vu_qpair->qpair);
			return 0;
		}
	}
out:
	return post_completion(ctrlr, &ctrlr->qp[0]->cq, 0, 0, cmd->cid, sc, sct);
}
static int handle_create_io_q ( struct nvmf_vfio_user_ctrlr * ctrlr , struct spdk_nvme_cmd * cmd , const bool is_cq ) { uint16_t qid ; uint32_t qsize ; uint16_t sc = SPDK_NVME_SC_SUCCESS ; uint16_t sct = SPDK_NVME_SCT_GENERIC ; int err = 0 ; struct nvmf_vfio_user_qpair * vu_qpair ; struct nvme_q * io_q ; assert ( ctrlr != NULL ) ; assert ( cmd != NULL ) ; qid = cmd -> cdw10_bits . create_io_q . qid ; if ( qid == 0 || qid >= NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) , qid , NVMF_VFIO_USER_DEFAULT_MAX_QPAIRS_PER_CTRLR ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER ; goto out ; } if ( io_q_exists ( ctrlr , qid , is_cq ) ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) , is_cq ? ' ' : ' ' , qid ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER ; goto out ; } qsize = cmd -> cdw10_bits . create_io_q . qsize + 1 ; if ( qsize == 1 || qsize > max_queue_size ( ctrlr ) ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) , qsize ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_INVALID_QUEUE_SIZE ; goto out ; } SPDK_DEBUGLOG ( nvmf_vfio , " \n " , ctrlr_id ( ctrlr ) , is_cq ? ' ' : ' ' , qid , qsize ) ; if ( is_cq ) { err = init_qp ( ctrlr , ctrlr -> qp [ 0 ] -> qpair . transport , qsize , qid ) ; if ( err != 0 ) { sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR ; goto out ; } io_q = & ctrlr -> qp [ qid ] -> cq ; if ( cmd -> cdw11_bits . create_io_cq . pc != 0x1 ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) ) ; sc = SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF ; goto out ; } io_q -> ien = cmd -> cdw11_bits . create_io_cq . ien ; io_q -> iv = cmd -> cdw11_bits . create_io_cq . iv ; io_q -> phase = true ; } else { if ( cmd -> cdw11_bits . create_io_sq . cqid == 0 ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER ; goto out ; } if ( ! io_q_exists ( ctrlr , cmd -> cdw11_bits . create_io_sq . cqid , true ) ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) , cmd -> cdw11_bits . create_io_sq . cqid ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_COMPLETION_QUEUE_INVALID ; goto out ; } if ( cmd -> cdw11_bits . create_io_sq . pc != 0x1 ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) ) ; sc = SPDK_NVME_SC_INVALID_CONTROLLER_MEM_BUF ; goto out ; } if ( qid != cmd -> cdw11_bits . create_io_sq . cqid ) { SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) ) ; sct = SPDK_NVME_SCT_COMMAND_SPECIFIC ; sc = SPDK_NVME_SC_INVALID_QUEUE_IDENTIFIER ; } io_q = & ctrlr -> qp [ qid ] -> sq ; io_q -> cqid = cmd -> cdw11_bits . create_io_sq . cqid ; SPDK_DEBUGLOG ( nvmf_vfio , " \n " , ctrlr_id ( ctrlr ) , qid , io_q -> cqid ) ; } io_q -> is_cq = is_cq ; io_q -> size = qsize ; io_q -> prp1 = cmd -> dptr . prp . prp1 ; err = map_q ( ctrlr , io_q , is_cq , true ) ; if ( err ) { sc = SPDK_NVME_SC_INTERNAL_DEVICE_ERROR ; SPDK_ERRLOG ( " \n " , ctrlr_id ( ctrlr ) ) ; goto out ; } SPDK_DEBUGLOG ( nvmf_vfio , " \n " , ctrlr_id ( ctrlr ) , is_cq ? ' ' : ' ' , qid , cmd -> dptr . prp . prp1 , ( unsigned long long ) io_q -> addr ) ; if ( is_cq ) { * hdbl ( ctrlr , io_q ) = 0 ; } else { vu_qpair = ctrlr -> qp [ qid ] ; * tdbl ( ctrlr , io_q ) = 0 ; vu_qpair -> sq . head = 0 ; if ( vu_qpair -> state == VFIO_USER_QPAIR_SQ_DELETED ) { vu_qpair -> state = VFIO_USER_QPAIR_ACTIVE ; } else { vu_qpair -> create_io_sq_cmd = * cmd ; spdk_nvmf_tgt_new_qpair ( ctrlr -> transport -> transport . tgt , & vu_qpair -> qpair ) ; return 0 ; } } out : return post_completion ( ctrlr , & ctrlr -> qp [ 0 ] -> cq , 0 , 0 , cmd -> cid , sc , sct ) ; }
Creates a completion or submission I/O queue.
Creates a completion or submission I / O queue .
/* CQ must be created before SQ */ /* TODO: support shared IO CQ */ /*
			 * Create our new I/O qpair. This asynchronously invokes, on a
			 * suitable poll group, the nvmf_vfio_user_poll_group_add()
			 * callback, which will call spdk_nvmf_request_exec_fabrics()
			 * with a generated fabrics connect command. This command is
			 * then eventually completed via handle_queue_connect_rsp().
			 */