[mirror_ubuntu-jammy-kernel.git] / arch / tile / include / gxio / dma_queue.h

/*
 * Copyright 2012 Tilera Corporation. All Rights Reserved.
 *
 *   This program is free software; you can redistribute it and/or
 *   modify it under the terms of the GNU General Public License
 *   as published by the Free Software Foundation, version 2.
 *
 *   This program is distributed in the hope that it will be useful, but
 *   WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
 *   NON INFRINGEMENT.  See the GNU General Public License for
 *   more details.
 */

#ifndef _GXIO_DMA_QUEUE_H_
#define _GXIO_DMA_QUEUE_H_

/*
 * DMA queue management APIs shared between TRIO and mPIPE.
 */

#include <gxio/common.h>

/* The credit counter lives in the high 32 bits. */
#define DMA_QUEUE_CREDIT_SHIFT 32

/*
 * State object that tracks a DMA queue's head and tail indices, as
 * well as the number of commands posted and completed.  The
 * structure is accessed via a thread-safe, lock-free algorithm.
 */
typedef struct {
	/*
	 * Address of a MPIPE_EDMA_POST_REGION_VAL_t,
	 * TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t
	 * register.  These register have identical encodings and provide
	 * information about how many commands have been processed.
	 */
	void *post_region_addr;

	/*
	 * A lazily-updated count of how many edescs the hardware has
	 * completed.
	 */
	uint64_t hw_complete_count __attribute__ ((aligned(64)));

	/*
	 * High 32 bits are a count of available egress command credits,
	 * low 24 bits are the next egress "slot".
	 */
	int64_t credits_and_next_index;

} __gxio_dma_queue_t;

/* Initialize a dma queue. */
extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
				  void *post_region_addr,
				  unsigned int num_entries);

/*
 * Update the "credits_and_next_index" and "hw_complete_count" fields
 * based on pending hardware completions.  Note that some other thread
 * may have already done this and, importantly, may still be in the
 * process of updating "credits_and_next_index".
 */
extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue);

/* Wait for credits to become available. */
extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
						 int64_t modifier);

/* Reserve slots in the queue, optionally waiting for slots to become
 * available, and optionally returning a "completion_slot" suitable for
 * direct comparison to "hw_complete_count".
 */
static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue,
					       unsigned int num, bool wait,
					       bool completion)
{
	uint64_t slot;

	/*
	 * Try to reserve 'num' egress command slots.  We do this by
	 * constructing a constant that subtracts N credits and adds N to
	 * the index, and using fetchaddgez to only apply it if the credits
	 * count doesn't go negative.
	 */
	int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) | num;
	int64_t old =
		__insn_fetchaddgez(&dma_queue->credits_and_next_index,
				   modifier);

	if (unlikely(old + modifier < 0)) {
		/*
		 * We're out of credits.  Try once to get more by checking for
		 * completed egress commands.  If that fails, wait or fail.
		 */
		__gxio_dma_queue_update_credits(dma_queue);
		old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
					 modifier);
		if (old + modifier < 0) {
			if (wait)
				old = __gxio_dma_queue_wait_for_credits
					(dma_queue, modifier);
			else
				return GXIO_ERR_DMA_CREDITS;
		}
	}

	/* The bottom 24 bits of old encode the "slot". */
	slot = (old & 0xffffff);

	if (completion) {
		/*
		 * A "completion_slot" is a "slot" which can be compared to
		 * "hw_complete_count" at any time in the future.  To convert
		 * "slot" into a "completion_slot", we access "hw_complete_count"
		 * once (knowing that we have reserved a slot, and thus, it will
		 * be "basically" accurate), and combine its high 40 bits with
		 * the 24 bit "slot", and handle "wrapping" by adding "1 << 24"
		 * if the result is LESS than "hw_complete_count".
		 */
		uint64_t complete;
		complete = READ_ONCE(dma_queue->hw_complete_count);
		slot |= (complete & 0xffffffffff000000);
		if (slot < complete)
			slot += 0x1000000;
	}

	/*
	 * If any of our slots mod 256 were equivalent to 0, go ahead and
	 * collect some egress credits, and update "hw_complete_count", and
	 * make sure the index doesn't overflow into the credits.
	 */
	if (unlikely(((old + num) & 0xff) < num)) {
		__gxio_dma_queue_update_credits(dma_queue);

		/* Make sure the index doesn't overflow into the credits. */
#ifdef __BIG_ENDIAN__
		*(((uint8_t *)&dma_queue->credits_and_next_index) + 4) = 0;
#else
		*(((uint8_t *)&dma_queue->credits_and_next_index) + 3) = 0;
#endif
	}

	return slot;
}

/* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */
extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
					    unsigned int num, int wait);

/* Check whether a particular "completion slot" has completed.
 *
 * Note that this function requires a "completion slot", and thus
 * cannot be used with the result of any "reserve_fast" function.
 */
extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
					int64_t completion_slot, int update);

#endif /* !_GXIO_DMA_QUEUE_H_ */
Commit	Line	Data
63697980 CM	1	/*
	2	* Copyright 2012 Tilera Corporation. All Rights Reserved.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation, version 2.
	7	*
	8	* This program is distributed in the hope that it will be useful, but
	9	* WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
	11	* NON INFRINGEMENT. See the GNU General Public License for
	12	* more details.
	13	*/
	14
	15	#ifndef _GXIO_DMA_QUEUE_H_
	16	#define _GXIO_DMA_QUEUE_H_
	17
	18	/*
	19	* DMA queue management APIs shared between TRIO and mPIPE.
	20	*/
	21
a1ce3928	22	#include <gxio/common.h>
63697980 CM	23
	24	/* The credit counter lives in the high 32 bits. */
	25	#define DMA_QUEUE_CREDIT_SHIFT 32
	26
	27	/*
	28	* State object that tracks a DMA queue's head and tail indices, as
	29	* well as the number of commands posted and completed. The
	30	* structure is accessed via a thread-safe, lock-free algorithm.
	31	*/
	32	typedef struct {
	33	/*
	34	* Address of a MPIPE_EDMA_POST_REGION_VAL_t,
	35	* TRIO_PUSH_DMA_REGION_VAL_t, or TRIO_PULL_DMA_REGION_VAL_t
	36	* register. These register have identical encodings and provide
	37	* information about how many commands have been processed.
	38	*/
	39	void *post_region_addr;
	40
	41	/*
	42	* A lazily-updated count of how many edescs the hardware has
	43	* completed.
	44	*/
	45	uint64_t hw_complete_count __attribute__ ((aligned(64)));
	46
	47	/*
	48	* High 32 bits are a count of available egress command credits,
	49	* low 24 bits are the next egress "slot".
	50	*/
	51	int64_t credits_and_next_index;
	52
	53	} __gxio_dma_queue_t;
	54
	55	/* Initialize a dma queue. */
	56	extern void __gxio_dma_queue_init(__gxio_dma_queue_t *dma_queue,
	57	void *post_region_addr,
	58	unsigned int num_entries);
	59
	60	/*
	61	* Update the "credits_and_next_index" and "hw_complete_count" fields
	62	* based on pending hardware completions. Note that some other thread
	63	* may have already done this and, importantly, may still be in the
	64	* process of updating "credits_and_next_index".
	65	*/
	66	extern void __gxio_dma_queue_update_credits(__gxio_dma_queue_t *dma_queue);
	67
	68	/* Wait for credits to become available. */
	69	extern int64_t __gxio_dma_queue_wait_for_credits(__gxio_dma_queue_t *dma_queue,
	70	int64_t modifier);
	71
	72	/* Reserve slots in the queue, optionally waiting for slots to become
	73	* available, and optionally returning a "completion_slot" suitable for
	74	* direct comparison to "hw_complete_count".
	75	*/
	76	static inline int64_t __gxio_dma_queue_reserve(__gxio_dma_queue_t *dma_queue,
	77	unsigned int num, bool wait,
	78	bool completion)
	79	{
	80	uint64_t slot;
	81
	82	/*
	83	* Try to reserve 'num' egress command slots. We do this by
	84	* constructing a constant that subtracts N credits and adds N to
	85	* the index, and using fetchaddgez to only apply it if the credits
	86	* count doesn't go negative.
87	*/
88	int64_t modifier = (((int64_t)(-num)) << DMA_QUEUE_CREDIT_SHIFT) \| num;
89	int64_t old =
90	__insn_fetchaddgez(&dma_queue->credits_and_next_index,
91	modifier);
92
93	if (unlikely(old + modifier < 0)) {
94	/*
95	* We're out of credits. Try once to get more by checking for
96	* completed egress commands. If that fails, wait or fail.
97	*/
98	__gxio_dma_queue_update_credits(dma_queue);
99	old = __insn_fetchaddgez(&dma_queue->credits_and_next_index,
100	modifier);
101	if (old + modifier < 0) {
102	if (wait)
103	old = __gxio_dma_queue_wait_for_credits
104	(dma_queue, modifier);
105	else
106	return GXIO_ERR_DMA_CREDITS;
107	}
108	}
109
110	/* The bottom 24 bits of old encode the "slot". */
111	slot = (old & 0xffffff);
112
113	if (completion) {
114	/*
115	* A "completion_slot" is a "slot" which can be compared to
116	* "hw_complete_count" at any time in the future. To convert
117	* "slot" into a "completion_slot", we access "hw_complete_count"
118	* once (knowing that we have reserved a slot, and thus, it will
119	* be "basically" accurate), and combine its high 40 bits with
120	* the 24 bit "slot", and handle "wrapping" by adding "1 << 24"
121	* if the result is LESS than "hw_complete_count".
122	*/
123	uint64_t complete;
6aa7de05	124	complete = READ_ONCE(dma_queue->hw_complete_count);
63697980 CM	125	slot \|= (complete & 0xffffffffff000000);
	126	if (slot < complete)
	127	slot += 0x1000000;
	128	}
	129
	130	/*
	131	* If any of our slots mod 256 were equivalent to 0, go ahead and
	132	* collect some egress credits, and update "hw_complete_count", and
	133	* make sure the index doesn't overflow into the credits.
	134	*/
	135	if (unlikely(((old + num) & 0xff) < num)) {
	136	__gxio_dma_queue_update_credits(dma_queue);
	137
	138	/* Make sure the index doesn't overflow into the credits. */
	139	#ifdef __BIG_ENDIAN__
	140	(((uint8_t )&dma_queue->credits_and_next_index) + 4) = 0;
	141	#else
	142	(((uint8_t )&dma_queue->credits_and_next_index) + 3) = 0;
	143	#endif
	144	}
	145
	146	return slot;
	147	}
	148
	149	/* Non-inlinable "__gxio_dma_queue_reserve(..., true)". */
	150	extern int64_t __gxio_dma_queue_reserve_aux(__gxio_dma_queue_t *dma_queue,
	151	unsigned int num, int wait);
	152
	153	/* Check whether a particular "completion slot" has completed.
	154	*
	155	* Note that this function requires a "completion slot", and thus
	156	* cannot be used with the result of any "reserve_fast" function.
	157	*/
	158	extern int __gxio_dma_queue_is_complete(__gxio_dma_queue_t *dma_queue,
	159	int64_t completion_slot, int update);
	160
	161	#endif /* !_GXIO_DMA_QUEUE_H_ */