[mirror_ubuntu-artful-kernel.git] / drivers / staging / lustre / lustre / ptlrpc / sec_bulk.c

/*
 * GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see
 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
 *
 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
 * CA 95054 USA or visit www.sun.com if you need additional information or
 * have any questions.
 *
 * GPL HEADER END
 */
/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright (c) 2011, 2015, Intel Corporation.
 */
/*
 * This file is part of Lustre, http://www.lustre.org/
 * Lustre is a trademark of Sun Microsystems, Inc.
 *
 * lustre/ptlrpc/sec_bulk.c
 *
 * Author: Eric Mei <ericm@clusterfs.com>
 */

#define DEBUG_SUBSYSTEM S_SEC

#include "../../include/linux/libcfs/libcfs.h"
#include <linux/crypto.h>

#include "../include/obd.h"
#include "../include/obd_cksum.h"
#include "../include/obd_class.h"
#include "../include/obd_support.h"
#include "../include/lustre_net.h"
#include "../include/lustre_import.h"
#include "../include/lustre_dlm.h"
#include "../include/lustre_sec.h"

#include "ptlrpc_internal.h"

/****************************************
 * bulk encryption page pools	   *
 ****************************************/

#define POINTERS_PER_PAGE	(PAGE_SIZE / sizeof(void *))
#define PAGES_PER_POOL		(POINTERS_PER_PAGE)

#define IDLE_IDX_MAX	 (100)
#define IDLE_IDX_WEIGHT	 (3)

#define CACHE_QUIESCENT_PERIOD  (20)

static struct ptlrpc_enc_page_pool {
	/*
	 * constants
	 */
	unsigned long    epp_max_pages;   /* maximum pages can hold, const */
	unsigned int     epp_max_pools;   /* number of pools, const */

	/*
	 * wait queue in case of not enough free pages.
	 */
	wait_queue_head_t      epp_waitq;       /* waiting threads */
	unsigned int     epp_waitqlen;    /* wait queue length */
	unsigned long    epp_pages_short; /* # of pages wanted of in-q users */
	unsigned int     epp_growing:1;   /* during adding pages */

	/*
	 * indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	 * this is counted based on each time when getting pages from
	 * the pools, not based on time. which means in case that system
	 * is idled for a while but the idle_idx might still be low if no
	 * activities happened in the pools.
	 */
	unsigned long    epp_idle_idx;

	/* last shrink time due to mem tight */
	time64_t         epp_last_shrink;
	time64_t         epp_last_access;

	/*
	 * in-pool pages bookkeeping
	 */
	spinlock_t	 epp_lock;	   /* protect following fields */
	unsigned long    epp_total_pages; /* total pages in pools */
	unsigned long    epp_free_pages;  /* current pages available */

	/*
	 * statistics
	 */
	unsigned long    epp_st_max_pages;      /* # of pages ever reached */
	unsigned int     epp_st_grows;	  /* # of grows */
	unsigned int     epp_st_grow_fails;     /* # of add pages failures */
	unsigned int     epp_st_shrinks;	/* # of shrinks */
	unsigned long    epp_st_access;	 /* # of access */
	unsigned long    epp_st_missings;       /* # of cache missing */
	unsigned long    epp_st_lowfree;	/* lowest free pages reached */
	unsigned int     epp_st_max_wqlen;      /* highest waitqueue length */
	unsigned long       epp_st_max_wait;       /* in jiffies */
	/*
	 * pointers to pools
	 */
	struct page    ***epp_pools;
} page_pools;

/*
 * /sys/kernel/debug/lustre/sptlrpc/encrypt_page_pools
 */
int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
{
	spin_lock(&page_pools.epp_lock);

	seq_printf(m,
		   "physical pages:	  %lu\n"
		   "pages per pool:	  %lu\n"
		   "max pages:	       %lu\n"
		   "max pools:	       %u\n"
		   "total pages:	     %lu\n"
		   "total free:	      %lu\n"
		   "idle index:	      %lu/100\n"
		   "last shrink:	     %lds\n"
		   "last access:	     %lds\n"
		   "max pages reached:       %lu\n"
		   "grows:		   %u\n"
		   "grows failure:	   %u\n"
		   "shrinks:		 %u\n"
		   "cache access:	    %lu\n"
		   "cache missing:	   %lu\n"
		   "low free mark:	   %lu\n"
		   "max waitqueue depth:     %u\n"
		   "max wait time:	   %ld/%u\n",
		   totalram_pages,
		   PAGES_PER_POOL,
		   page_pools.epp_max_pages,
		   page_pools.epp_max_pools,
		   page_pools.epp_total_pages,
		   page_pools.epp_free_pages,
		   page_pools.epp_idle_idx,
		   (long)(ktime_get_seconds() - page_pools.epp_last_shrink),
		   (long)(ktime_get_seconds() - page_pools.epp_last_access),
		   page_pools.epp_st_max_pages,
		   page_pools.epp_st_grows,
		   page_pools.epp_st_grow_fails,
		   page_pools.epp_st_shrinks,
		   page_pools.epp_st_access,
		   page_pools.epp_st_missings,
		   page_pools.epp_st_lowfree,
		   page_pools.epp_st_max_wqlen,
		   page_pools.epp_st_max_wait,
		   HZ);

	spin_unlock(&page_pools.epp_lock);

	return 0;
}

static void enc_pools_release_free_pages(long npages)
{
	int p_idx, g_idx;
	int p_idx_max1, p_idx_max2;

	LASSERT(npages > 0);
	LASSERT(npages <= page_pools.epp_free_pages);
	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);

	/* max pool index before the release */
	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;

	page_pools.epp_free_pages -= npages;
	page_pools.epp_total_pages -= npages;

	/* max pool index after the release */
	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
		     ((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	LASSERT(page_pools.epp_pools[p_idx]);

	while (npages--) {
		LASSERT(page_pools.epp_pools[p_idx]);
		LASSERT(page_pools.epp_pools[p_idx][g_idx]);

		__free_page(page_pools.epp_pools[p_idx][g_idx]);
		page_pools.epp_pools[p_idx][g_idx] = NULL;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	/* free unused pools */
	while (p_idx_max1 < p_idx_max2) {
		LASSERT(page_pools.epp_pools[p_idx_max2]);
		kfree(page_pools.epp_pools[p_idx_max2]);
		page_pools.epp_pools[p_idx_max2] = NULL;
		p_idx_max2--;
	}
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_count(struct shrinker *s,
					    struct shrink_control *sc)
{
	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
		(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_scan(struct shrinker *s,
					   struct shrink_control *sc)
{
	spin_lock(&page_pools.epp_lock);
	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
			      page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	if (sc->nr_to_scan > 0) {
		enc_pools_release_free_pages(sc->nr_to_scan);
		CDEBUG(D_SEC, "released %ld pages, %ld left\n",
		       (long)sc->nr_to_scan, page_pools.epp_free_pages);

		page_pools.epp_st_shrinks++;
		page_pools.epp_last_shrink = ktime_get_seconds();
	}
	spin_unlock(&page_pools.epp_lock);

	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return sc->nr_to_scan;
}

static inline
int npages_to_npools(unsigned long npages)
{
	return (int) ((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
}

/*
 * return how many pages cleaned up.
 */
static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
{
	unsigned long cleaned = 0;
	int i, j;

	for (i = 0; i < npools; i++) {
		if (pools[i]) {
			for (j = 0; j < PAGES_PER_POOL; j++) {
				if (pools[i][j]) {
					__free_page(pools[i][j]);
					cleaned++;
				}
			}
			kfree(pools[i]);
			pools[i] = NULL;
		}
	}

	return cleaned;
}

static inline void enc_pools_wakeup(void)
{
	assert_spin_locked(&page_pools.epp_lock);

	if (unlikely(page_pools.epp_waitqlen)) {
		LASSERT(waitqueue_active(&page_pools.epp_waitq));
		wake_up_all(&page_pools.epp_waitq);
	}
}

void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
{
	int p_idx, g_idx;
	int i;

	if (!desc->bd_enc_iov)
		return;

	LASSERT(desc->bd_iov_count > 0);

	spin_lock(&page_pools.epp_lock);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;

	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
		page_pools.epp_total_pages);
	LASSERT(page_pools.epp_pools[p_idx]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		LASSERT(desc->bd_enc_iov[i].kiov_page);
		LASSERT(g_idx != 0 || page_pools.epp_pools[p_idx]);
		LASSERT(!page_pools.epp_pools[p_idx][g_idx]);

		page_pools.epp_pools[p_idx][g_idx] =
					desc->bd_enc_iov[i].kiov_page;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	page_pools.epp_free_pages += desc->bd_iov_count;

	enc_pools_wakeup();

	spin_unlock(&page_pools.epp_lock);

	kfree(desc->bd_enc_iov);
	desc->bd_enc_iov = NULL;
}
EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);

static inline void enc_pools_alloc(void)
{
	LASSERT(page_pools.epp_max_pools);
	page_pools.epp_pools =
		libcfs_kvzalloc(page_pools.epp_max_pools *
				sizeof(*page_pools.epp_pools),
				GFP_NOFS);
}

static inline void enc_pools_free(void)
{
	LASSERT(page_pools.epp_max_pools);
	LASSERT(page_pools.epp_pools);

	kvfree(page_pools.epp_pools);
}

static struct shrinker pools_shrinker = {
	.count_objects	= enc_pools_shrink_count,
	.scan_objects	= enc_pools_shrink_scan,
	.seeks		= DEFAULT_SEEKS,
};

int sptlrpc_enc_pool_init(void)
{
	/*
	 * maximum capacity is 1/8 of total physical memory.
	 * is the 1/8 a good number?
	 */
	page_pools.epp_max_pages = totalram_pages / 8;
	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);

	init_waitqueue_head(&page_pools.epp_waitq);
	page_pools.epp_waitqlen = 0;
	page_pools.epp_pages_short = 0;

	page_pools.epp_growing = 0;

	page_pools.epp_idle_idx = 0;
	page_pools.epp_last_shrink = ktime_get_seconds();
	page_pools.epp_last_access = ktime_get_seconds();

	spin_lock_init(&page_pools.epp_lock);
	page_pools.epp_total_pages = 0;
	page_pools.epp_free_pages = 0;

	page_pools.epp_st_max_pages = 0;
	page_pools.epp_st_grows = 0;
	page_pools.epp_st_grow_fails = 0;
	page_pools.epp_st_shrinks = 0;
	page_pools.epp_st_access = 0;
	page_pools.epp_st_missings = 0;
	page_pools.epp_st_lowfree = 0;
	page_pools.epp_st_max_wqlen = 0;
	page_pools.epp_st_max_wait = 0;

	enc_pools_alloc();
	if (!page_pools.epp_pools)
		return -ENOMEM;

	register_shrinker(&pools_shrinker);

	return 0;
}

void sptlrpc_enc_pool_fini(void)
{
	unsigned long cleaned, npools;

	LASSERT(page_pools.epp_pools);
	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);

	unregister_shrinker(&pools_shrinker);

	npools = npages_to_npools(page_pools.epp_total_pages);
	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	LASSERT(cleaned == page_pools.epp_total_pages);

	enc_pools_free();

	if (page_pools.epp_st_access > 0) {
		CDEBUG(D_SEC,
		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
		       page_pools.epp_st_max_pages, page_pools.epp_st_grows,
		       page_pools.epp_st_grow_fails,
		       page_pools.epp_st_shrinks, page_pools.epp_st_access,
		       page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
		       page_pools.epp_st_max_wait, HZ);
	}
}

static int cfs_hash_alg_id[] = {
	[BULK_HASH_ALG_NULL]	= CFS_HASH_ALG_NULL,
	[BULK_HASH_ALG_ADLER32]	= CFS_HASH_ALG_ADLER32,
	[BULK_HASH_ALG_CRC32]	= CFS_HASH_ALG_CRC32,
	[BULK_HASH_ALG_MD5]	= CFS_HASH_ALG_MD5,
	[BULK_HASH_ALG_SHA1]	= CFS_HASH_ALG_SHA1,
	[BULK_HASH_ALG_SHA256]	= CFS_HASH_ALG_SHA256,
	[BULK_HASH_ALG_SHA384]	= CFS_HASH_ALG_SHA384,
	[BULK_HASH_ALG_SHA512]	= CFS_HASH_ALG_SHA512,
};

const char *sptlrpc_get_hash_name(__u8 hash_alg)
{
	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
}
EXPORT_SYMBOL(sptlrpc_get_hash_name);

__u8 sptlrpc_get_hash_alg(const char *algname)
{
	return cfs_crypto_hash_alg(algname);
}
EXPORT_SYMBOL(sptlrpc_get_hash_alg);

int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
{
	struct ptlrpc_bulk_sec_desc *bsd;
	int			  size = msg->lm_buflens[offset];

	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
	if (!bsd) {
		CERROR("Invalid bulk sec desc: size %d\n", size);
		return -EINVAL;
	}

	if (swabbed)
		__swab32s(&bsd->bsd_nob);

	if (unlikely(bsd->bsd_version != 0)) {
		CERROR("Unexpected version %u\n", bsd->bsd_version);
		return -EPROTO;
	}

	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
		CERROR("Invalid type %u\n", bsd->bsd_type);
		return -EPROTO;
	}

	/* FIXME more sanity check here */

	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
		CERROR("Invalid svc %u\n", bsd->bsd_svc);
		return -EPROTO;
	}

	return 0;
}
EXPORT_SYMBOL(bulk_sec_desc_unpack);

int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
			      void *buf, int buflen)
{
	struct cfs_crypto_hash_desc *hdesc;
	int hashsize;
	char hashbuf[64];
	unsigned int bufsize;
	int i, err;

	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	LASSERT(buflen >= 4);

	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	if (IS_ERR(hdesc)) {
		CERROR("Unable to initialize checksum hash %s\n",
		       cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
		return PTR_ERR(hdesc);
	}

	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
				  desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK,
				  desc->bd_iov[i].kiov_len);
	}
	if (hashsize > buflen) {
		bufsize = sizeof(hashbuf);
		err = cfs_crypto_hash_final(hdesc, (unsigned char *)hashbuf,
					    &bufsize);
		memcpy(buf, hashbuf, buflen);
	} else {
		bufsize = buflen;
		err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
	}

	if (err)
		cfs_crypto_hash_final(hdesc, NULL, NULL);
	return err;
}
EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);
Commit	Line	Data
d7e09d03 PT	1	/*
	2	* GPL HEADER START
	3	*
	4	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 only,
	8	* as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License version 2 for more details (a copy is included
	14	* in the LICENSE file that accompanied this code).
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* version 2 along with this program; If not, see
	18	* http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
	19	*
	20	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
	21	* CA 95054 USA or visit www.sun.com if you need additional information or
	22	* have any questions.
	23	*
	24	* GPL HEADER END
	25	*/
	26	/*
	27	* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
	28	* Use is subject to license terms.
	29	*
1dc563a6	30	* Copyright (c) 2011, 2015, Intel Corporation.
d7e09d03 PT	31	*/
	32	/*
	33	* This file is part of Lustre, http://www.lustre.org/
	34	* Lustre is a trademark of Sun Microsystems, Inc.
	35	*
	36	* lustre/ptlrpc/sec_bulk.c
	37	*
	38	* Author: Eric Mei <ericm@clusterfs.com>
	39	*/
	40
	41	#define DEBUG_SUBSYSTEM S_SEC
	42
9fdaf8c0	43	#include "../../include/linux/libcfs/libcfs.h"
d7e09d03 PT	44	#include <linux/crypto.h>
d7e09d03 PT	45
e27db149 GKH	46	#include "../include/obd.h"
	47	#include "../include/obd_cksum.h"
	48	#include "../include/obd_class.h"
	49	#include "../include/obd_support.h"
	50	#include "../include/lustre_net.h"
	51	#include "../include/lustre_import.h"
	52	#include "../include/lustre_dlm.h"
	53	#include "../include/lustre_sec.h"
d7e09d03 PT	54
	55	#include "ptlrpc_internal.h"
	56
	57	/****************************************
	58	* bulk encryption page pools *
	59	****************************************/
	60
09cbfeaf	61	#define POINTERS_PER_PAGE (PAGE_SIZE / sizeof(void *))
ae18c5c6	62	#define PAGES_PER_POOL (POINTERS_PER_PAGE)
d7e09d03	63
d0bfef31	64	#define IDLE_IDX_MAX (100)
d7e09d03 PT	65	#define IDLE_IDX_WEIGHT (3)
	66
	67	#define CACHE_QUIESCENT_PERIOD (20)
	68
	69	static struct ptlrpc_enc_page_pool {
	70	/*
	71	* constants
	72	*/
	73	unsigned long epp_max_pages; /* maximum pages can hold, const */
	74	unsigned int epp_max_pools; /* number of pools, const */
	75
	76	/*
	77	* wait queue in case of not enough free pages.
	78	*/
	79	wait_queue_head_t epp_waitq; /* waiting threads */
	80	unsigned int epp_waitqlen; /* wait queue length */
	81	unsigned long epp_pages_short; /* # of pages wanted of in-q users */
	82	unsigned int epp_growing:1; /* during adding pages */
	83
	84	/*
	85	* indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	86	* this is counted based on each time when getting pages from
	87	* the pools, not based on time. which means in case that system
	88	* is idled for a while but the idle_idx might still be low if no
	89	* activities happened in the pools.
	90	*/
	91	unsigned long epp_idle_idx;
	92
	93	/* last shrink time due to mem tight */
80018a9e AB	94	time64_t epp_last_shrink;
80018a9e AB	95	time64_t epp_last_access;
d7e09d03 PT	96
	97	/*
	98	* in-pool pages bookkeeping
	99	*/
	100	spinlock_t epp_lock; /* protect following fields */
	101	unsigned long epp_total_pages; /* total pages in pools */
	102	unsigned long epp_free_pages; /* current pages available */
	103
	104	/*
	105	* statistics
	106	*/
	107	unsigned long epp_st_max_pages; /* # of pages ever reached */
	108	unsigned int epp_st_grows; /* # of grows */
	109	unsigned int epp_st_grow_fails; /* # of add pages failures */
	110	unsigned int epp_st_shrinks; /* # of shrinks */
	111	unsigned long epp_st_access; /* # of access */
	112	unsigned long epp_st_missings; /* # of cache missing */
	113	unsigned long epp_st_lowfree; /* lowest free pages reached */
	114	unsigned int epp_st_max_wqlen; /* highest waitqueue length */
a649ad1d	115	unsigned long epp_st_max_wait; /* in jiffies */
d7e09d03 PT	116	/*
	117	* pointers to pools
	118	*/
	119	struct page ***epp_pools;
	120	} page_pools;
	121
d7e09d03	122	/*
f6e42a40	123	* /sys/kernel/debug/lustre/sptlrpc/encrypt_page_pools
d7e09d03	124	*/
73bb1da6	125	int sptlrpc_proc_enc_pool_seq_show(struct seq_file m, void v)
d7e09d03	126	{
d7e09d03 PT	127	spin_lock(&page_pools.epp_lock);
d7e09d03 PT	128
91b3a685 JP	129	seq_printf(m,
	130	"physical pages: %lu\n"
	131	"pages per pool: %lu\n"
	132	"max pages: %lu\n"
	133	"max pools: %u\n"
	134	"total pages: %lu\n"
	135	"total free: %lu\n"
	136	"idle index: %lu/100\n"
	137	"last shrink: %lds\n"
	138	"last access: %lds\n"
	139	"max pages reached: %lu\n"
	140	"grows: %u\n"
	141	"grows failure: %u\n"
	142	"shrinks: %u\n"
	143	"cache access: %lu\n"
	144	"cache missing: %lu\n"
	145	"low free mark: %lu\n"
	146	"max waitqueue depth: %u\n"
93d3a405	147	"max wait time: %ld/%u\n",
91b3a685 JP	148	totalram_pages,
	149	PAGES_PER_POOL,
	150	page_pools.epp_max_pages,
	151	page_pools.epp_max_pools,
	152	page_pools.epp_total_pages,
	153	page_pools.epp_free_pages,
	154	page_pools.epp_idle_idx,
80018a9e AB	155	(long)(ktime_get_seconds() - page_pools.epp_last_shrink),
80018a9e AB	156	(long)(ktime_get_seconds() - page_pools.epp_last_access),
91b3a685 JP	157	page_pools.epp_st_max_pages,
	158	page_pools.epp_st_grows,
	159	page_pools.epp_st_grow_fails,
	160	page_pools.epp_st_shrinks,
	161	page_pools.epp_st_access,
	162	page_pools.epp_st_missings,
	163	page_pools.epp_st_lowfree,
	164	page_pools.epp_st_max_wqlen,
	165	page_pools.epp_st_max_wait,
	166	HZ);
d7e09d03 PT	167
d7e09d03 PT	168	spin_unlock(&page_pools.epp_lock);
91b3a685 JP	169
91b3a685 JP	170	return 0;
d7e09d03 PT	171	}
	172
	173	static void enc_pools_release_free_pages(long npages)
	174	{
d0bfef31 CH	175	int p_idx, g_idx;
d0bfef31 CH	176	int p_idx_max1, p_idx_max2;
d7e09d03 PT	177
	178	LASSERT(npages > 0);
	179	LASSERT(npages <= page_pools.epp_free_pages);
	180	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);
	181
	182	/* max pool index before the release */
	183	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;
	184
	185	page_pools.epp_free_pages -= npages;
	186	page_pools.epp_total_pages -= npages;
	187
	188	/* max pool index after the release */
	189	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
	190	((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);
	191
	192	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	193	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	194	LASSERT(page_pools.epp_pools[p_idx]);
	195
	196	while (npages--) {
	197	LASSERT(page_pools.epp_pools[p_idx]);
8b382089	198	LASSERT(page_pools.epp_pools[p_idx][g_idx]);
d7e09d03 PT	199
	200	__free_page(page_pools.epp_pools[p_idx][g_idx]);
	201	page_pools.epp_pools[p_idx][g_idx] = NULL;
	202
	203	if (++g_idx == PAGES_PER_POOL) {
	204	p_idx++;
	205	g_idx = 0;
	206	}
9076b09e	207	}
d7e09d03 PT	208
	209	/* free unused pools */
	210	while (p_idx_max1 < p_idx_max2) {
	211	LASSERT(page_pools.epp_pools[p_idx_max2]);
9ae10597	212	kfree(page_pools.epp_pools[p_idx_max2]);
d7e09d03 PT	213	page_pools.epp_pools[p_idx_max2] = NULL;
	214	p_idx_max2--;
	215	}
	216	}
	217
	218	/*
d7e09d03 PT	219	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
d7e09d03 PT	220	*/
3bb22ec5 PT	221	static unsigned long enc_pools_shrink_count(struct shrinker *s,
3bb22ec5 PT	222	struct shrink_control *sc)
d7e09d03	223	{
3bb22ec5 PT	224	/*
	225	* if no pool access for a long time, we consider it's fully idle.
	226	* a little race here is fine.
	227	*/
80018a9e	228	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
3bb22ec5	229	CACHE_QUIESCENT_PERIOD)) {
d7e09d03	230	spin_lock(&page_pools.epp_lock);
3bb22ec5	231	page_pools.epp_idle_idx = IDLE_IDX_MAX;
d7e09d03 PT	232	spin_unlock(&page_pools.epp_lock);
	233	}
	234
3bb22ec5 PT	235	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	236	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
	237	(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
	238	}
	239
	240	/*
	241	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
	242	*/
	243	static unsigned long enc_pools_shrink_scan(struct shrinker *s,
	244	struct shrink_control *sc)
	245	{
	246	spin_lock(&page_pools.epp_lock);
	247	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
	248	page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	249	if (sc->nr_to_scan > 0) {
	250	enc_pools_release_free_pages(sc->nr_to_scan);
	251	CDEBUG(D_SEC, "released %ld pages, %ld left\n",
	252	(long)sc->nr_to_scan, page_pools.epp_free_pages);
	253
	254	page_pools.epp_st_shrinks++;
80018a9e	255	page_pools.epp_last_shrink = ktime_get_seconds();
3bb22ec5 PT	256	}
	257	spin_unlock(&page_pools.epp_lock);
	258
d7e09d03 PT	259	/*
	260	* if no pool access for a long time, we consider it's fully idle.
	261	* a little race here is fine.
	262	*/
80018a9e	263	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
d7e09d03 PT	264	CACHE_QUIESCENT_PERIOD)) {
	265	spin_lock(&page_pools.epp_lock);
	266	page_pools.epp_idle_idx = IDLE_IDX_MAX;
	267	spin_unlock(&page_pools.epp_lock);
	268	}
	269
	270	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
3bb22ec5	271	return sc->nr_to_scan;
d7e09d03 PT	272	}
	273
	274	static inline
	275	int npages_to_npools(unsigned long npages)
	276	{
	277	return (int) ((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
	278	}
	279
	280	/*
	281	* return how many pages cleaned up.
	282	*/
	283	static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
	284	{
	285	unsigned long cleaned = 0;
d0bfef31	286	int i, j;
d7e09d03 PT	287
	288	for (i = 0; i < npools; i++) {
	289	if (pools[i]) {
	290	for (j = 0; j < PAGES_PER_POOL; j++) {
	291	if (pools[i][j]) {
	292	__free_page(pools[i][j]);
	293	cleaned++;
	294	}
	295	}
9ae10597	296	kfree(pools[i]);
d7e09d03 PT	297	pools[i] = NULL;
	298	}
	299	}
	300
	301	return cleaned;
	302	}
	303
d7e09d03 PT	304	static inline void enc_pools_wakeup(void)
d7e09d03 PT	305	{
5e42bc9d	306	assert_spin_locked(&page_pools.epp_lock);
d7e09d03 PT	307
	308	if (unlikely(page_pools.epp_waitqlen)) {
	309	LASSERT(waitqueue_active(&page_pools.epp_waitq));
	310	wake_up_all(&page_pools.epp_waitq);
	311	}
	312	}
	313
d7e09d03 PT	314	void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
d7e09d03 PT	315	{
d0bfef31 CH	316	int p_idx, g_idx;
d0bfef31 CH	317	int i;
d7e09d03	318
8b382089	319	if (!desc->bd_enc_iov)
d7e09d03 PT	320	return;
	321
	322	LASSERT(desc->bd_iov_count > 0);
	323
	324	spin_lock(&page_pools.epp_lock);
	325
	326	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	327	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	328
	329	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
	330	page_pools.epp_total_pages);
	331	LASSERT(page_pools.epp_pools[p_idx]);
	332
	333	for (i = 0; i < desc->bd_iov_count; i++) {
8b382089	334	LASSERT(desc->bd_enc_iov[i].kiov_page);
d7e09d03	335	LASSERT(g_idx != 0 \|\| page_pools.epp_pools[p_idx]);
8b382089	336	LASSERT(!page_pools.epp_pools[p_idx][g_idx]);
d7e09d03 PT	337
	338	page_pools.epp_pools[p_idx][g_idx] =
	339	desc->bd_enc_iov[i].kiov_page;
	340
	341	if (++g_idx == PAGES_PER_POOL) {
	342	p_idx++;
	343	g_idx = 0;
	344	}
	345	}
	346
	347	page_pools.epp_free_pages += desc->bd_iov_count;
	348
	349	enc_pools_wakeup();
	350
	351	spin_unlock(&page_pools.epp_lock);
	352
9ae10597	353	kfree(desc->bd_enc_iov);
d7e09d03 PT	354	desc->bd_enc_iov = NULL;
	355	}
	356	EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);
	357
d7e09d03 PT	358	static inline void enc_pools_alloc(void)
	359	{
	360	LASSERT(page_pools.epp_max_pools);
ee0ec194 JL	361	page_pools.epp_pools =
	362	libcfs_kvzalloc(page_pools.epp_max_pools *
	363	sizeof(*page_pools.epp_pools),
	364	GFP_NOFS);
d7e09d03 PT	365	}
	366
	367	static inline void enc_pools_free(void)
	368	{
	369	LASSERT(page_pools.epp_max_pools);
	370	LASSERT(page_pools.epp_pools);
	371
ee0ec194	372	kvfree(page_pools.epp_pools);
d7e09d03 PT	373	}
d7e09d03 PT	374
3bb22ec5 PT	375	static struct shrinker pools_shrinker = {
	376	.count_objects = enc_pools_shrink_count,
	377	.scan_objects = enc_pools_shrink_scan,
	378	.seeks = DEFAULT_SEEKS,
	379	};
	380
d7e09d03 PT	381	int sptlrpc_enc_pool_init(void)
	382	{
	383	/*
	384	* maximum capacity is 1/8 of total physical memory.
	385	* is the 1/8 a good number?
	386	*/
4f6cc9ab	387	page_pools.epp_max_pages = totalram_pages / 8;
d7e09d03 PT	388	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
	389
	390	init_waitqueue_head(&page_pools.epp_waitq);
	391	page_pools.epp_waitqlen = 0;
	392	page_pools.epp_pages_short = 0;
	393
	394	page_pools.epp_growing = 0;
	395
	396	page_pools.epp_idle_idx = 0;
80018a9e AB	397	page_pools.epp_last_shrink = ktime_get_seconds();
80018a9e AB	398	page_pools.epp_last_access = ktime_get_seconds();
d7e09d03 PT	399
	400	spin_lock_init(&page_pools.epp_lock);
	401	page_pools.epp_total_pages = 0;
	402	page_pools.epp_free_pages = 0;
	403
	404	page_pools.epp_st_max_pages = 0;
	405	page_pools.epp_st_grows = 0;
	406	page_pools.epp_st_grow_fails = 0;
	407	page_pools.epp_st_shrinks = 0;
	408	page_pools.epp_st_access = 0;
	409	page_pools.epp_st_missings = 0;
	410	page_pools.epp_st_lowfree = 0;
	411	page_pools.epp_st_max_wqlen = 0;
	412	page_pools.epp_st_max_wait = 0;
	413
	414	enc_pools_alloc();
8b382089	415	if (!page_pools.epp_pools)
d7e09d03 PT	416	return -ENOMEM;
d7e09d03 PT	417
3bb22ec5	418	register_shrinker(&pools_shrinker);
d7e09d03 PT	419
	420	return 0;
	421	}
	422
	423	void sptlrpc_enc_pool_fini(void)
	424	{
	425	unsigned long cleaned, npools;
	426
d7e09d03 PT	427	LASSERT(page_pools.epp_pools);
	428	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
	429
3bb22ec5	430	unregister_shrinker(&pools_shrinker);
d7e09d03 PT	431
	432	npools = npages_to_npools(page_pools.epp_total_pages);
	433	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	434	LASSERT(cleaned == page_pools.epp_total_pages);
	435
	436	enc_pools_free();
	437
	438	if (page_pools.epp_st_access > 0) {
	439	CDEBUG(D_SEC,
93d3a405	440	"max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
d7e09d03 PT	441	page_pools.epp_st_max_pages, page_pools.epp_st_grows,
	442	page_pools.epp_st_grow_fails,
	443	page_pools.epp_st_shrinks, page_pools.epp_st_access,
	444	page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
	445	page_pools.epp_st_max_wait, HZ);
	446	}
	447	}
	448
d7e09d03 PT	449	static int cfs_hash_alg_id[] = {
	450	[BULK_HASH_ALG_NULL] = CFS_HASH_ALG_NULL,
	451	[BULK_HASH_ALG_ADLER32] = CFS_HASH_ALG_ADLER32,
	452	[BULK_HASH_ALG_CRC32] = CFS_HASH_ALG_CRC32,
	453	[BULK_HASH_ALG_MD5] = CFS_HASH_ALG_MD5,
	454	[BULK_HASH_ALG_SHA1] = CFS_HASH_ALG_SHA1,
	455	[BULK_HASH_ALG_SHA256] = CFS_HASH_ALG_SHA256,
	456	[BULK_HASH_ALG_SHA384] = CFS_HASH_ALG_SHA384,
	457	[BULK_HASH_ALG_SHA512] = CFS_HASH_ALG_SHA512,
	458	};
c9f6bb96	459
aff9d8e8	460	const char *sptlrpc_get_hash_name(__u8 hash_alg)
d7e09d03 PT	461	{
	462	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
	463	}
	464	EXPORT_SYMBOL(sptlrpc_get_hash_name);
	465
	466	__u8 sptlrpc_get_hash_alg(const char *algname)
	467	{
	468	return cfs_crypto_hash_alg(algname);
	469	}
	470	EXPORT_SYMBOL(sptlrpc_get_hash_alg);
	471
	472	int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
	473	{
	474	struct ptlrpc_bulk_sec_desc *bsd;
	475	int size = msg->lm_buflens[offset];
	476
	477	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
8b382089	478	if (!bsd) {
d7e09d03 PT	479	CERROR("Invalid bulk sec desc: size %d\n", size);
	480	return -EINVAL;
	481	}
	482
acf60c3d	483	if (swabbed)
d7e09d03	484	__swab32s(&bsd->bsd_nob);
d7e09d03 PT	485
	486	if (unlikely(bsd->bsd_version != 0)) {
	487	CERROR("Unexpected version %u\n", bsd->bsd_version);
	488	return -EPROTO;
	489	}
	490
	491	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
	492	CERROR("Invalid type %u\n", bsd->bsd_type);
	493	return -EPROTO;
	494	}
	495
	496	/* FIXME more sanity check here */
	497
	498	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
	499	bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
	500	bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
	501	CERROR("Invalid svc %u\n", bsd->bsd_svc);
	502	return -EPROTO;
	503	}
	504
	505	return 0;
	506	}
	507	EXPORT_SYMBOL(bulk_sec_desc_unpack);
	508
	509	int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
	510	void *buf, int buflen)
	511	{
d0bfef31 CH	512	struct cfs_crypto_hash_desc *hdesc;
	513	int hashsize;
	514	char hashbuf[64];
	515	unsigned int bufsize;
	516	int i, err;
d7e09d03 PT	517
	518	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	519	LASSERT(buflen >= 4);
	520
	521	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	522	if (IS_ERR(hdesc)) {
	523	CERROR("Unable to initialize checksum hash %s\n",
	524	cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
	525	return PTR_ERR(hdesc);
	526	}
	527
	528	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);
	529
	530	for (i = 0; i < desc->bd_iov_count; i++) {
	531	cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
	532	desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK,
	533	desc->bd_iov[i].kiov_len);
	534	}
	535	if (hashsize > buflen) {
	536	bufsize = sizeof(hashbuf);
	537	err = cfs_crypto_hash_final(hdesc, (unsigned char *)hashbuf,
	538	&bufsize);
	539	memcpy(buf, hashbuf, buflen);
	540	} else {
	541	bufsize = buflen;
3df82ddc	542	err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
d7e09d03 PT	543	}
	544
	545	if (err)
	546	cfs_crypto_hash_final(hdesc, NULL, NULL);
	547	return err;
	548	}
	549	EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);