[mirror_ubuntu-hirsute-kernel.git] / drivers / staging / lustre / lustre / ptlrpc / sec_bulk.c

/*
 * GPL HEADER START
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 only,
 * as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License version 2 for more details (a copy is included
 * in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU General Public License
 * version 2 along with this program; If not, see
 * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
 *
 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
 * CA 95054 USA or visit www.sun.com if you need additional information or
 * have any questions.
 *
 * GPL HEADER END
 */
/*
 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
 * Use is subject to license terms.
 *
 * Copyright (c) 2011, 2012, Intel Corporation.
 */
/*
 * This file is part of Lustre, http://www.lustre.org/
 * Lustre is a trademark of Sun Microsystems, Inc.
 *
 * lustre/ptlrpc/sec_bulk.c
 *
 * Author: Eric Mei <ericm@clusterfs.com>
 */

#define DEBUG_SUBSYSTEM S_SEC

#include "../../include/linux/libcfs/libcfs.h"
#include <linux/crypto.h>

#include "../include/obd.h"
#include "../include/obd_cksum.h"
#include "../include/obd_class.h"
#include "../include/obd_support.h"
#include "../include/lustre_net.h"
#include "../include/lustre_import.h"
#include "../include/lustre_dlm.h"
#include "../include/lustre_sec.h"

#include "ptlrpc_internal.h"

/****************************************
 * bulk encryption page pools	   *
 ****************************************/

#define POINTERS_PER_PAGE	(PAGE_CACHE_SIZE / sizeof(void *))
#define PAGES_PER_POOL		(POINTERS_PER_PAGE)

#define IDLE_IDX_MAX	 (100)
#define IDLE_IDX_WEIGHT	 (3)

#define CACHE_QUIESCENT_PERIOD  (20)

static struct ptlrpc_enc_page_pool {
	/*
	 * constants
	 */
	unsigned long    epp_max_pages;   /* maximum pages can hold, const */
	unsigned int     epp_max_pools;   /* number of pools, const */

	/*
	 * wait queue in case of not enough free pages.
	 */
	wait_queue_head_t      epp_waitq;       /* waiting threads */
	unsigned int     epp_waitqlen;    /* wait queue length */
	unsigned long    epp_pages_short; /* # of pages wanted of in-q users */
	unsigned int     epp_growing:1;   /* during adding pages */

	/*
	 * indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	 * this is counted based on each time when getting pages from
	 * the pools, not based on time. which means in case that system
	 * is idled for a while but the idle_idx might still be low if no
	 * activities happened in the pools.
	 */
	unsigned long    epp_idle_idx;

	/* last shrink time due to mem tight */
	time64_t         epp_last_shrink;
	time64_t         epp_last_access;

	/*
	 * in-pool pages bookkeeping
	 */
	spinlock_t	 epp_lock;	   /* protect following fields */
	unsigned long    epp_total_pages; /* total pages in pools */
	unsigned long    epp_free_pages;  /* current pages available */

	/*
	 * statistics
	 */
	unsigned long    epp_st_max_pages;      /* # of pages ever reached */
	unsigned int     epp_st_grows;	  /* # of grows */
	unsigned int     epp_st_grow_fails;     /* # of add pages failures */
	unsigned int     epp_st_shrinks;	/* # of shrinks */
	unsigned long    epp_st_access;	 /* # of access */
	unsigned long    epp_st_missings;       /* # of cache missing */
	unsigned long    epp_st_lowfree;	/* lowest free pages reached */
	unsigned int     epp_st_max_wqlen;      /* highest waitqueue length */
	unsigned long       epp_st_max_wait;       /* in jiffies */
	/*
	 * pointers to pools
	 */
	struct page    ***epp_pools;
} page_pools;

/*
 * /proc/fs/lustre/sptlrpc/encrypt_page_pools
 */
int sptlrpc_proc_enc_pool_seq_show(struct seq_file *m, void *v)
{
	spin_lock(&page_pools.epp_lock);

	seq_printf(m,
		   "physical pages:	  %lu\n"
		   "pages per pool:	  %lu\n"
		   "max pages:	       %lu\n"
		   "max pools:	       %u\n"
		   "total pages:	     %lu\n"
		   "total free:	      %lu\n"
		   "idle index:	      %lu/100\n"
		   "last shrink:	     %lds\n"
		   "last access:	     %lds\n"
		   "max pages reached:       %lu\n"
		   "grows:		   %u\n"
		   "grows failure:	   %u\n"
		   "shrinks:		 %u\n"
		   "cache access:	    %lu\n"
		   "cache missing:	   %lu\n"
		   "low free mark:	   %lu\n"
		   "max waitqueue depth:     %u\n"
		   "max wait time:	   %ld/%u\n",
		   totalram_pages,
		   PAGES_PER_POOL,
		   page_pools.epp_max_pages,
		   page_pools.epp_max_pools,
		   page_pools.epp_total_pages,
		   page_pools.epp_free_pages,
		   page_pools.epp_idle_idx,
		   (long)(ktime_get_seconds() - page_pools.epp_last_shrink),
		   (long)(ktime_get_seconds() - page_pools.epp_last_access),
		   page_pools.epp_st_max_pages,
		   page_pools.epp_st_grows,
		   page_pools.epp_st_grow_fails,
		   page_pools.epp_st_shrinks,
		   page_pools.epp_st_access,
		   page_pools.epp_st_missings,
		   page_pools.epp_st_lowfree,
		   page_pools.epp_st_max_wqlen,
		   page_pools.epp_st_max_wait,
		   HZ);

	spin_unlock(&page_pools.epp_lock);

	return 0;
}

static void enc_pools_release_free_pages(long npages)
{
	int p_idx, g_idx;
	int p_idx_max1, p_idx_max2;

	LASSERT(npages > 0);
	LASSERT(npages <= page_pools.epp_free_pages);
	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);

	/* max pool index before the release */
	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;

	page_pools.epp_free_pages -= npages;
	page_pools.epp_total_pages -= npages;

	/* max pool index after the release */
	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
		     ((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	LASSERT(page_pools.epp_pools[p_idx]);

	while (npages--) {
		LASSERT(page_pools.epp_pools[p_idx]);
		LASSERT(page_pools.epp_pools[p_idx][g_idx] != NULL);

		__free_page(page_pools.epp_pools[p_idx][g_idx]);
		page_pools.epp_pools[p_idx][g_idx] = NULL;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	/* free unused pools */
	while (p_idx_max1 < p_idx_max2) {
		LASSERT(page_pools.epp_pools[p_idx_max2]);
		kfree(page_pools.epp_pools[p_idx_max2]);
		page_pools.epp_pools[p_idx_max2] = NULL;
		p_idx_max2--;
	}
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_count(struct shrinker *s,
					    struct shrink_control *sc)
{
	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
		(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
}

/*
 * we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
 */
static unsigned long enc_pools_shrink_scan(struct shrinker *s,
					   struct shrink_control *sc)
{
	spin_lock(&page_pools.epp_lock);
	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
			      page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	if (sc->nr_to_scan > 0) {
		enc_pools_release_free_pages(sc->nr_to_scan);
		CDEBUG(D_SEC, "released %ld pages, %ld left\n",
		       (long)sc->nr_to_scan, page_pools.epp_free_pages);

		page_pools.epp_st_shrinks++;
		page_pools.epp_last_shrink = ktime_get_seconds();
	}
	spin_unlock(&page_pools.epp_lock);

	/*
	 * if no pool access for a long time, we consider it's fully idle.
	 * a little race here is fine.
	 */
	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
		     CACHE_QUIESCENT_PERIOD)) {
		spin_lock(&page_pools.epp_lock);
		page_pools.epp_idle_idx = IDLE_IDX_MAX;
		spin_unlock(&page_pools.epp_lock);
	}

	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	return sc->nr_to_scan;
}

static inline
int npages_to_npools(unsigned long npages)
{
	return (int) ((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
}

/*
 * return how many pages cleaned up.
 */
static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
{
	unsigned long cleaned = 0;
	int i, j;

	for (i = 0; i < npools; i++) {
		if (pools[i]) {
			for (j = 0; j < PAGES_PER_POOL; j++) {
				if (pools[i][j]) {
					__free_page(pools[i][j]);
					cleaned++;
				}
			}
			kfree(pools[i]);
			pools[i] = NULL;
		}
	}

	return cleaned;
}

static inline void enc_pools_wakeup(void)
{
	assert_spin_locked(&page_pools.epp_lock);
	LASSERT(page_pools.epp_waitqlen >= 0);

	if (unlikely(page_pools.epp_waitqlen)) {
		LASSERT(waitqueue_active(&page_pools.epp_waitq));
		wake_up_all(&page_pools.epp_waitq);
	}
}

void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
{
	int p_idx, g_idx;
	int i;

	if (desc->bd_enc_iov == NULL)
		return;

	LASSERT(desc->bd_iov_count > 0);

	spin_lock(&page_pools.epp_lock);

	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;

	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
		page_pools.epp_total_pages);
	LASSERT(page_pools.epp_pools[p_idx]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		LASSERT(desc->bd_enc_iov[i].kiov_page != NULL);
		LASSERT(g_idx != 0 || page_pools.epp_pools[p_idx]);
		LASSERT(page_pools.epp_pools[p_idx][g_idx] == NULL);

		page_pools.epp_pools[p_idx][g_idx] =
					desc->bd_enc_iov[i].kiov_page;

		if (++g_idx == PAGES_PER_POOL) {
			p_idx++;
			g_idx = 0;
		}
	}

	page_pools.epp_free_pages += desc->bd_iov_count;

	enc_pools_wakeup();

	spin_unlock(&page_pools.epp_lock);

	kfree(desc->bd_enc_iov);
	desc->bd_enc_iov = NULL;
}
EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);

static inline void enc_pools_alloc(void)
{
	LASSERT(page_pools.epp_max_pools);
	page_pools.epp_pools =
		libcfs_kvzalloc(page_pools.epp_max_pools *
				sizeof(*page_pools.epp_pools),
				GFP_NOFS);
}

static inline void enc_pools_free(void)
{
	LASSERT(page_pools.epp_max_pools);
	LASSERT(page_pools.epp_pools);

	kvfree(page_pools.epp_pools);
}

static struct shrinker pools_shrinker = {
	.count_objects	= enc_pools_shrink_count,
	.scan_objects	= enc_pools_shrink_scan,
	.seeks		= DEFAULT_SEEKS,
};

int sptlrpc_enc_pool_init(void)
{
	/*
	 * maximum capacity is 1/8 of total physical memory.
	 * is the 1/8 a good number?
	 */
	page_pools.epp_max_pages = totalram_pages / 8;
	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);

	init_waitqueue_head(&page_pools.epp_waitq);
	page_pools.epp_waitqlen = 0;
	page_pools.epp_pages_short = 0;

	page_pools.epp_growing = 0;

	page_pools.epp_idle_idx = 0;
	page_pools.epp_last_shrink = ktime_get_seconds();
	page_pools.epp_last_access = ktime_get_seconds();

	spin_lock_init(&page_pools.epp_lock);
	page_pools.epp_total_pages = 0;
	page_pools.epp_free_pages = 0;

	page_pools.epp_st_max_pages = 0;
	page_pools.epp_st_grows = 0;
	page_pools.epp_st_grow_fails = 0;
	page_pools.epp_st_shrinks = 0;
	page_pools.epp_st_access = 0;
	page_pools.epp_st_missings = 0;
	page_pools.epp_st_lowfree = 0;
	page_pools.epp_st_max_wqlen = 0;
	page_pools.epp_st_max_wait = 0;

	enc_pools_alloc();
	if (page_pools.epp_pools == NULL)
		return -ENOMEM;

	register_shrinker(&pools_shrinker);

	return 0;
}

void sptlrpc_enc_pool_fini(void)
{
	unsigned long cleaned, npools;

	LASSERT(page_pools.epp_pools);
	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);

	unregister_shrinker(&pools_shrinker);

	npools = npages_to_npools(page_pools.epp_total_pages);
	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	LASSERT(cleaned == page_pools.epp_total_pages);

	enc_pools_free();

	if (page_pools.epp_st_access > 0) {
		CDEBUG(D_SEC,
		       "max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
		       page_pools.epp_st_max_pages, page_pools.epp_st_grows,
		       page_pools.epp_st_grow_fails,
		       page_pools.epp_st_shrinks, page_pools.epp_st_access,
		       page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
		       page_pools.epp_st_max_wait, HZ);
	}
}

static int cfs_hash_alg_id[] = {
	[BULK_HASH_ALG_NULL]	= CFS_HASH_ALG_NULL,
	[BULK_HASH_ALG_ADLER32]	= CFS_HASH_ALG_ADLER32,
	[BULK_HASH_ALG_CRC32]	= CFS_HASH_ALG_CRC32,
	[BULK_HASH_ALG_MD5]	= CFS_HASH_ALG_MD5,
	[BULK_HASH_ALG_SHA1]	= CFS_HASH_ALG_SHA1,
	[BULK_HASH_ALG_SHA256]	= CFS_HASH_ALG_SHA256,
	[BULK_HASH_ALG_SHA384]	= CFS_HASH_ALG_SHA384,
	[BULK_HASH_ALG_SHA512]	= CFS_HASH_ALG_SHA512,
};

const char *sptlrpc_get_hash_name(__u8 hash_alg)
{
	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
}
EXPORT_SYMBOL(sptlrpc_get_hash_name);

__u8 sptlrpc_get_hash_alg(const char *algname)
{
	return cfs_crypto_hash_alg(algname);
}
EXPORT_SYMBOL(sptlrpc_get_hash_alg);

int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
{
	struct ptlrpc_bulk_sec_desc *bsd;
	int			  size = msg->lm_buflens[offset];

	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
	if (bsd == NULL) {
		CERROR("Invalid bulk sec desc: size %d\n", size);
		return -EINVAL;
	}

	if (swabbed)
		__swab32s(&bsd->bsd_nob);

	if (unlikely(bsd->bsd_version != 0)) {
		CERROR("Unexpected version %u\n", bsd->bsd_version);
		return -EPROTO;
	}

	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
		CERROR("Invalid type %u\n", bsd->bsd_type);
		return -EPROTO;
	}

	/* FIXME more sanity check here */

	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
		     bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
		CERROR("Invalid svc %u\n", bsd->bsd_svc);
		return -EPROTO;
	}

	return 0;
}
EXPORT_SYMBOL(bulk_sec_desc_unpack);

int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
			      void *buf, int buflen)
{
	struct cfs_crypto_hash_desc *hdesc;
	int hashsize;
	char hashbuf[64];
	unsigned int bufsize;
	int i, err;

	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	LASSERT(buflen >= 4);

	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	if (IS_ERR(hdesc)) {
		CERROR("Unable to initialize checksum hash %s\n",
		       cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
		return PTR_ERR(hdesc);
	}

	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);

	for (i = 0; i < desc->bd_iov_count; i++) {
		cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
				  desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK,
				  desc->bd_iov[i].kiov_len);
	}
	if (hashsize > buflen) {
		bufsize = sizeof(hashbuf);
		err = cfs_crypto_hash_final(hdesc, (unsigned char *)hashbuf,
					    &bufsize);
		memcpy(buf, hashbuf, buflen);
	} else {
		bufsize = buflen;
		err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
	}

	if (err)
		cfs_crypto_hash_final(hdesc, NULL, NULL);
	return err;
}
EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);
Commit	Line	Data
d7e09d03 PT	1	/*
	2	* GPL HEADER START
	3	*
	4	* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 only,
	8	* as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License version 2 for more details (a copy is included
	14	* in the LICENSE file that accompanied this code).
	15	*
	16	* You should have received a copy of the GNU General Public License
	17	* version 2 along with this program; If not, see
	18	* http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
	19	*
	20	* Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
	21	* CA 95054 USA or visit www.sun.com if you need additional information or
	22	* have any questions.
	23	*
	24	* GPL HEADER END
	25	*/
	26	/*
	27	* Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
	28	* Use is subject to license terms.
	29	*
	30	* Copyright (c) 2011, 2012, Intel Corporation.
	31	*/
	32	/*
	33	* This file is part of Lustre, http://www.lustre.org/
	34	* Lustre is a trademark of Sun Microsystems, Inc.
	35	*
	36	* lustre/ptlrpc/sec_bulk.c
	37	*
	38	* Author: Eric Mei <ericm@clusterfs.com>
	39	*/
	40
	41	#define DEBUG_SUBSYSTEM S_SEC
	42
9fdaf8c0	43	#include "../../include/linux/libcfs/libcfs.h"
d7e09d03 PT	44	#include <linux/crypto.h>
d7e09d03 PT	45
e27db149 GKH	46	#include "../include/obd.h"
	47	#include "../include/obd_cksum.h"
	48	#include "../include/obd_class.h"
	49	#include "../include/obd_support.h"
	50	#include "../include/lustre_net.h"
	51	#include "../include/lustre_import.h"
	52	#include "../include/lustre_dlm.h"
	53	#include "../include/lustre_sec.h"
d7e09d03 PT	54
	55	#include "ptlrpc_internal.h"
	56
	57	/****************************************
	58	* bulk encryption page pools *
	59	****************************************/
	60
ae18c5c6 AM	61	#define POINTERS_PER_PAGE (PAGE_CACHE_SIZE / sizeof(void *))
ae18c5c6 AM	62	#define PAGES_PER_POOL (POINTERS_PER_PAGE)
d7e09d03	63
d0bfef31	64	#define IDLE_IDX_MAX (100)
d7e09d03 PT	65	#define IDLE_IDX_WEIGHT (3)
	66
	67	#define CACHE_QUIESCENT_PERIOD (20)
	68
	69	static struct ptlrpc_enc_page_pool {
	70	/*
	71	* constants
	72	*/
	73	unsigned long epp_max_pages; /* maximum pages can hold, const */
	74	unsigned int epp_max_pools; /* number of pools, const */
	75
	76	/*
	77	* wait queue in case of not enough free pages.
	78	*/
	79	wait_queue_head_t epp_waitq; /* waiting threads */
	80	unsigned int epp_waitqlen; /* wait queue length */
	81	unsigned long epp_pages_short; /* # of pages wanted of in-q users */
	82	unsigned int epp_growing:1; /* during adding pages */
	83
	84	/*
	85	* indicating how idle the pools are, from 0 to MAX_IDLE_IDX
	86	* this is counted based on each time when getting pages from
	87	* the pools, not based on time. which means in case that system
	88	* is idled for a while but the idle_idx might still be low if no
	89	* activities happened in the pools.
	90	*/
	91	unsigned long epp_idle_idx;
	92
	93	/* last shrink time due to mem tight */
80018a9e AB	94	time64_t epp_last_shrink;
80018a9e AB	95	time64_t epp_last_access;
d7e09d03 PT	96
	97	/*
	98	* in-pool pages bookkeeping
	99	*/
	100	spinlock_t epp_lock; /* protect following fields */
	101	unsigned long epp_total_pages; /* total pages in pools */
	102	unsigned long epp_free_pages; /* current pages available */
	103
	104	/*
	105	* statistics
	106	*/
	107	unsigned long epp_st_max_pages; /* # of pages ever reached */
	108	unsigned int epp_st_grows; /* # of grows */
	109	unsigned int epp_st_grow_fails; /* # of add pages failures */
	110	unsigned int epp_st_shrinks; /* # of shrinks */
	111	unsigned long epp_st_access; /* # of access */
	112	unsigned long epp_st_missings; /* # of cache missing */
	113	unsigned long epp_st_lowfree; /* lowest free pages reached */
	114	unsigned int epp_st_max_wqlen; /* highest waitqueue length */
a649ad1d	115	unsigned long epp_st_max_wait; /* in jiffies */
d7e09d03 PT	116	/*
	117	* pointers to pools
	118	*/
	119	struct page ***epp_pools;
	120	} page_pools;
	121
d7e09d03 PT	122	/*
	123	* /proc/fs/lustre/sptlrpc/encrypt_page_pools
	124	*/
73bb1da6	125	int sptlrpc_proc_enc_pool_seq_show(struct seq_file m, void v)
d7e09d03	126	{
d7e09d03 PT	127	spin_lock(&page_pools.epp_lock);
d7e09d03 PT	128
91b3a685 JP	129	seq_printf(m,
	130	"physical pages: %lu\n"
	131	"pages per pool: %lu\n"
	132	"max pages: %lu\n"
	133	"max pools: %u\n"
	134	"total pages: %lu\n"
	135	"total free: %lu\n"
	136	"idle index: %lu/100\n"
	137	"last shrink: %lds\n"
	138	"last access: %lds\n"
	139	"max pages reached: %lu\n"
	140	"grows: %u\n"
	141	"grows failure: %u\n"
	142	"shrinks: %u\n"
	143	"cache access: %lu\n"
	144	"cache missing: %lu\n"
	145	"low free mark: %lu\n"
	146	"max waitqueue depth: %u\n"
93d3a405	147	"max wait time: %ld/%u\n",
91b3a685 JP	148	totalram_pages,
	149	PAGES_PER_POOL,
	150	page_pools.epp_max_pages,
	151	page_pools.epp_max_pools,
	152	page_pools.epp_total_pages,
	153	page_pools.epp_free_pages,
	154	page_pools.epp_idle_idx,
80018a9e AB	155	(long)(ktime_get_seconds() - page_pools.epp_last_shrink),
80018a9e AB	156	(long)(ktime_get_seconds() - page_pools.epp_last_access),
91b3a685 JP	157	page_pools.epp_st_max_pages,
	158	page_pools.epp_st_grows,
	159	page_pools.epp_st_grow_fails,
	160	page_pools.epp_st_shrinks,
	161	page_pools.epp_st_access,
	162	page_pools.epp_st_missings,
	163	page_pools.epp_st_lowfree,
	164	page_pools.epp_st_max_wqlen,
	165	page_pools.epp_st_max_wait,
	166	HZ);
d7e09d03 PT	167
d7e09d03 PT	168	spin_unlock(&page_pools.epp_lock);
91b3a685 JP	169
91b3a685 JP	170	return 0;
d7e09d03 PT	171	}
	172
	173	static void enc_pools_release_free_pages(long npages)
	174	{
d0bfef31 CH	175	int p_idx, g_idx;
d0bfef31 CH	176	int p_idx_max1, p_idx_max2;
d7e09d03 PT	177
	178	LASSERT(npages > 0);
	179	LASSERT(npages <= page_pools.epp_free_pages);
	180	LASSERT(page_pools.epp_free_pages <= page_pools.epp_total_pages);
	181
	182	/* max pool index before the release */
	183	p_idx_max2 = (page_pools.epp_total_pages - 1) / PAGES_PER_POOL;
	184
	185	page_pools.epp_free_pages -= npages;
	186	page_pools.epp_total_pages -= npages;
	187
	188	/* max pool index after the release */
	189	p_idx_max1 = page_pools.epp_total_pages == 0 ? -1 :
	190	((page_pools.epp_total_pages - 1) / PAGES_PER_POOL);
	191
	192	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	193	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	194	LASSERT(page_pools.epp_pools[p_idx]);
	195
	196	while (npages--) {
	197	LASSERT(page_pools.epp_pools[p_idx]);
	198	LASSERT(page_pools.epp_pools[p_idx][g_idx] != NULL);
	199
	200	__free_page(page_pools.epp_pools[p_idx][g_idx]);
	201	page_pools.epp_pools[p_idx][g_idx] = NULL;
	202
	203	if (++g_idx == PAGES_PER_POOL) {
	204	p_idx++;
	205	g_idx = 0;
	206	}
9076b09e	207	}
d7e09d03 PT	208
	209	/* free unused pools */
	210	while (p_idx_max1 < p_idx_max2) {
	211	LASSERT(page_pools.epp_pools[p_idx_max2]);
9ae10597	212	kfree(page_pools.epp_pools[p_idx_max2]);
d7e09d03 PT	213	page_pools.epp_pools[p_idx_max2] = NULL;
	214	p_idx_max2--;
	215	}
	216	}
	217
	218	/*
d7e09d03 PT	219	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
d7e09d03 PT	220	*/
3bb22ec5 PT	221	static unsigned long enc_pools_shrink_count(struct shrinker *s,
3bb22ec5 PT	222	struct shrink_control *sc)
d7e09d03	223	{
3bb22ec5 PT	224	/*
	225	* if no pool access for a long time, we consider it's fully idle.
	226	* a little race here is fine.
	227	*/
80018a9e	228	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
3bb22ec5	229	CACHE_QUIESCENT_PERIOD)) {
d7e09d03	230	spin_lock(&page_pools.epp_lock);
3bb22ec5	231	page_pools.epp_idle_idx = IDLE_IDX_MAX;
d7e09d03 PT	232	spin_unlock(&page_pools.epp_lock);
	233	}
	234
3bb22ec5 PT	235	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
	236	return max((int)page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES, 0) *
	237	(IDLE_IDX_MAX - page_pools.epp_idle_idx) / IDLE_IDX_MAX;
	238	}
	239
	240	/*
	241	* we try to keep at least PTLRPC_MAX_BRW_PAGES pages in the pool.
	242	*/
	243	static unsigned long enc_pools_shrink_scan(struct shrinker *s,
	244	struct shrink_control *sc)
	245	{
	246	spin_lock(&page_pools.epp_lock);
	247	sc->nr_to_scan = min_t(unsigned long, sc->nr_to_scan,
	248	page_pools.epp_free_pages - PTLRPC_MAX_BRW_PAGES);
	249	if (sc->nr_to_scan > 0) {
	250	enc_pools_release_free_pages(sc->nr_to_scan);
	251	CDEBUG(D_SEC, "released %ld pages, %ld left\n",
	252	(long)sc->nr_to_scan, page_pools.epp_free_pages);
	253
	254	page_pools.epp_st_shrinks++;
80018a9e	255	page_pools.epp_last_shrink = ktime_get_seconds();
3bb22ec5 PT	256	}
	257	spin_unlock(&page_pools.epp_lock);
	258
d7e09d03 PT	259	/*
	260	* if no pool access for a long time, we consider it's fully idle.
	261	* a little race here is fine.
	262	*/
80018a9e	263	if (unlikely(ktime_get_seconds() - page_pools.epp_last_access >
d7e09d03 PT	264	CACHE_QUIESCENT_PERIOD)) {
	265	spin_lock(&page_pools.epp_lock);
	266	page_pools.epp_idle_idx = IDLE_IDX_MAX;
	267	spin_unlock(&page_pools.epp_lock);
	268	}
	269
	270	LASSERT(page_pools.epp_idle_idx <= IDLE_IDX_MAX);
3bb22ec5	271	return sc->nr_to_scan;
d7e09d03 PT	272	}
	273
	274	static inline
	275	int npages_to_npools(unsigned long npages)
	276	{
	277	return (int) ((npages + PAGES_PER_POOL - 1) / PAGES_PER_POOL);
	278	}
	279
	280	/*
	281	* return how many pages cleaned up.
	282	*/
	283	static unsigned long enc_pools_cleanup(struct page ***pools, int npools)
	284	{
	285	unsigned long cleaned = 0;
d0bfef31	286	int i, j;
d7e09d03 PT	287
	288	for (i = 0; i < npools; i++) {
	289	if (pools[i]) {
	290	for (j = 0; j < PAGES_PER_POOL; j++) {
	291	if (pools[i][j]) {
	292	__free_page(pools[i][j]);
	293	cleaned++;
	294	}
	295	}
9ae10597	296	kfree(pools[i]);
d7e09d03 PT	297	pools[i] = NULL;
	298	}
	299	}
	300
	301	return cleaned;
	302	}
	303
d7e09d03 PT	304	static inline void enc_pools_wakeup(void)
d7e09d03 PT	305	{
5e42bc9d	306	assert_spin_locked(&page_pools.epp_lock);
d7e09d03 PT	307	LASSERT(page_pools.epp_waitqlen >= 0);
	308
	309	if (unlikely(page_pools.epp_waitqlen)) {
	310	LASSERT(waitqueue_active(&page_pools.epp_waitq));
	311	wake_up_all(&page_pools.epp_waitq);
	312	}
	313	}
	314
d7e09d03 PT	315	void sptlrpc_enc_pool_put_pages(struct ptlrpc_bulk_desc *desc)
d7e09d03 PT	316	{
d0bfef31 CH	317	int p_idx, g_idx;
d0bfef31 CH	318	int i;
d7e09d03 PT	319
	320	if (desc->bd_enc_iov == NULL)
	321	return;
	322
	323	LASSERT(desc->bd_iov_count > 0);
	324
	325	spin_lock(&page_pools.epp_lock);
	326
	327	p_idx = page_pools.epp_free_pages / PAGES_PER_POOL;
	328	g_idx = page_pools.epp_free_pages % PAGES_PER_POOL;
	329
	330	LASSERT(page_pools.epp_free_pages + desc->bd_iov_count <=
	331	page_pools.epp_total_pages);
	332	LASSERT(page_pools.epp_pools[p_idx]);
	333
	334	for (i = 0; i < desc->bd_iov_count; i++) {
	335	LASSERT(desc->bd_enc_iov[i].kiov_page != NULL);
	336	LASSERT(g_idx != 0 \|\| page_pools.epp_pools[p_idx]);
	337	LASSERT(page_pools.epp_pools[p_idx][g_idx] == NULL);
	338
	339	page_pools.epp_pools[p_idx][g_idx] =
	340	desc->bd_enc_iov[i].kiov_page;
	341
	342	if (++g_idx == PAGES_PER_POOL) {
	343	p_idx++;
	344	g_idx = 0;
	345	}
	346	}
	347
	348	page_pools.epp_free_pages += desc->bd_iov_count;
	349
	350	enc_pools_wakeup();
	351
	352	spin_unlock(&page_pools.epp_lock);
	353
9ae10597	354	kfree(desc->bd_enc_iov);
d7e09d03 PT	355	desc->bd_enc_iov = NULL;
	356	}
	357	EXPORT_SYMBOL(sptlrpc_enc_pool_put_pages);
	358
d7e09d03 PT	359	static inline void enc_pools_alloc(void)
	360	{
	361	LASSERT(page_pools.epp_max_pools);
ee0ec194 JL	362	page_pools.epp_pools =
	363	libcfs_kvzalloc(page_pools.epp_max_pools *
	364	sizeof(*page_pools.epp_pools),
	365	GFP_NOFS);
d7e09d03 PT	366	}
	367
	368	static inline void enc_pools_free(void)
	369	{
	370	LASSERT(page_pools.epp_max_pools);
	371	LASSERT(page_pools.epp_pools);
	372
ee0ec194	373	kvfree(page_pools.epp_pools);
d7e09d03 PT	374	}
d7e09d03 PT	375
3bb22ec5 PT	376	static struct shrinker pools_shrinker = {
	377	.count_objects = enc_pools_shrink_count,
	378	.scan_objects = enc_pools_shrink_scan,
	379	.seeks = DEFAULT_SEEKS,
	380	};
	381
d7e09d03 PT	382	int sptlrpc_enc_pool_init(void)
	383	{
	384	/*
	385	* maximum capacity is 1/8 of total physical memory.
	386	* is the 1/8 a good number?
	387	*/
4f6cc9ab	388	page_pools.epp_max_pages = totalram_pages / 8;
d7e09d03 PT	389	page_pools.epp_max_pools = npages_to_npools(page_pools.epp_max_pages);
	390
	391	init_waitqueue_head(&page_pools.epp_waitq);
	392	page_pools.epp_waitqlen = 0;
	393	page_pools.epp_pages_short = 0;
	394
	395	page_pools.epp_growing = 0;
	396
	397	page_pools.epp_idle_idx = 0;
80018a9e AB	398	page_pools.epp_last_shrink = ktime_get_seconds();
80018a9e AB	399	page_pools.epp_last_access = ktime_get_seconds();
d7e09d03 PT	400
	401	spin_lock_init(&page_pools.epp_lock);
	402	page_pools.epp_total_pages = 0;
	403	page_pools.epp_free_pages = 0;
	404
	405	page_pools.epp_st_max_pages = 0;
	406	page_pools.epp_st_grows = 0;
	407	page_pools.epp_st_grow_fails = 0;
	408	page_pools.epp_st_shrinks = 0;
	409	page_pools.epp_st_access = 0;
	410	page_pools.epp_st_missings = 0;
	411	page_pools.epp_st_lowfree = 0;
	412	page_pools.epp_st_max_wqlen = 0;
	413	page_pools.epp_st_max_wait = 0;
	414
	415	enc_pools_alloc();
	416	if (page_pools.epp_pools == NULL)
	417	return -ENOMEM;
	418
3bb22ec5	419	register_shrinker(&pools_shrinker);
d7e09d03 PT	420
	421	return 0;
	422	}
	423
	424	void sptlrpc_enc_pool_fini(void)
	425	{
	426	unsigned long cleaned, npools;
	427
d7e09d03 PT	428	LASSERT(page_pools.epp_pools);
	429	LASSERT(page_pools.epp_total_pages == page_pools.epp_free_pages);
	430
3bb22ec5	431	unregister_shrinker(&pools_shrinker);
d7e09d03 PT	432
	433	npools = npages_to_npools(page_pools.epp_total_pages);
	434	cleaned = enc_pools_cleanup(page_pools.epp_pools, npools);
	435	LASSERT(cleaned == page_pools.epp_total_pages);
	436
	437	enc_pools_free();
	438
	439	if (page_pools.epp_st_access > 0) {
	440	CDEBUG(D_SEC,
93d3a405	441	"max pages %lu, grows %u, grow fails %u, shrinks %u, access %lu, missing %lu, max qlen %u, max wait %ld/%d\n",
d7e09d03 PT	442	page_pools.epp_st_max_pages, page_pools.epp_st_grows,
	443	page_pools.epp_st_grow_fails,
	444	page_pools.epp_st_shrinks, page_pools.epp_st_access,
	445	page_pools.epp_st_missings, page_pools.epp_st_max_wqlen,
	446	page_pools.epp_st_max_wait, HZ);
	447	}
	448	}
	449
d7e09d03 PT	450	static int cfs_hash_alg_id[] = {
	451	[BULK_HASH_ALG_NULL] = CFS_HASH_ALG_NULL,
	452	[BULK_HASH_ALG_ADLER32] = CFS_HASH_ALG_ADLER32,
	453	[BULK_HASH_ALG_CRC32] = CFS_HASH_ALG_CRC32,
	454	[BULK_HASH_ALG_MD5] = CFS_HASH_ALG_MD5,
	455	[BULK_HASH_ALG_SHA1] = CFS_HASH_ALG_SHA1,
	456	[BULK_HASH_ALG_SHA256] = CFS_HASH_ALG_SHA256,
	457	[BULK_HASH_ALG_SHA384] = CFS_HASH_ALG_SHA384,
	458	[BULK_HASH_ALG_SHA512] = CFS_HASH_ALG_SHA512,
	459	};
c9f6bb96	460
aff9d8e8	461	const char *sptlrpc_get_hash_name(__u8 hash_alg)
d7e09d03 PT	462	{
	463	return cfs_crypto_hash_name(cfs_hash_alg_id[hash_alg]);
	464	}
	465	EXPORT_SYMBOL(sptlrpc_get_hash_name);
	466
	467	__u8 sptlrpc_get_hash_alg(const char *algname)
	468	{
	469	return cfs_crypto_hash_alg(algname);
	470	}
	471	EXPORT_SYMBOL(sptlrpc_get_hash_alg);
	472
	473	int bulk_sec_desc_unpack(struct lustre_msg *msg, int offset, int swabbed)
	474	{
	475	struct ptlrpc_bulk_sec_desc *bsd;
	476	int size = msg->lm_buflens[offset];
	477
	478	bsd = lustre_msg_buf(msg, offset, sizeof(*bsd));
	479	if (bsd == NULL) {
	480	CERROR("Invalid bulk sec desc: size %d\n", size);
	481	return -EINVAL;
	482	}
	483
acf60c3d	484	if (swabbed)
d7e09d03	485	__swab32s(&bsd->bsd_nob);
d7e09d03 PT	486
	487	if (unlikely(bsd->bsd_version != 0)) {
	488	CERROR("Unexpected version %u\n", bsd->bsd_version);
	489	return -EPROTO;
	490	}
	491
	492	if (unlikely(bsd->bsd_type >= SPTLRPC_BULK_MAX)) {
	493	CERROR("Invalid type %u\n", bsd->bsd_type);
	494	return -EPROTO;
	495	}
	496
	497	/* FIXME more sanity check here */
	498
	499	if (unlikely(bsd->bsd_svc != SPTLRPC_BULK_SVC_NULL &&
	500	bsd->bsd_svc != SPTLRPC_BULK_SVC_INTG &&
	501	bsd->bsd_svc != SPTLRPC_BULK_SVC_PRIV)) {
	502	CERROR("Invalid svc %u\n", bsd->bsd_svc);
	503	return -EPROTO;
	504	}
	505
	506	return 0;
	507	}
	508	EXPORT_SYMBOL(bulk_sec_desc_unpack);
	509
	510	int sptlrpc_get_bulk_checksum(struct ptlrpc_bulk_desc *desc, __u8 alg,
	511	void *buf, int buflen)
	512	{
d0bfef31 CH	513	struct cfs_crypto_hash_desc *hdesc;
	514	int hashsize;
	515	char hashbuf[64];
	516	unsigned int bufsize;
	517	int i, err;
d7e09d03 PT	518
	519	LASSERT(alg > BULK_HASH_ALG_NULL && alg < BULK_HASH_ALG_MAX);
	520	LASSERT(buflen >= 4);
	521
	522	hdesc = cfs_crypto_hash_init(cfs_hash_alg_id[alg], NULL, 0);
	523	if (IS_ERR(hdesc)) {
	524	CERROR("Unable to initialize checksum hash %s\n",
	525	cfs_crypto_hash_name(cfs_hash_alg_id[alg]));
	526	return PTR_ERR(hdesc);
	527	}
	528
	529	hashsize = cfs_crypto_hash_digestsize(cfs_hash_alg_id[alg]);
	530
	531	for (i = 0; i < desc->bd_iov_count; i++) {
	532	cfs_crypto_hash_update_page(hdesc, desc->bd_iov[i].kiov_page,
	533	desc->bd_iov[i].kiov_offset & ~CFS_PAGE_MASK,
	534	desc->bd_iov[i].kiov_len);
	535	}
	536	if (hashsize > buflen) {
	537	bufsize = sizeof(hashbuf);
	538	err = cfs_crypto_hash_final(hdesc, (unsigned char *)hashbuf,
	539	&bufsize);
	540	memcpy(buf, hashbuf, buflen);
	541	} else {
	542	bufsize = buflen;
3df82ddc	543	err = cfs_crypto_hash_final(hdesc, buf, &bufsize);
d7e09d03 PT	544	}
	545
	546	if (err)
	547	cfs_crypto_hash_final(hdesc, NULL, NULL);
	548	return err;
	549	}
	550	EXPORT_SYMBOL(sptlrpc_get_bulk_checksum);