diff options
Diffstat (limited to 'block')
| -rw-r--r-- | block/Kconfig | 17 | ||||
| -rw-r--r-- | block/Makefile | 3 | ||||
| -rw-r--r-- | block/bio-crypt-ctx.c | 142 | ||||
| -rw-r--r-- | block/bio.c | 26 | ||||
| -rw-r--r-- | block/blk-core.c | 34 | ||||
| -rw-r--r-- | block/blk-crypto-fallback.c | 644 | ||||
| -rw-r--r-- | block/blk-crypto-internal.h | 67 | ||||
| -rw-r--r-- | block/blk-crypto.c | 306 | ||||
| -rw-r--r-- | block/blk-merge.c | 11 | ||||
| -rw-r--r-- | block/cfq-iosched.c | 81 | ||||
| -rw-r--r-- | block/keyslot-manager.c | 664 |
11 files changed, 1976 insertions, 19 deletions
diff --git a/block/Kconfig b/block/Kconfig index 28ec55752b68..4d9bcb951d83 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -184,6 +184,23 @@ config BLK_SED_OPAL Enabling this option enables users to setup/unlock/lock Locking ranges for SED devices using the Opal protocol. +config BLK_INLINE_ENCRYPTION + bool "Enable inline encryption support in block layer" + help + Build the blk-crypto subsystem. Enabling this lets the + block layer handle encryption, so users can take + advantage of inline encryption hardware if present. + +config BLK_INLINE_ENCRYPTION_FALLBACK + bool "Enable crypto API fallback for blk-crypto" + depends on BLK_INLINE_ENCRYPTION + select CRYPTO + select CRYPTO_BLKCIPHER + help + Enabling this lets the block layer handle inline encryption + by falling back to the kernel crypto API when inline + encryption hardware is not present. + menu "Partition Types" source "block/partitions/Kconfig" diff --git a/block/Makefile b/block/Makefile index 6a56303b9925..ab14055d8222 100644 --- a/block/Makefile +++ b/block/Makefile @@ -35,3 +35,6 @@ obj-$(CONFIG_BLK_DEV_ZONED) += blk-zoned.o obj-$(CONFIG_BLK_WBT) += blk-wbt.o obj-$(CONFIG_BLK_DEBUG_FS) += blk-mq-debugfs.o obj-$(CONFIG_BLK_SED_OPAL) += sed-opal.o +obj-$(CONFIG_BLK_INLINE_ENCRYPTION) += keyslot-manager.o bio-crypt-ctx.o \ + blk-crypto.o +obj-$(CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK) += blk-crypto-fallback.o
\ No newline at end of file diff --git a/block/bio-crypt-ctx.c b/block/bio-crypt-ctx.c new file mode 100644 index 000000000000..75008b2afea2 --- /dev/null +++ b/block/bio-crypt-ctx.c @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +#include <linux/bio.h> +#include <linux/blkdev.h> +#include <linux/keyslot-manager.h> +#include <linux/module.h> +#include <linux/slab.h> + +#include "blk-crypto-internal.h" + +static int num_prealloc_crypt_ctxs = 128; + +module_param(num_prealloc_crypt_ctxs, int, 0444); +MODULE_PARM_DESC(num_prealloc_crypt_ctxs, + "Number of bio crypto contexts to preallocate"); + +static struct kmem_cache *bio_crypt_ctx_cache; +static mempool_t *bio_crypt_ctx_pool; + +int __init bio_crypt_ctx_init(void) +{ + size_t i; + + bio_crypt_ctx_cache = KMEM_CACHE(bio_crypt_ctx, 0); + if (!bio_crypt_ctx_cache) + return -ENOMEM; + + bio_crypt_ctx_pool = mempool_create_slab_pool(num_prealloc_crypt_ctxs, + bio_crypt_ctx_cache); + if (!bio_crypt_ctx_pool) + return -ENOMEM; + + /* This is assumed in various places. */ + BUILD_BUG_ON(BLK_ENCRYPTION_MODE_INVALID != 0); + + /* Sanity check that no algorithm exceeds the defined limits. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) { + BUG_ON(blk_crypto_modes[i].keysize > BLK_CRYPTO_MAX_KEY_SIZE); + BUG_ON(blk_crypto_modes[i].ivsize > BLK_CRYPTO_MAX_IV_SIZE); + } + + return 0; +} + +struct bio_crypt_ctx *bio_crypt_alloc_ctx(gfp_t gfp_mask) +{ + return mempool_alloc(bio_crypt_ctx_pool, gfp_mask); +} +EXPORT_SYMBOL_GPL(bio_crypt_alloc_ctx); + +void bio_crypt_free_ctx(struct bio *bio) +{ + mempool_free(bio->bi_crypt_context, bio_crypt_ctx_pool); + bio->bi_crypt_context = NULL; +} + +void bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask) +{ + const struct bio_crypt_ctx *src_bc = src->bi_crypt_context; + + bio_clone_skip_dm_default_key(dst, src); + + /* + * If a bio is fallback_crypted, then it will be decrypted when + * bio_endio is called. As we only want the data to be decrypted once, + * copies of the bio must not have have a crypt context. + */ + if (!src_bc || bio_crypt_fallback_crypted(src_bc)) + return; + + dst->bi_crypt_context = bio_crypt_alloc_ctx(gfp_mask); + *dst->bi_crypt_context = *src_bc; + + if (src_bc->bc_keyslot >= 0) + keyslot_manager_get_slot(src_bc->bc_ksm, src_bc->bc_keyslot); +} +EXPORT_SYMBOL_GPL(bio_crypt_clone); + +bool bio_crypt_should_process(struct request *rq) +{ + struct bio *bio = rq->bio; + + if (!bio || !bio->bi_crypt_context) + return false; + + return rq->q->ksm == bio->bi_crypt_context->bc_ksm; +} +EXPORT_SYMBOL_GPL(bio_crypt_should_process); + +/* + * Checks that two bio crypt contexts are compatible - i.e. that + * they are mergeable except for data_unit_num continuity. + */ +bool bio_crypt_ctx_compatible(struct bio *b_1, struct bio *b_2) +{ + struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; + struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; + + if (!bc1) + return !bc2; + return bc2 && bc1->bc_key == bc2->bc_key; +} + +/* + * Checks that two bio crypt contexts are compatible, and also + * that their data_unit_nums are continuous (and can hence be merged) + * in the order b_1 followed by b_2. + */ +bool bio_crypt_ctx_mergeable(struct bio *b_1, unsigned int b1_bytes, + struct bio *b_2) +{ + struct bio_crypt_ctx *bc1 = b_1->bi_crypt_context; + struct bio_crypt_ctx *bc2 = b_2->bi_crypt_context; + + if (!bio_crypt_ctx_compatible(b_1, b_2)) + return false; + + return !bc1 || bio_crypt_dun_is_contiguous(bc1, b1_bytes, bc2->bc_dun); +} + +void bio_crypt_ctx_release_keyslot(struct bio_crypt_ctx *bc) +{ + keyslot_manager_put_slot(bc->bc_ksm, bc->bc_keyslot); + bc->bc_ksm = NULL; + bc->bc_keyslot = -1; +} + +int bio_crypt_ctx_acquire_keyslot(struct bio_crypt_ctx *bc, + struct keyslot_manager *ksm) +{ + int slot = keyslot_manager_get_slot_for_key(ksm, bc->bc_key); + + if (slot < 0) + return slot; + + bc->bc_keyslot = slot; + bc->bc_ksm = ksm; + return 0; +} diff --git a/block/bio.c b/block/bio.c index 30df1b45dde8..a7089b2d2943 100644 --- a/block/bio.c +++ b/block/bio.c @@ -28,6 +28,7 @@ #include <linux/mempool.h> #include <linux/workqueue.h> #include <linux/cgroup.h> +#include <linux/blk-crypto.h> #include <trace/events/block.h> #include "blk.h" @@ -243,6 +244,8 @@ fallback: void bio_uninit(struct bio *bio) { bio_disassociate_task(bio); + + bio_crypt_free_ctx(bio); } EXPORT_SYMBOL(bio_uninit); @@ -628,15 +631,12 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs) __bio_clone_fast(b, bio); - if (bio_integrity(bio)) { - int ret; - - ret = bio_integrity_clone(b, bio, gfp_mask); + bio_crypt_clone(b, bio, gfp_mask); - if (ret < 0) { - bio_put(b); - return NULL; - } + if (bio_integrity(bio) && + bio_integrity_clone(b, bio, gfp_mask) < 0) { + bio_put(b); + return NULL; } return b; @@ -704,6 +704,8 @@ struct bio *bio_clone_bioset(struct bio *bio_src, gfp_t gfp_mask, break; } + bio_crypt_clone(bio, bio_src, gfp_mask); + if (bio_integrity(bio_src)) { int ret; @@ -876,6 +878,9 @@ void __bio_add_page(struct bio *bio, struct page *page, bio->bi_iter.bi_size += len; bio->bi_vcnt++; + + if (!bio_flagged(bio, BIO_WORKINGSET) && unlikely(PageWorkingset(page))) + bio_set_flag(bio, BIO_WORKINGSET); } EXPORT_SYMBOL_GPL(__bio_add_page); @@ -1032,6 +1037,7 @@ void bio_advance(struct bio *bio, unsigned bytes) if (bio_integrity(bio)) bio_integrity_advance(bio, bytes); + bio_crypt_advance(bio, bytes); bio_advance_iter(bio, &bio->bi_iter, bytes); } EXPORT_SYMBOL(bio_advance); @@ -1889,6 +1895,10 @@ void bio_endio(struct bio *bio) again: if (!bio_remaining_done(bio)) return; + + if (!blk_crypto_endio(bio)) + return; + if (!bio_integrity_endio(bio)) return; diff --git a/block/blk-core.c b/block/blk-core.c index 2407c898ba7d..41cb061c29f2 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -34,6 +34,8 @@ #include <linux/pm_runtime.h> #include <linux/blk-cgroup.h> #include <linux/debugfs.h> +#include <linux/psi.h> +#include <linux/blk-crypto.h> #define CREATE_TRACE_POINTS #include <trace/events/block.h> @@ -2226,7 +2228,9 @@ blk_qc_t generic_make_request(struct bio *bio) /* Create a fresh bio_list for all subordinate requests */ bio_list_on_stack[1] = bio_list_on_stack[0]; bio_list_init(&bio_list_on_stack[0]); - ret = q->make_request_fn(q, bio); + + if (!blk_crypto_submit_bio(&bio)) + ret = q->make_request_fn(q, bio); blk_queue_exit(q); @@ -2271,6 +2275,10 @@ EXPORT_SYMBOL(generic_make_request); */ blk_qc_t submit_bio(struct bio *bio) { + bool workingset_read = false; + unsigned long pflags; + blk_qc_t ret; + /* * If it's a regular read/write or a barrier with data attached, * go through the normal accounting stuff before submission. @@ -2286,6 +2294,8 @@ blk_qc_t submit_bio(struct bio *bio) if (op_is_write(bio_op(bio))) { count_vm_events(PGPGOUT, count); } else { + if (bio_flagged(bio, BIO_WORKINGSET)) + workingset_read = true; task_io_account_read(bio->bi_iter.bi_size); count_vm_events(PGPGIN, count); } @@ -2300,7 +2310,21 @@ blk_qc_t submit_bio(struct bio *bio) } } - return generic_make_request(bio); + /* + * If we're reading data that is part of the userspace + * workingset, count submission time as memory stall. When the + * device is congested, or the submitting cgroup IO-throttled, + * submission can be a significant part of overall IO time. + */ + if (workingset_read) + psi_memstall_enter(&pflags); + + ret = generic_make_request(bio); + + if (workingset_read) + psi_memstall_leave(&pflags); + + return ret; } EXPORT_SYMBOL(submit_bio); @@ -3648,5 +3672,11 @@ int __init blk_dev_init(void) blk_debugfs_root = debugfs_create_dir("block", NULL); #endif + if (bio_crypt_ctx_init() < 0) + panic("Failed to allocate mem for bio crypt ctxs\n"); + + if (blk_crypto_fallback_init() < 0) + panic("Failed to init blk-crypto-fallback\n"); + return 0; } diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c new file mode 100644 index 000000000000..ad83e1077ba3 --- /dev/null +++ b/block/blk-crypto-fallback.c @@ -0,0 +1,644 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +/* + * Refer to Documentation/block/inline-encryption.rst for detailed explanation. + */ + +#define pr_fmt(fmt) "blk-crypto-fallback: " fmt + +#include <crypto/skcipher.h> +#include <linux/blk-cgroup.h> +#include <linux/blk-crypto.h> +#include <linux/crypto.h> +#include <linux/keyslot-manager.h> +#include <linux/mempool.h> +#include <linux/module.h> +#include <linux/random.h> + +#include "blk-crypto-internal.h" + +static unsigned int num_prealloc_bounce_pg = 32; +module_param(num_prealloc_bounce_pg, uint, 0); +MODULE_PARM_DESC(num_prealloc_bounce_pg, + "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); + +static unsigned int blk_crypto_num_keyslots = 100; +module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); +MODULE_PARM_DESC(num_keyslots, + "Number of keyslots for the blk-crypto crypto API fallback"); + +static unsigned int num_prealloc_fallback_crypt_ctxs = 128; +module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); +MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, + "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); + +struct bio_fallback_crypt_ctx { + struct bio_crypt_ctx crypt_ctx; + /* + * Copy of the bvec_iter when this bio was submitted. + * We only want to en/decrypt the part of the bio as described by the + * bvec_iter upon submission because bio might be split before being + * resubmitted + */ + struct bvec_iter crypt_iter; + u64 fallback_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; +}; + +/* The following few vars are only used during the crypto API fallback */ +static struct kmem_cache *bio_fallback_crypt_ctx_cache; +static mempool_t *bio_fallback_crypt_ctx_pool; + +/* + * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate + * all of a mode's tfms when that mode starts being used. Since each mode may + * need all the keyslots at some point, each mode needs its own tfm for each + * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to + * match the behavior of real inline encryption hardware (which only supports a + * single encryption context per keyslot), we only allow one tfm per keyslot to + * be used at a time - the rest of the unused tfms have their keys cleared. + */ +static DEFINE_MUTEX(tfms_init_lock); +static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; + +struct blk_crypto_decrypt_work { + struct work_struct work; + struct bio *bio; +}; + +static struct blk_crypto_keyslot { + struct crypto_skcipher *tfm; + enum blk_crypto_mode_num crypto_mode; + struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; +} *blk_crypto_keyslots; + +/* The following few vars are only used during the crypto API fallback */ +static struct keyslot_manager *blk_crypto_ksm; +static struct workqueue_struct *blk_crypto_wq; +static mempool_t *blk_crypto_bounce_page_pool; +static struct kmem_cache *blk_crypto_decrypt_work_cache; + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return bc && bc->bc_ksm == blk_crypto_ksm; +} + +/* + * This is the key we set when evicting a keyslot. This *should* be the all 0's + * key, but AES-XTS rejects that key, so we use some random bytes instead. + */ +static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; + +static void blk_crypto_evict_keyslot(unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; + int err; + + WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); + + /* Clear the key in the skcipher */ + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + blk_crypto_modes[crypto_mode].keysize); + WARN_ON(err); + slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; +} + +static int blk_crypto_keyslot_program(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + struct blk_crypto_keyslot *slotp = &blk_crypto_keyslots[slot]; + const enum blk_crypto_mode_num crypto_mode = key->crypto_mode; + int err; + + if (crypto_mode != slotp->crypto_mode && + slotp->crypto_mode != BLK_ENCRYPTION_MODE_INVALID) { + blk_crypto_evict_keyslot(slot); + } + + if (!slotp->tfms[crypto_mode]) + return -ENOMEM; + slotp->crypto_mode = crypto_mode; + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], key->raw, + key->size); + if (err) { + blk_crypto_evict_keyslot(slot); + return err; + } + return 0; +} + +static int blk_crypto_keyslot_evict(struct keyslot_manager *ksm, + const struct blk_crypto_key *key, + unsigned int slot) +{ + blk_crypto_evict_keyslot(slot); + return 0; +} + +/* + * The crypto API fallback KSM ops - only used for a bio when it specifies a + * blk_crypto_mode for which we failed to get a keyslot in the device's inline + * encryption hardware (which probably means the device doesn't have inline + * encryption hardware that supports that crypto mode). + */ +static const struct keyslot_mgmt_ll_ops blk_crypto_ksm_ll_ops = { + .keyslot_program = blk_crypto_keyslot_program, + .keyslot_evict = blk_crypto_keyslot_evict, +}; + +static void blk_crypto_encrypt_endio(struct bio *enc_bio) +{ + struct bio *src_bio = enc_bio->bi_private; + int i; + + for (i = 0; i < enc_bio->bi_vcnt; i++) + mempool_free(enc_bio->bi_io_vec[i].bv_page, + blk_crypto_bounce_page_pool); + + src_bio->bi_status = enc_bio->bi_status; + + bio_put(enc_bio); + bio_endio(src_bio); +} + +static struct bio *blk_crypto_clone_bio(struct bio *bio_src) +{ + struct bvec_iter iter; + struct bio_vec bv; + struct bio *bio; + + bio = bio_alloc_bioset(GFP_NOIO, bio_segments(bio_src), NULL); + if (!bio) + return NULL; + bio->bi_disk = bio_src->bi_disk; + bio->bi_opf = bio_src->bi_opf; + bio->bi_ioprio = bio_src->bi_ioprio; + bio->bi_write_hint = bio_src->bi_write_hint; + bio->bi_iter.bi_sector = bio_src->bi_iter.bi_sector; + bio->bi_iter.bi_size = bio_src->bi_iter.bi_size; + + bio_for_each_segment(bv, bio_src, iter) + bio->bi_io_vec[bio->bi_vcnt++] = bv; + + if (bio_integrity(bio_src) && + bio_integrity_clone(bio, bio_src, GFP_NOIO) < 0) { + bio_put(bio); + return NULL; + } + + bio_clone_blkcg_association(bio, bio_src); + + bio_clone_skip_dm_default_key(bio, bio_src); + + return bio; +} + +static int blk_crypto_alloc_cipher_req(struct bio *src_bio, + struct skcipher_request **ciph_req_ret, + struct crypto_wait *wait) +{ + struct skcipher_request *ciph_req; + const struct blk_crypto_keyslot *slotp; + + slotp = &blk_crypto_keyslots[src_bio->bi_crypt_context->bc_keyslot]; + ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], + GFP_NOIO); + if (!ciph_req) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + skcipher_request_set_callback(ciph_req, + CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, wait); + *ciph_req_ret = ciph_req; + return 0; +} + +static int blk_crypto_split_bio_if_needed(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + unsigned int i = 0; + unsigned int num_sectors = 0; + struct bio_vec bv; + struct bvec_iter iter; + + bio_for_each_segment(bv, bio, iter) { + num_sectors += bv.bv_len >> SECTOR_SHIFT; + if (++i == BIO_MAX_PAGES) + break; + } + if (num_sectors < bio_sectors(bio)) { + struct bio *split_bio; + + split_bio = bio_split(bio, num_sectors, GFP_NOIO, NULL); + if (!split_bio) { + bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + bio_chain(split_bio, bio); + generic_make_request(bio); + *bio_ptr = split_bio; + } + return 0; +} + +union blk_crypto_iv { + __le64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + u8 bytes[BLK_CRYPTO_MAX_IV_SIZE]; +}; + +static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], + union blk_crypto_iv *iv) +{ + int i; + + for (i = 0; i < BLK_CRYPTO_DUN_ARRAY_SIZE; i++) + iv->dun[i] = cpu_to_le64(dun[i]); +} + +/* + * The crypto API fallback's encryption routine. + * Allocate a bounce bio for encryption, encrypt the input bio using crypto API, + * and replace *bio_ptr with the bounce bio. May split input bio if it's too + * large. + */ +static int blk_crypto_encrypt_bio(struct bio **bio_ptr) +{ + struct bio *src_bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist src, dst; + struct bio *enc_bio; + unsigned int i, j; + int data_unit_size; + struct bio_crypt_ctx *bc; + int err = 0; + + /* Split the bio if it's too big for single page bvec */ + err = blk_crypto_split_bio_if_needed(bio_ptr); + if (err) + return err; + + src_bio = *bio_ptr; + bc = src_bio->bi_crypt_context; + data_unit_size = bc->bc_key->data_unit_size; + + /* Allocate bounce bio for encryption */ + enc_bio = blk_crypto_clone_bio(src_bio); + if (!enc_bio) { + src_bio->bi_status = BLK_STS_RESOURCE; + return -ENOMEM; + } + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + err = bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm); + if (err) { + src_bio->bi_status = BLK_STS_IOERR; + goto out_put_enc_bio; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(src_bio, &ciph_req, &wait); + if (err) + goto out_release_keyslot; + + memcpy(curr_dun, bc->bc_dun, sizeof(curr_dun)); + sg_init_table(&src, 1); + sg_init_table(&dst, 1); + + skcipher_request_set_crypt(ciph_req, &src, &dst, data_unit_size, + iv.bytes); + + /* Encrypt each page in the bounce bio */ + for (i = 0; i < enc_bio->bi_vcnt; i++) { + struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; + struct page *plaintext_page = enc_bvec->bv_page; + struct page *ciphertext_page = + mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); + + enc_bvec->bv_page = ciphertext_page; + + if (!ciphertext_page) { + src_bio->bi_status = BLK_STS_RESOURCE; + err = -ENOMEM; + goto out_free_bounce_pages; + } + + sg_set_page(&src, plaintext_page, data_unit_size, + enc_bvec->bv_offset); + sg_set_page(&dst, ciphertext_page, data_unit_size, + enc_bvec->bv_offset); + + /* Encrypt each data unit in this page */ + for (j = 0; j < enc_bvec->bv_len; j += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + err = crypto_wait_req(crypto_skcipher_encrypt(ciph_req), + &wait); + if (err) { + i++; + src_bio->bi_status = BLK_STS_RESOURCE; + goto out_free_bounce_pages; + } + bio_crypt_dun_increment(curr_dun, 1); + src.offset += data_unit_size; + dst.offset += data_unit_size; + } + } + + enc_bio->bi_private = src_bio; + enc_bio->bi_end_io = blk_crypto_encrypt_endio; + *bio_ptr = enc_bio; + + enc_bio = NULL; + err = 0; + goto out_free_ciph_req; + +out_free_bounce_pages: + while (i > 0) + mempool_free(enc_bio->bi_io_vec[--i].bv_page, + blk_crypto_bounce_page_pool); +out_free_ciph_req: + skcipher_request_free(ciph_req); +out_release_keyslot: + bio_crypt_ctx_release_keyslot(bc); +out_put_enc_bio: + if (enc_bio) + bio_put(enc_bio); + + return err; +} + +static void blk_crypto_free_fallback_crypt_ctx(struct bio *bio) +{ + mempool_free(container_of(bio->bi_crypt_context, + struct bio_fallback_crypt_ctx, + crypt_ctx), + bio_fallback_crypt_ctx_pool); + bio->bi_crypt_context = NULL; +} + +/* + * The crypto API fallback's main decryption routine. + * Decrypts input bio in place. + */ +static void blk_crypto_decrypt_bio(struct work_struct *work) +{ + struct blk_crypto_decrypt_work *decrypt_work = + container_of(work, struct blk_crypto_decrypt_work, work); + struct bio *bio = decrypt_work->bio; + struct skcipher_request *ciph_req = NULL; + DECLARE_CRYPTO_WAIT(wait); + struct bio_vec bv; + struct bvec_iter iter; + u64 curr_dun[BLK_CRYPTO_DUN_ARRAY_SIZE]; + union blk_crypto_iv iv; + struct scatterlist sg; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx = + container_of(bc, struct bio_fallback_crypt_ctx, crypt_ctx); + const int data_unit_size = bc->bc_key->data_unit_size; + unsigned int i; + int err; + + /* + * Use the crypto API fallback keyslot manager to get a crypto_skcipher + * for the algorithm and key specified for this bio. + */ + if (bio_crypt_ctx_acquire_keyslot(bc, blk_crypto_ksm)) { + bio->bi_status = BLK_STS_RESOURCE; + goto out_no_keyslot; + } + + /* and then allocate an skcipher_request for it */ + err = blk_crypto_alloc_cipher_req(bio, &ciph_req, &wait); + if (err) + goto out; + + memcpy(curr_dun, f_ctx->fallback_dun, sizeof(curr_dun)); + sg_init_table(&sg, 1); + skcipher_request_set_crypt(ciph_req, &sg, &sg, data_unit_size, + iv.bytes); + + /* Decrypt each segment in the bio */ + __bio_for_each_segment(bv, bio, iter, f_ctx->crypt_iter) { + struct page *page = bv.bv_page; + + sg_set_page(&sg, page, data_unit_size, bv.bv_offset); + + /* Decrypt each data unit in the segment */ + for (i = 0; i < bv.bv_len; i += data_unit_size) { + blk_crypto_dun_to_iv(curr_dun, &iv); + if (crypto_wait_req(crypto_skcipher_decrypt(ciph_req), + &wait)) { + bio->bi_status = BLK_STS_IOERR; + goto out; + } + bio_crypt_dun_increment(curr_dun, 1); + sg.offset += data_unit_size; + } + } + +out: + skcipher_request_free(ciph_req); + bio_crypt_ctx_release_keyslot(bc); +out_no_keyslot: + kmem_cache_free(blk_crypto_decrypt_work_cache, decrypt_work); + blk_crypto_free_fallback_crypt_ctx(bio); + bio_endio(bio); +} + +/* + * Queue bio for decryption. + * Returns true iff bio was queued for decryption. + */ +bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + struct blk_crypto_decrypt_work *decrypt_work; + + /* If there was an IO error, don't queue for decrypt. */ + if (bio->bi_status) + goto out; + + decrypt_work = kmem_cache_zalloc(blk_crypto_decrypt_work_cache, + GFP_ATOMIC); + if (!decrypt_work) { + bio->bi_status = BLK_STS_RESOURCE; + goto out; + } + + INIT_WORK(&decrypt_work->work, blk_crypto_decrypt_bio); + decrypt_work->bio = bio; + queue_work(blk_crypto_wq, &decrypt_work->work); + + return true; +out: + blk_crypto_free_fallback_crypt_ctx(bio); + return false; +} + +/* + * Prepare blk-crypto-fallback for the specified crypto mode. + * Returns -ENOPKG if the needed crypto API support is missing. + */ +int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) +{ + const char *cipher_str = blk_crypto_modes[mode_num].cipher_str; + struct blk_crypto_keyslot *slotp; + unsigned int i; + int err = 0; + + /* + * Fast path + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we try to access them. + */ + if (likely(smp_load_acquire(&tfms_inited[mode_num]))) + return 0; + + mutex_lock(&tfms_init_lock); + if (likely(tfms_inited[mode_num])) + goto out; + + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); + if (IS_ERR(slotp->tfms[mode_num])) { + err = PTR_ERR(slotp->tfms[mode_num]); + if (err == -ENOENT) { + pr_warn_once("Missing crypto API support for \"%s\"\n", + cipher_str); + err = -ENOPKG; + } + slotp->tfms[mode_num] = NULL; + goto out_free_tfms; + } + + crypto_skcipher_set_flags(slotp->tfms[mode_num], + CRYPTO_TFM_REQ_WEAK_KEY); + } + + /* + * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] + * for each i are visible before we set tfms_inited[mode_num]. + */ + smp_store_release(&tfms_inited[mode_num], true); + goto out; + +out_free_tfms: + for (i = 0; i < blk_crypto_num_keyslots; i++) { + slotp = &blk_crypto_keyslots[i]; + crypto_free_skcipher(slotp->tfms[mode_num]); + slotp->tfms[mode_num] = NULL; + } +out: + mutex_unlock(&tfms_init_lock); + return err; +} + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return keyslot_manager_evict_key(blk_crypto_ksm, key); +} + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + struct bio_fallback_crypt_ctx *f_ctx; + + if (bc->bc_key->is_hw_wrapped) { + pr_warn_once("HW wrapped key cannot be used with fallback.\n"); + bio->bi_status = BLK_STS_NOTSUPP; + return -EOPNOTSUPP; + } + + if (!tfms_inited[bc->bc_key->crypto_mode]) { + bio->bi_status = BLK_STS_IOERR; + return -EIO; + } + + if (bio_data_dir(bio) == WRITE) + return blk_crypto_encrypt_bio(bio_ptr); + + /* + * Mark bio as fallback crypted and replace the bio_crypt_ctx with + * another one contained in a bio_fallback_crypt_ctx, so that the + * fallback has space to store the info it needs for decryption. + */ + bc->bc_ksm = blk_crypto_ksm; + f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); + f_ctx->crypt_ctx = *bc; + memcpy(f_ctx->fallback_dun, bc->bc_dun, sizeof(f_ctx->fallback_dun)); + f_ctx->crypt_iter = bio->bi_iter; + + bio_crypt_free_ctx(bio); + bio->bi_crypt_context = &f_ctx->crypt_ctx; + + return 0; +} + +int __init blk_crypto_fallback_init(void) +{ + int i; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; + + prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + + /* All blk-crypto modes have a crypto API fallback. */ + for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) + crypto_mode_supported[i] = 0xFFFFFFFF; + crypto_mode_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; + + blk_crypto_ksm = keyslot_manager_create( + NULL, blk_crypto_num_keyslots, + &blk_crypto_ksm_ll_ops, + BLK_CRYPTO_FEATURE_STANDARD_KEYS, + crypto_mode_supported, NULL); + if (!blk_crypto_ksm) + return -ENOMEM; + + blk_crypto_wq = alloc_workqueue("blk_crypto_wq", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, num_online_cpus()); + if (!blk_crypto_wq) + return -ENOMEM; + + blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, + sizeof(blk_crypto_keyslots[0]), + GFP_KERNEL); + if (!blk_crypto_keyslots) + return -ENOMEM; + + blk_crypto_bounce_page_pool = + mempool_create_page_pool(num_prealloc_bounce_pg, 0); + if (!blk_crypto_bounce_page_pool) + return -ENOMEM; + + blk_crypto_decrypt_work_cache = KMEM_CACHE(blk_crypto_decrypt_work, + SLAB_RECLAIM_ACCOUNT); + if (!blk_crypto_decrypt_work_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); + if (!bio_fallback_crypt_ctx_cache) + return -ENOMEM; + + bio_fallback_crypt_ctx_pool = + mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, + bio_fallback_crypt_ctx_cache); + if (!bio_fallback_crypt_ctx_pool) + return -ENOMEM; + + return 0; +} diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h new file mode 100644 index 000000000000..4da998c803f2 --- /dev/null +++ b/block/blk-crypto-internal.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2019 Google LLC + */ + +#ifndef __LINUX_BLK_CRYPTO_INTERNAL_H +#define __LINUX_BLK_CRYPTO_INTERNAL_H + +#include <linux/bio.h> + +/* Represents a crypto mode supported by blk-crypto */ +struct blk_crypto_mode { + const char *cipher_str; /* crypto API name (for fallback case) */ + unsigned int keysize; /* key size in bytes */ + unsigned int ivsize; /* iv size in bytes */ +}; + +extern const struct blk_crypto_mode blk_crypto_modes[]; + +#ifdef CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK + +int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num); + +int blk_crypto_fallback_submit_bio(struct bio **bio_ptr); + +bool blk_crypto_queue_decrypt_bio(struct bio *bio); + +int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key); + +bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc); + +#else /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +static inline int +blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) +{ + pr_warn_once("crypto API fallback is disabled\n"); + return -ENOPKG; +} + +static inline bool bio_crypt_fallback_crypted(const struct bio_crypt_ctx *bc) +{ + return false; +} + +static inline int blk_crypto_fallback_submit_bio(struct bio **bio_ptr) +{ + pr_warn_once("crypto API fallback disabled; failing request\n"); + (*bio_ptr)->bi_status = BLK_STS_NOTSUPP; + return -EIO; +} + +static inline bool blk_crypto_queue_decrypt_bio(struct bio *bio) +{ + WARN_ON(1); + return false; +} + +static inline int +blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) +{ + return 0; +} + +#endif /* CONFIG_BLK_INLINE_ENCRYPTION_FALLBACK */ + +#endif /* __LINUX_BLK_CRYPTO_INTERNAL_H */ diff --git a/block/blk-crypto.c b/block/blk-crypto.c new file mode 100644 index 000000000000..e07a37cf8b5f --- /dev/null +++ b/block/blk-crypto.c @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +/* + * Refer to Documentation/block/inline-encryption.rst for detailed explanation. + */ + +#define pr_fmt(fmt) "blk-crypto: " fmt + +#include <linux/blk-crypto.h> +#include <linux/blkdev.h> +#include <linux/keyslot-manager.h> +#include <linux/random.h> +#include <linux/siphash.h> + +#include "blk-crypto-internal.h" + +const struct blk_crypto_mode blk_crypto_modes[] = { + [BLK_ENCRYPTION_MODE_AES_256_XTS] = { + .cipher_str = "xts(aes)", + .keysize = 64, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_AES_128_CBC_ESSIV] = { + .cipher_str = "essiv(cbc(aes),sha256)", + .keysize = 16, + .ivsize = 16, + }, + [BLK_ENCRYPTION_MODE_ADIANTUM] = { + .cipher_str = "adiantum(xchacha12,aes)", + .keysize = 32, + .ivsize = 32, + }, +}; + +/* Check that all I/O segments are data unit aligned */ +static int bio_crypt_check_alignment(struct bio *bio) +{ + const unsigned int data_unit_size = + bio->bi_crypt_context->bc_key->data_unit_size; + struct bvec_iter iter; + struct bio_vec bv; + + bio_for_each_segment(bv, bio, iter) { + if (!IS_ALIGNED(bv.bv_len | bv.bv_offset, data_unit_size)) + return -EIO; + } + return 0; +} + +/** + * blk_crypto_submit_bio - handle submitting bio for inline encryption + * + * @bio_ptr: pointer to original bio pointer + * + * If the bio doesn't have inline encryption enabled or the submitter already + * specified a keyslot for the target device, do nothing. Else, a raw key must + * have been provided, so acquire a device keyslot for it if supported. Else, + * use the crypto API fallback. + * + * When the crypto API fallback is used for encryption, blk-crypto may choose to + * split the bio into 2 - the first one that will continue to be processed and + * the second one that will be resubmitted via generic_make_request. + * A bounce bio will be allocated to encrypt the contents of the aforementioned + * "first one", and *bio_ptr will be updated to this bounce bio. + * + * Return: 0 if bio submission should continue; nonzero if bio_endio() was + * already called so bio submission should abort. + */ +int blk_crypto_submit_bio(struct bio **bio_ptr) +{ + struct bio *bio = *bio_ptr; + struct request_queue *q; + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + int err; + + if (!bc || !bio_has_data(bio)) + return 0; + + /* + * When a read bio is marked for fallback decryption, its bi_iter is + * saved so that when we decrypt the bio later, we know what part of it + * was marked for fallback decryption (when the bio is passed down after + * blk_crypto_submit bio, it may be split or advanced so we cannot rely + * on the bi_iter while decrypting in blk_crypto_endio) + */ + if (bio_crypt_fallback_crypted(bc)) + return 0; + + err = bio_crypt_check_alignment(bio); + if (err) { + bio->bi_status = BLK_STS_IOERR; + goto out; + } + + q = bio->bi_disk->queue; + + if (bc->bc_ksm) { + /* Key already programmed into device? */ + if (q->ksm == bc->bc_ksm) + return 0; + + /* Nope, release the existing keyslot. */ + bio_crypt_ctx_release_keyslot(bc); + } + + /* Get device keyslot if supported */ + if (keyslot_manager_crypto_mode_supported(q->ksm, + bc->bc_key->crypto_mode, + blk_crypto_key_dun_bytes(bc->bc_key), + bc->bc_key->data_unit_size, + bc->bc_key->is_hw_wrapped)) { + err = bio_crypt_ctx_acquire_keyslot(bc, q->ksm); + if (!err) + return 0; + + pr_warn_once("Failed to acquire keyslot for %s (err=%d). Falling back to crypto API.\n", + bio->bi_disk->disk_name, err); + } + + /* Fallback to crypto API */ + err = blk_crypto_fallback_submit_bio(bio_ptr); + if (err) + goto out; + + return 0; +out: + bio_endio(*bio_ptr); + return err; +} + +/** + * blk_crypto_endio - clean up bio w.r.t inline encryption during bio_endio + * + * @bio: the bio to clean up + * + * If blk_crypto_submit_bio decided to fallback to crypto API for this bio, + * we queue the bio for decryption into a workqueue and return false, + * and call bio_endio(bio) at a later time (after the bio has been decrypted). + * + * If the bio is not to be decrypted by the crypto API, this function releases + * the reference to the keyslot that blk_crypto_submit_bio got. + * + * Return: true if bio_endio should continue; false otherwise (bio_endio will + * be called again when bio has been decrypted). + */ +bool blk_crypto_endio(struct bio *bio) +{ + struct bio_crypt_ctx *bc = bio->bi_crypt_context; + + if (!bc) + return true; + + if (bio_crypt_fallback_crypted(bc)) { + /* + * The only bios who's crypto is handled by the blk-crypto + * fallback when they reach here are those with + * bio_data_dir(bio) == READ, since WRITE bios that are + * encrypted by the crypto API fallback are handled by + * blk_crypto_encrypt_endio(). + */ + return !blk_crypto_queue_decrypt_bio(bio); + } + + if (bc->bc_keyslot >= 0) + bio_crypt_ctx_release_keyslot(bc); + + return true; +} + +/** + * blk_crypto_init_key() - Prepare a key for use with blk-crypto + * @blk_key: Pointer to the blk_crypto_key to initialize. + * @raw_key: Pointer to the raw key. + * @raw_key_size: Size of raw key. Must be at least the required size for the + * chosen @crypto_mode; see blk_crypto_modes[]. (It's allowed + * to be longer than the mode's actual key size, in order to + * support inline encryption hardware that accepts wrapped keys. + * @is_hw_wrapped has to be set for such keys) + * @is_hw_wrapped: Denotes @raw_key is wrapped. + * @crypto_mode: identifier for the encryption algorithm to use + * @dun_bytes: number of bytes that will be used to specify the DUN when this + * key is used + * @data_unit_size: the data unit size to use for en/decryption + * + * Return: The blk_crypto_key that was prepared, or an ERR_PTR() on error. When + * done using the key, it must be freed with blk_crypto_free_key(). + */ +int blk_crypto_init_key(struct blk_crypto_key *blk_key, + const u8 *raw_key, unsigned int raw_key_size, + bool is_hw_wrapped, + enum blk_crypto_mode_num crypto_mode, + unsigned int dun_bytes, + unsigned int data_unit_size) +{ + const struct blk_crypto_mode *mode; + static siphash_key_t hash_key; + u32 hash; + + memset(blk_key, 0, sizeof(*blk_key)); + + if (crypto_mode >= ARRAY_SIZE(blk_crypto_modes)) + return -EINVAL; + + BUILD_BUG_ON(BLK_CRYPTO_MAX_WRAPPED_KEY_SIZE < BLK_CRYPTO_MAX_KEY_SIZE); + + mode = &blk_crypto_modes[crypto_mode]; + if (is_hw_wrapped) { + if (raw_key_size < mode->keysize || + raw_key_size > BLK_CRYPTO_MAX_WRAPPED_KEY_SIZE) + return -EINVAL; + } else { + if (raw_key_size != mode->keysize) + return -EINVAL; + } + + if (dun_bytes <= 0 || dun_bytes > BLK_CRYPTO_MAX_IV_SIZE) + return -EINVAL; + + if (!is_power_of_2(data_unit_size)) + return -EINVAL; + + blk_key->crypto_mode = crypto_mode; + blk_key->data_unit_size = data_unit_size; + blk_key->data_unit_size_bits = ilog2(data_unit_size); + blk_key->size = raw_key_size; + blk_key->is_hw_wrapped = is_hw_wrapped; + memcpy(blk_key->raw, raw_key, raw_key_size); + + /* + * The keyslot manager uses the SipHash of the key to implement O(1) key + * lookups while avoiding leaking information about the keys. It's + * precomputed here so that it only needs to be computed once per key. + */ + get_random_once(&hash_key, sizeof(hash_key)); + hash = (u32)siphash(raw_key, raw_key_size, &hash_key); + blk_crypto_key_set_hash_and_dun_bytes(blk_key, hash, dun_bytes); + + return 0; +} +EXPORT_SYMBOL_GPL(blk_crypto_init_key); + +/** + * blk_crypto_start_using_mode() - Start using blk-crypto on a device + * @crypto_mode: the crypto mode that will be used + * @dun_bytes: number of bytes that will be used to specify the DUN + * @data_unit_size: the data unit size that will be used + * @is_hw_wrapped_key: whether the key will be hardware-wrapped + * @q: the request queue for the device + * + * Upper layers must call this function to ensure that either the hardware + * supports the needed crypto settings, or the crypto API fallback has + * transforms for the needed mode allocated and ready to go. + * + * Return: 0 on success; -ENOPKG if the hardware doesn't support the crypto + * settings and blk-crypto-fallback is either disabled or the needed + * algorithm is disabled in the crypto API; or another -errno code. + */ +int blk_crypto_start_using_mode(enum blk_crypto_mode_num crypto_mode, + unsigned int dun_bytes, + unsigned int data_unit_size, + bool is_hw_wrapped_key, + struct request_queue *q) +{ + if (keyslot_manager_crypto_mode_supported(q->ksm, crypto_mode, + dun_bytes, data_unit_size, + is_hw_wrapped_key)) + return 0; + if (is_hw_wrapped_key) { + pr_warn_once("hardware doesn't support wrapped keys\n"); + return -EOPNOTSUPP; + } + return blk_crypto_fallback_start_using_mode(crypto_mode); +} +EXPORT_SYMBOL_GPL(blk_crypto_start_using_mode); + +/** + * blk_crypto_evict_key() - Evict a key from any inline encryption hardware + * it may have been programmed into + * @q: The request queue who's keyslot manager this key might have been + * programmed into + * @key: The key to evict + * + * Upper layers (filesystems) should call this function to ensure that a key + * is evicted from hardware that it might have been programmed into. This + * will call keyslot_manager_evict_key on the queue's keyslot manager, if one + * exists, and supports the crypto algorithm with the specified data unit size. + * Otherwise, it will evict the key from the blk-crypto-fallback's ksm. + * + * Return: 0 on success, -err on error. + */ +int blk_crypto_evict_key(struct request_queue *q, + const struct blk_crypto_key *key) +{ + if (q->ksm && + keyslot_manager_crypto_mode_supported(q->ksm, key->crypto_mode, + blk_crypto_key_dun_bytes(key), + key->data_unit_size, + key->is_hw_wrapped)) + return keyslot_manager_evict_key(q->ksm, key); + + return blk_crypto_fallback_evict_key(key); +} +EXPORT_SYMBOL_GPL(blk_crypto_evict_key); diff --git a/block/blk-merge.c b/block/blk-merge.c index 415b5dafd9e6..71768dc75602 100644 --- a/block/blk-merge.c +++ b/block/blk-merge.c @@ -504,6 +504,8 @@ int ll_back_merge_fn(struct request_queue *q, struct request *req, req_set_nomerge(q, req); return 0; } + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), bio)) + return 0; if (!bio_flagged(req->biotail, BIO_SEG_VALID)) blk_recount_segments(q, req->biotail); if (!bio_flagged(bio, BIO_SEG_VALID)) @@ -526,6 +528,8 @@ int ll_front_merge_fn(struct request_queue *q, struct request *req, req_set_nomerge(q, req); return 0; } + if (!bio_crypt_ctx_mergeable(bio, bio->bi_iter.bi_size, req->bio)) + return 0; if (!bio_flagged(bio, BIO_SEG_VALID)) blk_recount_segments(q, bio); if (!bio_flagged(req->bio, BIO_SEG_VALID)) @@ -602,6 +606,9 @@ static int ll_merge_requests_fn(struct request_queue *q, struct request *req, if (blk_integrity_merge_rq(q, req, next) == false) return 0; + if (!bio_crypt_ctx_mergeable(req->bio, blk_rq_bytes(req), next->bio)) + return 0; + /* Merge is OK... */ req->nr_phys_segments = total_phys_segments; return 1; @@ -850,6 +857,10 @@ bool blk_rq_merge_ok(struct request *rq, struct bio *bio) if (rq->write_hint != bio->bi_write_hint) return false; + /* Only merge if the crypt contexts are compatible */ + if (!bio_crypt_ctx_compatible(bio, rq->bio)) + return false; + return true; } diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 9c4f1c496c90..8df0fecac3a8 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -225,6 +225,7 @@ struct cfq_group_data { unsigned int weight; unsigned int leaf_weight; + u64 group_idle; }; /* This is per cgroup per device grouping structure */ @@ -310,6 +311,7 @@ struct cfq_group { struct cfq_queue *async_cfqq[2][IOPRIO_BE_NR]; struct cfq_queue *async_idle_cfqq; + u64 group_idle; }; struct cfq_io_cq { @@ -802,6 +804,17 @@ static inline void cfqg_stats_update_completion(struct cfq_group *cfqg, #endif /* CONFIG_CFQ_GROUP_IOSCHED */ +static inline u64 get_group_idle(struct cfq_data *cfqd) +{ +#ifdef CONFIG_CFQ_GROUP_IOSCHED + struct cfq_queue *cfqq = cfqd->active_queue; + + if (cfqq && cfqq->cfqg) + return cfqq->cfqg->group_idle; +#endif + return cfqd->cfq_group_idle; +} + #define cfq_log(cfqd, fmt, args...) \ blk_add_trace_msg((cfqd)->queue, "cfq " fmt, ##args) @@ -822,7 +835,7 @@ static inline bool cfq_io_thinktime_big(struct cfq_data *cfqd, if (!sample_valid(ttime->ttime_samples)) return false; if (group_idle) - slice = cfqd->cfq_group_idle; + slice = get_group_idle(cfqd); else slice = cfqd->cfq_slice_idle; return ttime->ttime_mean > slice; @@ -1589,6 +1602,7 @@ static void cfq_cpd_init(struct blkcg_policy_data *cpd) cgd->weight = weight; cgd->leaf_weight = weight; + cgd->group_idle = cfq_group_idle; } static void cfq_cpd_free(struct blkcg_policy_data *cpd) @@ -1633,6 +1647,7 @@ static void cfq_pd_init(struct blkg_policy_data *pd) cfqg->weight = cgd->weight; cfqg->leaf_weight = cgd->leaf_weight; + cfqg->group_idle = cgd->group_idle; } static void cfq_pd_offline(struct blkg_policy_data *pd) @@ -1754,6 +1769,19 @@ static int cfq_print_leaf_weight(struct seq_file *sf, void *v) return 0; } +static int cfq_print_group_idle(struct seq_file *sf, void *v) +{ + struct blkcg *blkcg = css_to_blkcg(seq_css(sf)); + struct cfq_group_data *cgd = blkcg_to_cfqgd(blkcg); + u64 val = 0; + + if (cgd) + val = cgd->group_idle; + + seq_printf(sf, "%llu\n", div_u64(val, NSEC_PER_USEC)); + return 0; +} + static ssize_t __cfqg_set_weight_device(struct kernfs_open_file *of, char *buf, size_t nbytes, loff_t off, bool on_dfl, bool is_leaf_weight) @@ -1875,6 +1903,37 @@ static int cfq_set_leaf_weight(struct cgroup_subsys_state *css, return __cfq_set_weight(css, val, false, false, true); } +static int cfq_set_group_idle(struct cgroup_subsys_state *css, + struct cftype *cft, u64 val) +{ + struct blkcg *blkcg = css_to_blkcg(css); + struct cfq_group_data *cfqgd; + struct blkcg_gq *blkg; + int ret = 0; + + spin_lock_irq(&blkcg->lock); + cfqgd = blkcg_to_cfqgd(blkcg); + if (!cfqgd) { + ret = -EINVAL; + goto out; + } + + cfqgd->group_idle = val * NSEC_PER_USEC; + + hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) { + struct cfq_group *cfqg = blkg_to_cfqg(blkg); + + if (!cfqg) + continue; + + cfqg->group_idle = cfqgd->group_idle; + } + +out: + spin_unlock_irq(&blkcg->lock); + return ret; +} + static int cfqg_print_stat(struct seq_file *sf, void *v) { blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat, @@ -2020,6 +2079,11 @@ static struct cftype cfq_blkcg_legacy_files[] = { .seq_show = cfq_print_leaf_weight, .write_u64 = cfq_set_leaf_weight, }, + { + .name = "group_idle", + .seq_show = cfq_print_group_idle, + .write_u64 = cfq_set_group_idle, + }, /* statistics, covers only the tasks in the cfqg */ { @@ -2914,7 +2978,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) * with sync vs async workloads. */ if (blk_queue_nonrot(cfqd->queue) && cfqd->hw_tag && - !cfqd->cfq_group_idle) + !get_group_idle(cfqd)) return; WARN_ON(!RB_EMPTY_ROOT(&cfqq->sort_list)); @@ -2925,9 +2989,8 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) */ if (!cfq_should_idle(cfqd, cfqq)) { /* no queue idling. Check for group idling */ - if (cfqd->cfq_group_idle) - group_idle = cfqd->cfq_group_idle; - else + group_idle = get_group_idle(cfqd); + if (!group_idle) return; } @@ -2968,7 +3031,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd) cfq_mark_cfqq_wait_request(cfqq); if (group_idle) - sl = cfqd->cfq_group_idle; + sl = group_idle; else sl = cfqd->cfq_slice_idle; @@ -3317,7 +3380,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd) * this group, wait for requests to complete. */ check_group_idle: - if (cfqd->cfq_group_idle && cfqq->cfqg->nr_cfqq == 1 && + if (get_group_idle(cfqd) && cfqq->cfqg->nr_cfqq == 1 && cfqq->cfqg->dispatched && !cfq_io_thinktime_big(cfqd, &cfqq->cfqg->ttime, true)) { cfqq = NULL; @@ -3880,7 +3943,7 @@ cfq_update_io_thinktime(struct cfq_data *cfqd, struct cfq_queue *cfqq, cfqd->cfq_slice_idle); } #ifdef CONFIG_CFQ_GROUP_IOSCHED - __cfq_update_io_thinktime(&cfqq->cfqg->ttime, cfqd->cfq_group_idle); + __cfq_update_io_thinktime(&cfqq->cfqg->ttime, get_group_idle(cfqd)); #endif } @@ -4278,7 +4341,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq) if (cfq_should_wait_busy(cfqd, cfqq)) { u64 extend_sl = cfqd->cfq_slice_idle; if (!cfqd->cfq_slice_idle) - extend_sl = cfqd->cfq_group_idle; + extend_sl = get_group_idle(cfqd); cfqq->slice_end = now + extend_sl; cfq_mark_cfqq_wait_busy(cfqq); cfq_log_cfqq(cfqd, cfqq, "will busy wait"); diff --git a/block/keyslot-manager.c b/block/keyslot-manager.c new file mode 100644 index 000000000000..901545c5854c --- /dev/null +++ b/block/keyslot-manager.c @@ -0,0 +1,664 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2019 Google LLC + */ + +/** + * DOC: The Keyslot Manager + * + * Many devices with inline encryption support have a limited number of "slots" + * into which encryption contexts may be programmed, and requests can be tagged + * with a slot number to specify the key to use for en/decryption. + * + * As the number of slots are limited, and programming keys is expensive on + * many inline encryption hardware, we don't want to program the same key into + * multiple slots - if multiple requests are using the same key, we want to + * program just one slot with that key and use that slot for all requests. + * + * The keyslot manager manages these keyslots appropriately, and also acts as + * an abstraction between the inline encryption hardware and the upper layers. + * + * Lower layer devices will set up a keyslot manager in their request queue + * and tell it how to perform device specific operations like programming/ + * evicting keys from keyslots. + * + * Upper layers will call keyslot_manager_get_slot_for_key() to program a + * key into some slot in the inline encryption hardware. + */ +#include <crypto/algapi.h> +#include <linux/keyslot-manager.h> +#include <linux/atomic.h> +#include <linux/mutex.h> +#include <linux/pm_runtime.h> +#include <linux/wait.h> +#include <linux/blkdev.h> +#include <linux/overflow.h> + +struct keyslot { + atomic_t slot_refs; + struct list_head idle_slot_node; + struct hlist_node hash_node; + struct blk_crypto_key key; +}; + +struct keyslot_manager { + unsigned int num_slots; + struct keyslot_mgmt_ll_ops ksm_ll_ops; + unsigned int features; + unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX]; + unsigned int max_dun_bytes_supported; + void *ll_priv_data; + +#ifdef CONFIG_PM + /* Device for runtime power management (NULL if none) */ + struct device *dev; +#endif + + /* Protects programming and evicting keys from the device */ + struct rw_semaphore lock; + + /* List of idle slots, with least recently used slot at front */ + wait_queue_head_t idle_slots_wait_queue; + struct list_head idle_slots; + spinlock_t idle_slots_lock; + + /* + * Hash table which maps key hashes to keyslots, so that we can find a + * key's keyslot in O(1) time rather than O(num_slots). Protected by + * 'lock'. A cryptographic hash function is used so that timing attacks + * can't leak information about the raw keys. + */ + struct hlist_head *slot_hashtable; + unsigned int slot_hashtable_size; + + /* Per-keyslot data */ + struct keyslot slots[]; +}; + +static inline bool keyslot_manager_is_passthrough(struct keyslot_manager *ksm) +{ + return ksm->num_slots == 0; +} + +#ifdef CONFIG_PM +static inline void keyslot_manager_set_dev(struct keyslot_manager *ksm, + struct device *dev) +{ + ksm->dev = dev; +} + +/* If there's an underlying device and it's suspended, resume it. */ +static inline void keyslot_manager_pm_get(struct keyslot_manager *ksm) +{ + if (ksm->dev) + pm_runtime_get_sync(ksm->dev); +} + +static inline void keyslot_manager_pm_put(struct keyslot_manager *ksm) +{ + if (ksm->dev) + pm_runtime_put_sync(ksm->dev); +} +#else /* CONFIG_PM */ +static inline void keyslot_manager_set_dev(struct keyslot_manager *ksm, + struct device *dev) +{ +} + +static inline void keyslot_manager_pm_get(struct keyslot_manager *ksm) +{ +} + +static inline void keyslot_manager_pm_put(struct keyslot_manager *ksm) +{ +} +#endif /* !CONFIG_PM */ + +static inline void keyslot_manager_hw_enter(struct keyslot_manager *ksm) +{ + /* + * Calling into the driver requires ksm->lock held and the device + * resumed. But we must resume the device first, since that can acquire + * and release ksm->lock via keyslot_manager_reprogram_all_keys(). + */ + keyslot_manager_pm_get(ksm); + down_write(&ksm->lock); +} + +static inline void keyslot_manager_hw_exit(struct keyslot_manager *ksm) +{ + up_write(&ksm->lock); + keyslot_manager_pm_put(ksm); +} + +/** + * keyslot_manager_create() - Create a keyslot manager + * @dev: Device for runtime power management (NULL if none) + * @num_slots: The number of key slots to manage. + * @ksm_ll_ops: The struct keyslot_mgmt_ll_ops for the device that this keyslot + * manager will use to perform operations like programming and + * evicting keys. + * @features: The supported features as a bitmask of BLK_CRYPTO_FEATURE_* flags. + * Most drivers should set BLK_CRYPTO_FEATURE_STANDARD_KEYS here. + * @crypto_mode_supported: Array of size BLK_ENCRYPTION_MODE_MAX of + * bitmasks that represents whether a crypto mode + * and data unit size are supported. The i'th bit + * of crypto_mode_supported[crypto_mode] is set iff + * a data unit size of (1 << i) is supported. We + * only support data unit sizes that are powers of + * 2. + * @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops. + * + * Allocate memory for and initialize a keyslot manager. Called by e.g. + * storage drivers to set up a keyslot manager in their request_queue. + * + * Context: May sleep + * Return: Pointer to constructed keyslot manager or NULL on error. + */ +struct keyslot_manager *keyslot_manager_create( + struct device *dev, + unsigned int num_slots, + const struct keyslot_mgmt_ll_ops *ksm_ll_ops, + unsigned int features, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data) +{ + struct keyslot_manager *ksm; + unsigned int slot; + unsigned int i; + + if (num_slots == 0) + return NULL; + + /* Check that all ops are specified */ + if (ksm_ll_ops->keyslot_program == NULL || + ksm_ll_ops->keyslot_evict == NULL) + return NULL; + + ksm = kvzalloc(struct_size(ksm, slots, num_slots), GFP_KERNEL); + if (!ksm) + return NULL; + + ksm->num_slots = num_slots; + ksm->ksm_ll_ops = *ksm_ll_ops; + ksm->features = features; + memcpy(ksm->crypto_mode_supported, crypto_mode_supported, + sizeof(ksm->crypto_mode_supported)); + ksm->max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE; + ksm->ll_priv_data = ll_priv_data; + keyslot_manager_set_dev(ksm, dev); + + init_rwsem(&ksm->lock); + + init_waitqueue_head(&ksm->idle_slots_wait_queue); + INIT_LIST_HEAD(&ksm->idle_slots); + + for (slot = 0; slot < num_slots; slot++) { + list_add_tail(&ksm->slots[slot].idle_slot_node, + &ksm->idle_slots); + } + + spin_lock_init(&ksm->idle_slots_lock); + + ksm->slot_hashtable_size = roundup_pow_of_two(num_slots); + ksm->slot_hashtable = kvmalloc_array(ksm->slot_hashtable_size, + sizeof(ksm->slot_hashtable[0]), + GFP_KERNEL); + if (!ksm->slot_hashtable) + goto err_free_ksm; + for (i = 0; i < ksm->slot_hashtable_size; i++) + INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); + + return ksm; + +err_free_ksm: + keyslot_manager_destroy(ksm); + return NULL; +} +EXPORT_SYMBOL_GPL(keyslot_manager_create); + +void keyslot_manager_set_max_dun_bytes(struct keyslot_manager *ksm, + unsigned int max_dun_bytes) +{ + ksm->max_dun_bytes_supported = max_dun_bytes; +} +EXPORT_SYMBOL_GPL(keyslot_manager_set_max_dun_bytes); + +static inline struct hlist_head * +hash_bucket_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + return &ksm->slot_hashtable[blk_crypto_key_hash(key) & + (ksm->slot_hashtable_size - 1)]; +} + +static void remove_slot_from_lru_list(struct keyslot_manager *ksm, int slot) +{ + unsigned long flags; + + spin_lock_irqsave(&ksm->idle_slots_lock, flags); + list_del(&ksm->slots[slot].idle_slot_node); + spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); +} + +static int find_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + const struct hlist_head *head = hash_bucket_for_key(ksm, key); + const struct keyslot *slotp; + + hlist_for_each_entry(slotp, head, hash_node) { + if (slotp->key.hash == key->hash && + slotp->key.crypto_mode == key->crypto_mode && + slotp->key.size == key->size && + slotp->key.data_unit_size == key->data_unit_size && + !crypto_memneq(slotp->key.raw, key->raw, key->size)) + return slotp - ksm->slots; + } + return -ENOKEY; +} + +static int find_and_grab_keyslot(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + int slot; + + slot = find_keyslot(ksm, key); + if (slot < 0) + return slot; + if (atomic_inc_return(&ksm->slots[slot].slot_refs) == 1) { + /* Took first reference to this slot; remove it from LRU list */ + remove_slot_from_lru_list(ksm, slot); + } + return slot; +} + +/** + * keyslot_manager_get_slot_for_key() - Program a key into a keyslot. + * @ksm: The keyslot manager to program the key into. + * @key: Pointer to the key object to program, including the raw key, crypto + * mode, and data unit size. + * + * Get a keyslot that's been programmed with the specified key. If one already + * exists, return it with incremented refcount. Otherwise, wait for a keyslot + * to become idle and program it. + * + * Context: Process context. Takes and releases ksm->lock. + * Return: The keyslot on success, else a -errno value. + */ +int keyslot_manager_get_slot_for_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + int slot; + int err; + struct keyslot *idle_slot; + + if (keyslot_manager_is_passthrough(ksm)) + return 0; + + down_read(&ksm->lock); + slot = find_and_grab_keyslot(ksm, key); + up_read(&ksm->lock); + if (slot != -ENOKEY) + return slot; + + for (;;) { + keyslot_manager_hw_enter(ksm); + slot = find_and_grab_keyslot(ksm, key); + if (slot != -ENOKEY) { + keyslot_manager_hw_exit(ksm); + return slot; + } + + /* + * If we're here, that means there wasn't a slot that was + * already programmed with the key. So try to program it. + */ + if (!list_empty(&ksm->idle_slots)) + break; + + keyslot_manager_hw_exit(ksm); + wait_event(ksm->idle_slots_wait_queue, + !list_empty(&ksm->idle_slots)); + } + + idle_slot = list_first_entry(&ksm->idle_slots, struct keyslot, + idle_slot_node); + slot = idle_slot - ksm->slots; + + err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); + if (err) { + wake_up(&ksm->idle_slots_wait_queue); + keyslot_manager_hw_exit(ksm); + return err; + } + + /* Move this slot to the hash list for the new key. */ + if (idle_slot->key.crypto_mode != BLK_ENCRYPTION_MODE_INVALID) + hlist_del(&idle_slot->hash_node); + hlist_add_head(&idle_slot->hash_node, hash_bucket_for_key(ksm, key)); + + atomic_set(&idle_slot->slot_refs, 1); + idle_slot->key = *key; + + remove_slot_from_lru_list(ksm, slot); + + keyslot_manager_hw_exit(ksm); + return slot; +} + +/** + * keyslot_manager_get_slot() - Increment the refcount on the specified slot. + * @ksm: The keyslot manager that we want to modify. + * @slot: The slot to increment the refcount of. + * + * This function assumes that there is already an active reference to that slot + * and simply increments the refcount. This is useful when cloning a bio that + * already has a reference to a keyslot, and we want the cloned bio to also have + * its own reference. + * + * Context: Any context. + */ +void keyslot_manager_get_slot(struct keyslot_manager *ksm, unsigned int slot) +{ + if (keyslot_manager_is_passthrough(ksm)) + return; + + if (WARN_ON(slot >= ksm->num_slots)) + return; + + WARN_ON(atomic_inc_return(&ksm->slots[slot].slot_refs) < 2); +} + +/** + * keyslot_manager_put_slot() - Release a reference to a slot + * @ksm: The keyslot manager to release the reference from. + * @slot: The slot to release the reference from. + * + * Context: Any context. + */ +void keyslot_manager_put_slot(struct keyslot_manager *ksm, unsigned int slot) +{ + unsigned long flags; + + if (keyslot_manager_is_passthrough(ksm)) + return; + + if (WARN_ON(slot >= ksm->num_slots)) + return; + + if (atomic_dec_and_lock_irqsave(&ksm->slots[slot].slot_refs, + &ksm->idle_slots_lock, flags)) { + list_add_tail(&ksm->slots[slot].idle_slot_node, + &ksm->idle_slots); + spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); + wake_up(&ksm->idle_slots_wait_queue); + } +} + +/** + * keyslot_manager_crypto_mode_supported() - Find out if a crypto_mode / + * data unit size / is_hw_wrapped_key + * combination is supported by a ksm. + * @ksm: The keyslot manager to check + * @crypto_mode: The crypto mode to check for. + * @dun_bytes: The number of bytes that will be used to specify the DUN + * @data_unit_size: The data_unit_size for the mode. + * @is_hw_wrapped_key: Whether a hardware-wrapped key will be used. + * + * Calls and returns the result of the crypto_mode_supported function specified + * by the ksm. + * + * Context: Process context. + * Return: Whether or not this ksm supports the specified crypto settings. + */ +bool keyslot_manager_crypto_mode_supported(struct keyslot_manager *ksm, + enum blk_crypto_mode_num crypto_mode, + unsigned int dun_bytes, + unsigned int data_unit_size, + bool is_hw_wrapped_key) +{ + if (!ksm) + return false; + if (WARN_ON(crypto_mode >= BLK_ENCRYPTION_MODE_MAX)) + return false; + if (WARN_ON(!is_power_of_2(data_unit_size))) + return false; + if (is_hw_wrapped_key) { + if (!(ksm->features & BLK_CRYPTO_FEATURE_WRAPPED_KEYS)) + return false; + } else { + if (!(ksm->features & BLK_CRYPTO_FEATURE_STANDARD_KEYS)) + return false; + } + if (!(ksm->crypto_mode_supported[crypto_mode] & data_unit_size)) + return false; + + return ksm->max_dun_bytes_supported >= dun_bytes; +} + +/** + * keyslot_manager_evict_key() - Evict a key from the lower layer device. + * @ksm: The keyslot manager to evict from + * @key: The key to evict + * + * Find the keyslot that the specified key was programmed into, and evict that + * slot from the lower layer device if that slot is not currently in use. + * + * Context: Process context. Takes and releases ksm->lock. + * Return: 0 on success, -EBUSY if the key is still in use, or another + * -errno value on other error. + */ +int keyslot_manager_evict_key(struct keyslot_manager *ksm, + const struct blk_crypto_key *key) +{ + int slot; + int err; + struct keyslot *slotp; + + if (keyslot_manager_is_passthrough(ksm)) { + if (ksm->ksm_ll_ops.keyslot_evict) { + keyslot_manager_hw_enter(ksm); + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, -1); + keyslot_manager_hw_exit(ksm); + return err; + } + return 0; + } + + keyslot_manager_hw_enter(ksm); + + slot = find_keyslot(ksm, key); + if (slot < 0) { + err = slot; + goto out_unlock; + } + slotp = &ksm->slots[slot]; + + if (atomic_read(&slotp->slot_refs) != 0) { + err = -EBUSY; + goto out_unlock; + } + err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, slot); + if (err) + goto out_unlock; + + hlist_del(&slotp->hash_node); + memzero_explicit(&slotp->key, sizeof(slotp->key)); + err = 0; +out_unlock: + keyslot_manager_hw_exit(ksm); + return err; +} + +/** + * keyslot_manager_reprogram_all_keys() - Re-program all keyslots. + * @ksm: The keyslot manager + * + * Re-program all keyslots that are supposed to have a key programmed. This is + * intended only for use by drivers for hardware that loses its keys on reset. + * + * Context: Process context. Takes and releases ksm->lock. + */ +void keyslot_manager_reprogram_all_keys(struct keyslot_manager *ksm) +{ + unsigned int slot; + + if (WARN_ON(keyslot_manager_is_passthrough(ksm))) + return; + + /* This is for device initialization, so don't resume the device */ + down_write(&ksm->lock); + for (slot = 0; slot < ksm->num_slots; slot++) { + const struct keyslot *slotp = &ksm->slots[slot]; + int err; + + if (slotp->key.crypto_mode == BLK_ENCRYPTION_MODE_INVALID) + continue; + + err = ksm->ksm_ll_ops.keyslot_program(ksm, &slotp->key, slot); + WARN_ON(err); + } + up_write(&ksm->lock); +} +EXPORT_SYMBOL_GPL(keyslot_manager_reprogram_all_keys); + +/** + * keyslot_manager_private() - return the private data stored with ksm + * @ksm: The keyslot manager + * + * Returns the private data passed to the ksm when it was created. + */ +void *keyslot_manager_private(struct keyslot_manager *ksm) +{ + return ksm->ll_priv_data; +} +EXPORT_SYMBOL_GPL(keyslot_manager_private); + +void keyslot_manager_destroy(struct keyslot_manager *ksm) +{ + if (ksm) { + kvfree(ksm->slot_hashtable); + memzero_explicit(ksm, struct_size(ksm, slots, ksm->num_slots)); + kvfree(ksm); + } +} +EXPORT_SYMBOL_GPL(keyslot_manager_destroy); + +/** + * keyslot_manager_create_passthrough() - Create a passthrough keyslot manager + * @dev: Device for runtime power management (NULL if none) + * @ksm_ll_ops: The struct keyslot_mgmt_ll_ops + * @features: Bitmask of BLK_CRYPTO_FEATURE_* flags + * @crypto_mode_supported: Bitmasks for supported encryption modes + * @ll_priv_data: Private data passed as is to the functions in ksm_ll_ops. + * + * Allocate memory for and initialize a passthrough keyslot manager. + * Called by e.g. storage drivers to set up a keyslot manager in their + * request_queue, when the storage driver wants to manage its keys by itself. + * This is useful for inline encryption hardware that don't have a small fixed + * number of keyslots, and for layered devices. + * + * See keyslot_manager_create() for more details about the parameters. + * + * Context: This function may sleep + * Return: Pointer to constructed keyslot manager or NULL on error. + */ +struct keyslot_manager *keyslot_manager_create_passthrough( + struct device *dev, + const struct keyslot_mgmt_ll_ops *ksm_ll_ops, + unsigned int features, + const unsigned int crypto_mode_supported[BLK_ENCRYPTION_MODE_MAX], + void *ll_priv_data) +{ + struct keyslot_manager *ksm; + + ksm = kzalloc(sizeof(*ksm), GFP_KERNEL); + if (!ksm) + return NULL; + + ksm->ksm_ll_ops = *ksm_ll_ops; + ksm->features = features; + memcpy(ksm->crypto_mode_supported, crypto_mode_supported, + sizeof(ksm->crypto_mode_supported)); + ksm->max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE; + ksm->ll_priv_data = ll_priv_data; + keyslot_manager_set_dev(ksm, dev); + + init_rwsem(&ksm->lock); + + return ksm; +} +EXPORT_SYMBOL_GPL(keyslot_manager_create_passthrough); + +/** + * keyslot_manager_intersect_modes() - restrict supported modes by child device + * @parent: The keyslot manager for parent device + * @child: The keyslot manager for child device, or NULL + * + * Clear any crypto mode support bits in @parent that aren't set in @child. + * If @child is NULL, then all parent bits are cleared. + * + * Only use this when setting up the keyslot manager for a layered device, + * before it's been exposed yet. + */ +void keyslot_manager_intersect_modes(struct keyslot_manager *parent, + const struct keyslot_manager *child) +{ + if (child) { + unsigned int i; + + parent->features &= child->features; + parent->max_dun_bytes_supported = + min(parent->max_dun_bytes_supported, + child->max_dun_bytes_supported); + for (i = 0; i < ARRAY_SIZE(child->crypto_mode_supported); i++) { + parent->crypto_mode_supported[i] &= + child->crypto_mode_supported[i]; + } + } else { + parent->features = 0; + parent->max_dun_bytes_supported = 0; + memset(parent->crypto_mode_supported, 0, + sizeof(parent->crypto_mode_supported)); + } +} +EXPORT_SYMBOL_GPL(keyslot_manager_intersect_modes); + +/** + * keyslot_manager_derive_raw_secret() - Derive software secret from wrapped key + * @ksm: The keyslot manager + * @wrapped_key: The wrapped key + * @wrapped_key_size: Size of the wrapped key in bytes + * @secret: (output) the software secret + * @secret_size: (output) the number of secret bytes to derive + * + * Given a hardware-wrapped key, ask the hardware to derive a secret which + * software can use for cryptographic tasks other than inline encryption. The + * derived secret is guaranteed to be cryptographically isolated from the key + * with which any inline encryption with this wrapped key would actually be + * done. I.e., both will be derived from the unwrapped key. + * + * Return: 0 on success, -EOPNOTSUPP if hardware-wrapped keys are unsupported, + * or another -errno code. + */ +int keyslot_manager_derive_raw_secret(struct keyslot_manager *ksm, + const u8 *wrapped_key, + unsigned int wrapped_key_size, + u8 *secret, unsigned int secret_size) +{ + int err; + + if (ksm->ksm_ll_ops.derive_raw_secret) { + keyslot_manager_hw_enter(ksm); + err = ksm->ksm_ll_ops.derive_raw_secret(ksm, wrapped_key, + wrapped_key_size, + secret, secret_size); + keyslot_manager_hw_exit(ksm); + } else { + err = -EOPNOTSUPP; + } + + return err; +} +EXPORT_SYMBOL_GPL(keyslot_manager_derive_raw_secret); |