Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (c) 2025, Christoph Hellwig. * Copyright (c) 2025, Western Digital Corporation or its affiliates. * * Zoned Loop Device driver - exports a zoned block device using one file per * zone as backing storage. */#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/module.h>#include <linux/blk-mq.h>#include <linux/blkzoned.h>#include <linux/pagemap.h>#include <linux/miscdevice.h>#include <linux/falloc.h>#include <linux/mutex.h>#include <linux/parser.h>#include <linux/seq_file.h>#include <linux/xattr.h>
/* * Options for adding (and removing) a device. */enum { ZLOOP_OPT_ERR = 0, ZLOOP_OPT_ID = (1 << 0), ZLOOP_OPT_CAPACITY = (1 << 1), ZLOOP_OPT_ZONE_SIZE = (1 << 2), ZLOOP_OPT_ZONE_CAPACITY = (1 << 3), ZLOOP_OPT_NR_CONV_ZONES = (1 << 4), ZLOOP_OPT_BASE_DIR = (1 << 5), ZLOOP_OPT_NR_QUEUES = (1 << 6), ZLOOP_OPT_QUEUE_DEPTH = (1 << 7), ZLOOP_OPT_BUFFERED_IO = (1 << 8), ZLOOP_OPT_ZONE_APPEND = (1 << 9), ZLOOP_OPT_ORDERED_ZONE_APPEND = (1 << 10), ZLOOP_OPT_DISCARD_WRITE_CACHE = (1 << 11), ZLOOP_OPT_MAX_OPEN_ZONES = (1 << 12),};
static const match_table_t zloop_opt_tokens = { { ZLOOP_OPT_ID, "id=%d" }, { ZLOOP_OPT_CAPACITY, "capacity_mb=%u" }, { ZLOOP_OPT_ZONE_SIZE, "zone_size_mb=%u" }, { ZLOOP_OPT_ZONE_CAPACITY, "zone_capacity_mb=%u" }, { ZLOOP_OPT_NR_CONV_ZONES, "conv_zones=%u" }, { ZLOOP_OPT_BASE_DIR, "base_dir=%s" }, { ZLOOP_OPT_NR_QUEUES, "nr_queues=%u" }, { ZLOOP_OPT_QUEUE_DEPTH, "queue_depth=%u" }, { ZLOOP_OPT_BUFFERED_IO, "buffered_io" }, { ZLOOP_OPT_ZONE_APPEND, "zone_append=%u" }, { ZLOOP_OPT_ORDERED_ZONE_APPEND, "ordered_zone_append" }, { ZLOOP_OPT_DISCARD_WRITE_CACHE, "discard_write_cache" }, { ZLOOP_OPT_MAX_OPEN_ZONES, "max_open_zones=%u" }, { ZLOOP_OPT_ERR, NULL }};
/* Default values for the "add" operation. */#define ZLOOP_DEF_ID -1#define ZLOOP_DEF_ZONE_SIZE ((256ULL * SZ_1M) >> SECTOR_SHIFT)#define ZLOOP_DEF_NR_ZONES 64#define ZLOOP_DEF_NR_CONV_ZONES 8#define ZLOOP_DEF_MAX_OPEN_ZONES 0#define ZLOOP_DEF_BASE_DIR "/var/local/zloop"#define ZLOOP_DEF_NR_QUEUES 1#define ZLOOP_DEF_QUEUE_DEPTH 128#define ZLOOP_DEF_BUFFERED_IO false#define ZLOOP_DEF_ZONE_APPEND true#define ZLOOP_DEF_ORDERED_ZONE_APPEND false
/* Arbitrary limit on the zone size (16GB). */#define ZLOOP_MAX_ZONE_SIZE_MB 16384
struct zloop_options { unsigned int mask; int id; sector_t capacity; sector_t zone_size; sector_t zone_capacity; unsigned int nr_conv_zones; unsigned int max_open_zones; char *base_dir; unsigned int nr_queues; unsigned int queue_depth; bool buffered_io; bool zone_append; bool ordered_zone_append; bool discard_write_cache;};
/* * Device states. */enum { Zlo_creating = 0, Zlo_live, Zlo_deleting,};
enum zloop_zone_flags { ZLOOP_ZONE_CONV = 0, ZLOOP_ZONE_SEQ_ERROR,};
/* * Zone descriptor. * Locking order: z.lock -> z.wp_lock -> zlo.open_zones_lock */struct zloop_zone { struct list_head open_zone_entry; struct file *file;
unsigned long flags; struct mutex lock; spinlock_t wp_lock; enum blk_zone_cond cond; sector_t start; sector_t wp;
gfp_t old_gfp_mask;};
struct zloop_device { unsigned int id; unsigned int state;
struct blk_mq_tag_set tag_set; struct gendisk *disk;
struct workqueue_struct *workqueue; bool buffered_io; bool zone_append; bool ordered_zone_append; bool discard_write_cache;
const char *base_dir; struct file *data_dir;
unsigned int zone_shift; sector_t zone_size; sector_t zone_capacity; unsigned int nr_zones; unsigned int nr_conv_zones; unsigned int max_open_zones; unsigned int block_size;
spinlock_t open_zones_lock; struct list_head open_zones_lru_list; unsigned int nr_open_zones;
struct zloop_zone zones[] __counted_by(nr_zones);};
struct zloop_cmd { struct work_struct work; atomic_t ref; sector_t sector; sector_t nr_sectors; long ret; struct kiocb iocb; struct bio_vec *bvec;};
static DEFINE_IDR(zloop_index_idr);static DEFINE_MUTEX(zloop_ctl_mutex);
static unsigned int rq_zone_no(struct request *rq){ struct zloop_device *zlo = rq->q->queuedata;
return blk_rq_pos(rq) >> zlo->zone_shift;}
/* * Open an already open zone. This is mostly a no-op, except for the imp open -> * exp open condition change that may happen. We also move a zone at the tail of * the list of open zones so that if we need to * implicitly close one open zone, we can do so in LRU order. */static inline void zloop_lru_rotate_open_zone(struct zloop_device *zlo, struct zloop_zone *zone){ if (zlo->max_open_zones) { spin_lock(&zlo->open_zones_lock); list_move_tail(&zone->open_zone_entry, &zlo->open_zones_lru_list); spin_unlock(&zlo->open_zones_lock); }}
static inline void zloop_lru_remove_open_zone(struct zloop_device *zlo, struct zloop_zone *zone){ if (zone->cond == BLK_ZONE_COND_IMP_OPEN || zone->cond == BLK_ZONE_COND_EXP_OPEN) { spin_lock(&zlo->open_zones_lock); list_del_init(&zone->open_zone_entry); zlo->nr_open_zones--; spin_unlock(&zlo->open_zones_lock); }}
static inline bool zloop_can_open_zone(struct zloop_device *zlo){ return !zlo->max_open_zones || zlo->nr_open_zones < zlo->max_open_zones;}
/* * If we have reached the maximum open zones limit, attempt to close an * implicitly open zone (if we have any) so that we can implicitly open another * zone without exceeding the maximum number of open zones. */static bool zloop_close_imp_open_zone(struct zloop_device *zlo){ struct zloop_zone *zone;
lockdep_assert_held(&zlo->open_zones_lock);
if (zloop_can_open_zone(zlo)) return true;
list_for_each_entry(zone, &zlo->open_zones_lru_list, open_zone_entry) { if (zone->cond == BLK_ZONE_COND_IMP_OPEN) { zone->cond = BLK_ZONE_COND_CLOSED; list_del_init(&zone->open_zone_entry); zlo->nr_open_zones--; return true; } }
return false;}
static bool zloop_open_closed_or_empty_zone(struct zloop_device *zlo, struct zloop_zone *zone, bool explicit){ spin_lock(&zlo->open_zones_lock);
if (explicit) { /* * Explicit open: we cannot allow this if we have reached the * maximum open zones limit. */ if (!zloop_can_open_zone(zlo)) goto fail; zone->cond = BLK_ZONE_COND_EXP_OPEN; } else { /* * Implicit open case: if we have reached the maximum open zones * limit, try to close an implicitly open zone first. */ if (!zloop_close_imp_open_zone(zlo)) goto fail; zone->cond = BLK_ZONE_COND_IMP_OPEN; }
zlo->nr_open_zones++; list_add_tail(&zone->open_zone_entry, &zlo->open_zones_lru_list);
spin_unlock(&zlo->open_zones_lock);
return true;
fail: spin_unlock(&zlo->open_zones_lock);
return false;}
static bool zloop_do_open_zone(struct zloop_device *zlo, struct zloop_zone *zone, bool explicit){ switch (zone->cond) { case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: if (explicit) zone->cond = BLK_ZONE_COND_EXP_OPEN; zloop_lru_rotate_open_zone(zlo, zone); return true; case BLK_ZONE_COND_EMPTY: case BLK_ZONE_COND_CLOSED: return zloop_open_closed_or_empty_zone(zlo, zone, explicit); default: return false; }}
static void zloop_mark_full(struct zloop_device *zlo, struct zloop_zone *zone){ lockdep_assert_held(&zone->wp_lock);
zloop_lru_remove_open_zone(zlo, zone); zone->cond = BLK_ZONE_COND_FULL; zone->wp = ULLONG_MAX;}
static void zloop_mark_empty(struct zloop_device *zlo, struct zloop_zone *zone){ lockdep_assert_held(&zone->wp_lock);
zloop_lru_remove_open_zone(zlo, zone); zone->cond = BLK_ZONE_COND_EMPTY; zone->wp = zone->start;}
static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no){ struct zloop_zone *zone = &zlo->zones[zone_no]; struct kstat stat; sector_t file_sectors; int ret;
lockdep_assert_held(&zone->lock);
ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0); if (ret < 0) { pr_err("Failed to get zone %u file stat (err=%d)\n", zone_no, ret); set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); return ret; }
file_sectors = stat.size >> SECTOR_SHIFT; if (file_sectors > zlo->zone_capacity) { pr_err("Zone %u file too large (%llu sectors > %llu)\n", zone_no, file_sectors, zlo->zone_capacity); return -EINVAL; }
if (!IS_ALIGNED(stat.size, zlo->block_size)) { pr_err("Zone %u file size (%llu) not aligned to block size %u\n", zone_no, stat.size, zlo->block_size); return -EINVAL; }
spin_lock(&zone->wp_lock); if (!file_sectors) { zloop_mark_empty(zlo, zone); } else if (file_sectors == zlo->zone_capacity) { zloop_mark_full(zlo, zone); } else { if (zone->cond != BLK_ZONE_COND_IMP_OPEN && zone->cond != BLK_ZONE_COND_EXP_OPEN) zone->cond = BLK_ZONE_COND_CLOSED; zone->wp = zone->start + file_sectors; } spin_unlock(&zone->wp_lock);
return 0;}
static int zloop_open_zone(struct zloop_device *zlo, unsigned int zone_no){ struct zloop_zone *zone = &zlo->zones[zone_no]; int ret = 0;
if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) return -EIO;
mutex_lock(&zone->lock);
if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { ret = zloop_update_seq_zone(zlo, zone_no); if (ret) goto unlock; }
if (!zloop_do_open_zone(zlo, zone, true)) ret = -EIO;
unlock: mutex_unlock(&zone->lock);
return ret;}
static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no){ struct zloop_zone *zone = &zlo->zones[zone_no]; int ret = 0;
if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) return -EIO;
mutex_lock(&zone->lock);
if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { ret = zloop_update_seq_zone(zlo, zone_no); if (ret) goto unlock; }
switch (zone->cond) { case BLK_ZONE_COND_CLOSED: break; case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: spin_lock(&zone->wp_lock); zloop_lru_remove_open_zone(zlo, zone); if (zone->wp == zone->start) zone->cond = BLK_ZONE_COND_EMPTY; else zone->cond = BLK_ZONE_COND_CLOSED; spin_unlock(&zone->wp_lock); break; case BLK_ZONE_COND_EMPTY: case BLK_ZONE_COND_FULL: default: ret = -EIO; break; }
unlock: mutex_unlock(&zone->lock);
return ret;}
static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no){ struct zloop_zone *zone = &zlo->zones[zone_no]; int ret = 0;
if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) return -EIO;
mutex_lock(&zone->lock);
if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) && zone->cond == BLK_ZONE_COND_EMPTY) goto unlock;
if (vfs_truncate(&zone->file->f_path, 0)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; }
spin_lock(&zone->wp_lock); zloop_mark_empty(zlo, zone); clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); spin_unlock(&zone->wp_lock);
unlock: mutex_unlock(&zone->lock);
return ret;}
static int zloop_reset_all_zones(struct zloop_device *zlo){ unsigned int i; int ret;
for (i = zlo->nr_conv_zones; i < zlo->nr_zones; i++) { ret = zloop_reset_zone(zlo, i); if (ret) return ret; }
return 0;}
static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no){ struct zloop_zone *zone = &zlo->zones[zone_no]; int ret = 0;
if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) return -EIO;
mutex_lock(&zone->lock);
if (!test_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags) && zone->cond == BLK_ZONE_COND_FULL) goto unlock;
if (vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT)) { set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); ret = -EIO; goto unlock; }
spin_lock(&zone->wp_lock); zloop_mark_full(zlo, zone); clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); spin_unlock(&zone->wp_lock);
unlock: mutex_unlock(&zone->lock);
return ret;}
static void zloop_put_cmd(struct zloop_cmd *cmd){ struct request *rq = blk_mq_rq_from_pdu(cmd);
if (!atomic_dec_and_test(&cmd->ref)) return; kfree(cmd->bvec); cmd->bvec = NULL; if (likely(!blk_should_fake_timeout(rq->q))) blk_mq_complete_request(rq);}
static void zloop_rw_complete(struct kiocb *iocb, long ret){ struct zloop_cmd *cmd = container_of(iocb, struct zloop_cmd, iocb);
cmd->ret = ret; zloop_put_cmd(cmd);}
static int zloop_do_rw(struct zloop_cmd *cmd){ struct request *rq = blk_mq_rq_from_pdu(cmd); int rw = req_op(rq) == REQ_OP_READ ? ITER_DEST : ITER_SOURCE; unsigned int nr_bvec = blk_rq_nr_bvec(rq); struct zloop_device *zlo = rq->q->queuedata; struct zloop_zone *zone = &zlo->zones[rq_zone_no(rq)]; struct req_iterator rq_iter; struct iov_iter iter;
if (rq->bio != rq->biotail) { struct bio_vec tmp, *bvec;
cmd->bvec = kmalloc_objs(*cmd->bvec, nr_bvec, GFP_NOIO); if (!cmd->bvec) return -EIO;
/* * The bios of the request may be started from the middle of * the 'bvec' because of bio splitting, so we can't directly * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec * API will take care of all details for us. */ bvec = cmd->bvec; rq_for_each_bvec(tmp, rq, rq_iter) { *bvec = tmp; bvec++; } iov_iter_bvec(&iter, rw, cmd->bvec, nr_bvec, blk_rq_bytes(rq)); } else { /* * Same here, this bio may be started from the middle of the * 'bvec' because of bio splitting, so offset from the bvec * must be passed to iov iterator */ iov_iter_bvec(&iter, rw, __bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter), nr_bvec, blk_rq_bytes(rq)); iter.iov_offset = rq->bio->bi_iter.bi_bvec_done; }
cmd->iocb.ki_pos = (cmd->sector - zone->start) << SECTOR_SHIFT; cmd->iocb.ki_filp = zone->file; cmd->iocb.ki_complete = zloop_rw_complete; if (!zlo->buffered_io) cmd->iocb.ki_flags = IOCB_DIRECT; cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
if (rw == ITER_SOURCE) return zone->file->f_op->write_iter(&cmd->iocb, &iter); return zone->file->f_op->read_iter(&cmd->iocb, &iter);}
static int zloop_seq_write_prep(struct zloop_cmd *cmd){ struct request *rq = blk_mq_rq_from_pdu(cmd); struct zloop_device *zlo = rq->q->queuedata; unsigned int zone_no = rq_zone_no(rq); sector_t nr_sectors = blk_rq_sectors(rq); bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; struct zloop_zone *zone = &zlo->zones[zone_no]; sector_t zone_end = zone->start + zlo->zone_capacity; int ret = 0;
spin_lock(&zone->wp_lock);
/* * Zone append operations always go at the current write pointer, but * regular write operations must already be aligned to the write pointer * when submitted. */ if (is_append) { /* * If ordered zone append is in use, we already checked and set * the target sector in zloop_queue_rq(). */ if (!zlo->ordered_zone_append) { if (zone->cond == BLK_ZONE_COND_FULL || zone->wp + nr_sectors > zone_end) { ret = -EIO; goto out_unlock; } cmd->sector = zone->wp; } } else { if (cmd->sector != zone->wp) { pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n", zone_no, cmd->sector, zone->wp); ret = -EIO; goto out_unlock; } }
/* Implicitly open the target zone. */ if (!zloop_do_open_zone(zlo, zone, false)) { ret = -EIO; goto out_unlock; }
/* * Advance the write pointer, unless ordered zone append is in use. If * the write fails, the write pointer position will be corrected when * the next I/O starts execution. */ if (!is_append || !zlo->ordered_zone_append) { zone->wp += nr_sectors; if (zone->wp == zone_end) zloop_mark_full(zlo, zone); }out_unlock: spin_unlock(&zone->wp_lock); return ret;}
static void zloop_rw(struct zloop_cmd *cmd){ struct request *rq = blk_mq_rq_from_pdu(cmd); struct zloop_device *zlo = rq->q->queuedata; unsigned int zone_no = rq_zone_no(rq); sector_t nr_sectors = blk_rq_sectors(rq); bool is_append = req_op(rq) == REQ_OP_ZONE_APPEND; bool is_write = req_op(rq) == REQ_OP_WRITE || is_append; struct zloop_zone *zone; int ret = -EIO;
atomic_set(&cmd->ref, 2); cmd->sector = blk_rq_pos(rq); cmd->nr_sectors = nr_sectors; cmd->ret = 0;
if (WARN_ON_ONCE(is_append && !zlo->zone_append)) goto out;
/* We should never get an I/O beyond the device capacity. */ if (WARN_ON_ONCE(zone_no >= zlo->nr_zones)) goto out;
zone = &zlo->zones[zone_no];
/* * The block layer should never send requests that are not fully * contained within the zone. */ if (WARN_ON_ONCE(cmd->sector + nr_sectors > zone->start + zlo->zone_size)) goto out;
if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { mutex_lock(&zone->lock); ret = zloop_update_seq_zone(zlo, zone_no); mutex_unlock(&zone->lock); if (ret) goto out; }
if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) { mutex_lock(&zone->lock); ret = zloop_seq_write_prep(cmd); if (!ret) ret = zloop_do_rw(cmd); mutex_unlock(&zone->lock); } else { ret = zloop_do_rw(cmd); }out: if (ret != -EIOCBQUEUED) zloop_rw_complete(&cmd->iocb, ret); zloop_put_cmd(cmd);}
static inline bool zloop_zone_is_active(struct zloop_zone *zone){ switch (zone->cond) { case BLK_ZONE_COND_EXP_OPEN: case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_CLOSED: return true; default: return false; }}
static int zloop_record_safe_wps(struct zloop_device *zlo){ unsigned int i; int ret;
for (i = 0; i < zlo->nr_zones; i++) { struct zloop_zone *zone = &zlo->zones[i]; struct file *file = zone->file;
if (!zloop_zone_is_active(zone)) continue; ret = vfs_setxattr(file_mnt_idmap(file), file_dentry(file), "user.zloop.wp", &zone->wp, sizeof(zone->wp), 0); if (ret) { pr_err("%pg: failed to record write pointer (%d)\n", zlo->disk->part0, ret); return ret; } }
return 0;}
/* * Sync the entire FS containing the zone files instead of walking all files. */static int zloop_flush(struct zloop_device *zlo){ struct super_block *sb = file_inode(zlo->data_dir)->i_sb; int ret;
if (zlo->discard_write_cache) { ret = zloop_record_safe_wps(zlo); if (ret) return ret; }
down_read(&sb->s_umount); ret = sync_filesystem(sb); up_read(&sb->s_umount);
return ret;}
static void zloop_handle_cmd(struct zloop_cmd *cmd){ struct request *rq = blk_mq_rq_from_pdu(cmd); struct zloop_device *zlo = rq->q->queuedata;
/* We can block in this context, so ignore REQ_NOWAIT. */ if (rq->cmd_flags & REQ_NOWAIT) rq->cmd_flags &= ~REQ_NOWAIT;
switch (req_op(rq)) { case REQ_OP_READ: case REQ_OP_WRITE: case REQ_OP_ZONE_APPEND: /* * zloop_rw() always executes asynchronously or completes * directly. */ zloop_rw(cmd); return; case REQ_OP_FLUSH: cmd->ret = zloop_flush(zlo); break; case REQ_OP_ZONE_RESET: cmd->ret = zloop_reset_zone(zlo, rq_zone_no(rq)); break; case REQ_OP_ZONE_RESET_ALL: cmd->ret = zloop_reset_all_zones(zlo); break; case REQ_OP_ZONE_FINISH: cmd->ret = zloop_finish_zone(zlo, rq_zone_no(rq)); break; case REQ_OP_ZONE_OPEN: cmd->ret = zloop_open_zone(zlo, rq_zone_no(rq)); break; case REQ_OP_ZONE_CLOSE: cmd->ret = zloop_close_zone(zlo, rq_zone_no(rq)); break; default: WARN_ON_ONCE(1); pr_err("Unsupported operation %d\n", req_op(rq)); cmd->ret = -EOPNOTSUPP; break; }
blk_mq_complete_request(rq);}
static void zloop_cmd_workfn(struct work_struct *work){ struct zloop_cmd *cmd = container_of(work, struct zloop_cmd, work); int orig_flags = current->flags;
current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; zloop_handle_cmd(cmd); current->flags = orig_flags;}
static void zloop_complete_rq(struct request *rq){ struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); struct zloop_device *zlo = rq->q->queuedata; unsigned int zone_no = cmd->sector >> zlo->zone_shift; struct zloop_zone *zone = &zlo->zones[zone_no]; blk_status_t sts = BLK_STS_OK;
switch (req_op(rq)) { case REQ_OP_READ: if (cmd->ret < 0) pr_err("Zone %u: failed read sector %llu, %llu sectors\n", zone_no, cmd->sector, cmd->nr_sectors);
if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) { /* short read */ struct bio *bio;
__rq_for_each_bio(bio, rq) zero_fill_bio(bio); } break; case REQ_OP_WRITE: case REQ_OP_ZONE_APPEND: if (cmd->ret < 0) pr_err("Zone %u: failed %swrite sector %llu, %llu sectors\n", zone_no, req_op(rq) == REQ_OP_WRITE ? "" : "append ", cmd->sector, cmd->nr_sectors);
if (cmd->ret >= 0 && cmd->ret != blk_rq_bytes(rq)) { pr_err("Zone %u: partial write %ld/%u B\n", zone_no, cmd->ret, blk_rq_bytes(rq)); cmd->ret = -EIO; }
if (cmd->ret < 0 && !test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { /* * A write to a sequential zone file failed: mark the * zone as having an error. This will be corrected and * cleared when the next IO is submitted. */ set_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); break; } if (req_op(rq) == REQ_OP_ZONE_APPEND) rq->__sector = cmd->sector;
break; default: break; }
if (cmd->ret < 0) sts = errno_to_blk_status(cmd->ret); blk_mq_end_request(rq, sts);}
static bool zloop_set_zone_append_sector(struct request *rq){ struct zloop_device *zlo = rq->q->queuedata; unsigned int zone_no = rq_zone_no(rq); struct zloop_zone *zone = &zlo->zones[zone_no]; sector_t zone_end = zone->start + zlo->zone_capacity; sector_t nr_sectors = blk_rq_sectors(rq);
spin_lock(&zone->wp_lock);
if (zone->cond == BLK_ZONE_COND_FULL || zone->wp + nr_sectors > zone_end) { spin_unlock(&zone->wp_lock); return false; }
rq->__sector = zone->wp; zone->wp += blk_rq_sectors(rq); if (zone->wp >= zone_end) zloop_mark_full(zlo, zone);
spin_unlock(&zone->wp_lock);
return true;}
static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd){ struct request *rq = bd->rq; struct zloop_cmd *cmd = blk_mq_rq_to_pdu(rq); struct zloop_device *zlo = rq->q->queuedata;
if (data_race(READ_ONCE(zlo->state)) == Zlo_deleting) { rq->rq_flags |= RQF_QUIET; return BLK_STS_IOERR; }
/* * If we need to strongly order zone append operations, set the request * sector to the zone write pointer location now instead of when the * command work runs. */ if (zlo->ordered_zone_append && req_op(rq) == REQ_OP_ZONE_APPEND) { if (!zloop_set_zone_append_sector(rq)) return BLK_STS_IOERR; }
blk_mq_start_request(rq);
INIT_WORK(&cmd->work, zloop_cmd_workfn); queue_work(zlo->workqueue, &cmd->work);
return BLK_STS_OK;}
static const struct blk_mq_ops zloop_mq_ops = { .queue_rq = zloop_queue_rq, .complete = zloop_complete_rq,};
static int zloop_open(struct gendisk *disk, blk_mode_t mode){ struct zloop_device *zlo = disk->private_data; int ret;
ret = mutex_lock_killable(&zloop_ctl_mutex); if (ret) return ret;
if (zlo->state != Zlo_live) ret = -ENXIO; mutex_unlock(&zloop_ctl_mutex); return ret;}
static int zloop_report_zones(struct gendisk *disk, sector_t sector, unsigned int nr_zones, struct blk_report_zones_args *args){ struct zloop_device *zlo = disk->private_data; struct blk_zone blkz = {}; unsigned int first, i; int ret;
first = disk_zone_no(disk, sector); if (first >= zlo->nr_zones) return 0; nr_zones = min(nr_zones, zlo->nr_zones - first);
for (i = 0; i < nr_zones; i++) { unsigned int zone_no = first + i; struct zloop_zone *zone = &zlo->zones[zone_no];
mutex_lock(&zone->lock);
if (test_and_clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags)) { ret = zloop_update_seq_zone(zlo, zone_no); if (ret) { mutex_unlock(&zone->lock); return ret; } }
blkz.start = zone->start; blkz.len = zlo->zone_size; spin_lock(&zone->wp_lock); blkz.wp = zone->wp; spin_unlock(&zone->wp_lock); blkz.cond = zone->cond; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { blkz.type = BLK_ZONE_TYPE_CONVENTIONAL; blkz.capacity = zlo->zone_size; } else { blkz.type = BLK_ZONE_TYPE_SEQWRITE_REQ; blkz.capacity = zlo->zone_capacity; }
mutex_unlock(&zone->lock);
ret = disk_report_zone(disk, &blkz, i, args); if (ret) return ret; }
return nr_zones;}
static void zloop_free_disk(struct gendisk *disk){ struct zloop_device *zlo = disk->private_data; unsigned int i;
blk_mq_free_tag_set(&zlo->tag_set);
for (i = 0; i < zlo->nr_zones; i++) { struct zloop_zone *zone = &zlo->zones[i];
mapping_set_gfp_mask(zone->file->f_mapping, zone->old_gfp_mask); fput(zone->file); }
fput(zlo->data_dir); destroy_workqueue(zlo->workqueue); kfree(zlo->base_dir); kvfree(zlo);}
static const struct block_device_operations zloop_fops = { .owner = THIS_MODULE, .open = zloop_open, .report_zones = zloop_report_zones, .free_disk = zloop_free_disk,};
__printf(3, 4)static struct file *zloop_filp_open_fmt(int oflags, umode_t mode, const char *fmt, ...){ struct file *file; va_list ap; char *p;
va_start(ap, fmt); p = kvasprintf(GFP_KERNEL, fmt, ap); va_end(ap);
if (!p) return ERR_PTR(-ENOMEM); file = filp_open(p, oflags, mode); kfree(p); return file;}
static int zloop_get_block_size(struct zloop_device *zlo, struct zloop_zone *zone){ struct block_device *sb_bdev = zone->file->f_mapping->host->i_sb->s_bdev; struct kstat st;
/* * If the FS block size is lower than or equal to 4K, use that as the * device block size. Otherwise, fallback to the FS direct IO alignment * constraint if that is provided, and to the FS underlying device * physical block size if the direct IO alignment is unknown. */ if (file_inode(zone->file)->i_sb->s_blocksize <= SZ_4K) zlo->block_size = file_inode(zone->file)->i_sb->s_blocksize; else if (!vfs_getattr(&zone->file->f_path, &st, STATX_DIOALIGN, 0) && (st.result_mask & STATX_DIOALIGN)) zlo->block_size = st.dio_offset_align; else if (sb_bdev) zlo->block_size = bdev_physical_block_size(sb_bdev); else zlo->block_size = SECTOR_SIZE;
if (zlo->zone_capacity & ((zlo->block_size >> SECTOR_SHIFT) - 1)) { pr_err("Zone capacity is not aligned to block size %u\n", zlo->block_size); return -EINVAL; }
return 0;}
static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, unsigned int zone_no, bool restore){ struct zloop_zone *zone = &zlo->zones[zone_no]; int oflags = O_RDWR; struct kstat stat; sector_t file_sectors; int ret;
mutex_init(&zone->lock); INIT_LIST_HEAD(&zone->open_zone_entry); spin_lock_init(&zone->wp_lock); zone->start = (sector_t)zone_no << zlo->zone_shift;
if (!restore) oflags |= O_CREAT;
if (!opts->buffered_io) oflags |= O_DIRECT;
if (zone_no < zlo->nr_conv_zones) { /* Conventional zone file. */ set_bit(ZLOOP_ZONE_CONV, &zone->flags); zone->cond = BLK_ZONE_COND_NOT_WP; zone->wp = U64_MAX;
zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/cnv-%06u", zlo->base_dir, zlo->id, zone_no); if (IS_ERR(zone->file)) { pr_err("Failed to open zone %u file %s/%u/cnv-%06u (err=%ld)", zone_no, zlo->base_dir, zlo->id, zone_no, PTR_ERR(zone->file)); return PTR_ERR(zone->file); }
if (!zlo->block_size) { ret = zloop_get_block_size(zlo, zone); if (ret) return ret; }
ret = vfs_getattr(&zone->file->f_path, &stat, STATX_SIZE, 0); if (ret < 0) { pr_err("Failed to get zone %u file stat\n", zone_no); return ret; } file_sectors = stat.size >> SECTOR_SHIFT;
if (restore && file_sectors != zlo->zone_size) { pr_err("Invalid conventional zone %u file size (%llu sectors != %llu)\n", zone_no, file_sectors, zlo->zone_capacity); return ret; }
ret = vfs_truncate(&zone->file->f_path, zlo->zone_size << SECTOR_SHIFT); if (ret < 0) { pr_err("Failed to truncate zone %u file (err=%d)\n", zone_no, ret); return ret; }
return 0; }
/* Sequential zone file. */ zone->file = zloop_filp_open_fmt(oflags, 0600, "%s/%u/seq-%06u", zlo->base_dir, zlo->id, zone_no); if (IS_ERR(zone->file)) { pr_err("Failed to open zone %u file %s/%u/seq-%06u (err=%ld)", zone_no, zlo->base_dir, zlo->id, zone_no, PTR_ERR(zone->file)); return PTR_ERR(zone->file); }
if (!zlo->block_size) { ret = zloop_get_block_size(zlo, zone); if (ret) return ret; }
zloop_get_block_size(zlo, zone);
mutex_lock(&zone->lock); ret = zloop_update_seq_zone(zlo, zone_no); mutex_unlock(&zone->lock);
return ret;}
static bool zloop_dev_exists(struct zloop_device *zlo){ struct file *cnv, *seq; bool exists;
cnv = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/cnv-%06u", zlo->base_dir, zlo->id, 0); seq = zloop_filp_open_fmt(O_RDONLY, 0600, "%s/%u/seq-%06u", zlo->base_dir, zlo->id, 0); exists = !IS_ERR(cnv) || !IS_ERR(seq);
if (!IS_ERR(cnv)) fput(cnv); if (!IS_ERR(seq)) fput(seq);
return exists;}
static int zloop_ctl_add(struct zloop_options *opts){ struct queue_limits lim = { .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, .chunk_sectors = opts->zone_size, .features = BLK_FEAT_ZONED | BLK_FEAT_WRITE_CACHE,
}; unsigned int nr_zones, i, j; struct zloop_device *zlo; int ret = -EINVAL; bool restore;
__module_get(THIS_MODULE);
nr_zones = opts->capacity >> ilog2(opts->zone_size); if (opts->nr_conv_zones >= nr_zones) { pr_err("Invalid number of conventional zones %u\n", opts->nr_conv_zones); goto out; }
if (opts->max_open_zones > nr_zones - opts->nr_conv_zones) { pr_err("Invalid maximum number of open zones %u\n", opts->max_open_zones); goto out; }
zlo = kvzalloc_flex(*zlo, zones, nr_zones); if (!zlo) { ret = -ENOMEM; goto out; } WRITE_ONCE(zlo->state, Zlo_creating); spin_lock_init(&zlo->open_zones_lock); INIT_LIST_HEAD(&zlo->open_zones_lru_list);
ret = mutex_lock_killable(&zloop_ctl_mutex); if (ret) goto out_free_dev;
/* Allocate id, if @opts->id >= 0, we're requesting that specific id */ if (opts->id >= 0) { ret = idr_alloc(&zloop_index_idr, zlo, opts->id, opts->id + 1, GFP_KERNEL); if (ret == -ENOSPC) ret = -EEXIST; } else { ret = idr_alloc(&zloop_index_idr, zlo, 0, 0, GFP_KERNEL); } mutex_unlock(&zloop_ctl_mutex); if (ret < 0) goto out_free_dev;
zlo->id = ret; zlo->zone_shift = ilog2(opts->zone_size); zlo->zone_size = opts->zone_size; if (opts->zone_capacity) zlo->zone_capacity = opts->zone_capacity; else zlo->zone_capacity = zlo->zone_size; zlo->nr_zones = nr_zones; zlo->nr_conv_zones = opts->nr_conv_zones; zlo->max_open_zones = opts->max_open_zones; zlo->buffered_io = opts->buffered_io; zlo->zone_append = opts->zone_append; if (zlo->zone_append) zlo->ordered_zone_append = opts->ordered_zone_append; zlo->discard_write_cache = opts->discard_write_cache;
zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE, opts->nr_queues * opts->queue_depth, zlo->id); if (!zlo->workqueue) { ret = -ENOMEM; goto out_free_idr; }
if (opts->base_dir) zlo->base_dir = kstrdup(opts->base_dir, GFP_KERNEL); else zlo->base_dir = kstrdup(ZLOOP_DEF_BASE_DIR, GFP_KERNEL); if (!zlo->base_dir) { ret = -ENOMEM; goto out_destroy_workqueue; }
zlo->data_dir = zloop_filp_open_fmt(O_RDONLY | O_DIRECTORY, 0, "%s/%u", zlo->base_dir, zlo->id); if (IS_ERR(zlo->data_dir)) { ret = PTR_ERR(zlo->data_dir); pr_warn("Failed to open directory %s/%u (err=%d)\n", zlo->base_dir, zlo->id, ret); goto out_free_base_dir; }
/* * If we already have zone files, we are restoring a device created by a * previous add operation. In this case, zloop_init_zone() will check * that the zone files are consistent with the zone configuration given. */ restore = zloop_dev_exists(zlo); for (i = 0; i < nr_zones; i++) { ret = zloop_init_zone(zlo, opts, i, restore); if (ret) goto out_close_files; }
lim.physical_block_size = zlo->block_size; lim.logical_block_size = zlo->block_size; if (zlo->zone_append) lim.max_hw_zone_append_sectors = lim.max_hw_sectors; lim.max_open_zones = zlo->max_open_zones;
zlo->tag_set.ops = &zloop_mq_ops; zlo->tag_set.nr_hw_queues = opts->nr_queues; zlo->tag_set.queue_depth = opts->queue_depth; zlo->tag_set.numa_node = NUMA_NO_NODE; zlo->tag_set.cmd_size = sizeof(struct zloop_cmd); zlo->tag_set.driver_data = zlo;
ret = blk_mq_alloc_tag_set(&zlo->tag_set); if (ret) { pr_err("blk_mq_alloc_tag_set failed (err=%d)\n", ret); goto out_close_files; }
zlo->disk = blk_mq_alloc_disk(&zlo->tag_set, &lim, zlo); if (IS_ERR(zlo->disk)) { pr_err("blk_mq_alloc_disk failed (err=%d)\n", ret); ret = PTR_ERR(zlo->disk); goto out_cleanup_tags; } zlo->disk->flags = GENHD_FL_NO_PART; zlo->disk->fops = &zloop_fops; zlo->disk->private_data = zlo; sprintf(zlo->disk->disk_name, "zloop%d", zlo->id); set_capacity(zlo->disk, (u64)lim.chunk_sectors * zlo->nr_zones);
ret = blk_revalidate_disk_zones(zlo->disk); if (ret) goto out_cleanup_disk;
ret = add_disk(zlo->disk); if (ret) { pr_err("add_disk failed (err=%d)\n", ret); goto out_cleanup_disk; }
mutex_lock(&zloop_ctl_mutex); WRITE_ONCE(zlo->state, Zlo_live); mutex_unlock(&zloop_ctl_mutex);
pr_info("zloop: device %d, %u zones of %llu MiB, %u B block size\n", zlo->id, zlo->nr_zones, ((sector_t)zlo->zone_size << SECTOR_SHIFT) >> 20, zlo->block_size); pr_info("zloop%d: using %s%s zone append\n", zlo->id, zlo->ordered_zone_append ? "ordered " : "", zlo->zone_append ? "native" : "emulated");
return 0;
out_cleanup_disk: put_disk(zlo->disk);out_cleanup_tags: blk_mq_free_tag_set(&zlo->tag_set);out_close_files: for (j = 0; j < i; j++) { struct zloop_zone *zone = &zlo->zones[j];
if (!IS_ERR_OR_NULL(zone->file)) fput(zone->file); } fput(zlo->data_dir);out_free_base_dir: kfree(zlo->base_dir);out_destroy_workqueue: destroy_workqueue(zlo->workqueue);out_free_idr: mutex_lock(&zloop_ctl_mutex); idr_remove(&zloop_index_idr, zlo->id); mutex_unlock(&zloop_ctl_mutex);out_free_dev: kvfree(zlo);out: module_put(THIS_MODULE); if (ret == -ENOENT) ret = -EINVAL; return ret;}
static void zloop_forget_cache(struct zloop_device *zlo){ unsigned int i; int ret;
pr_info("%pg: discarding volatile write cache\n", zlo->disk->part0);
for (i = 0; i < zlo->nr_zones; i++) { struct zloop_zone *zone = &zlo->zones[i]; struct file *file = zone->file; sector_t old_wp;
if (!zloop_zone_is_active(zone)) continue;
ret = vfs_getxattr(file_mnt_idmap(file), file_dentry(file), "user.zloop.wp", &old_wp, sizeof(old_wp)); if (ret == -ENODATA) { old_wp = 0; } else if (ret != sizeof(old_wp)) { pr_err("%pg: failed to retrieve write pointer (%d)\n", zlo->disk->part0, ret); continue; }
if (old_wp > zone->wp) continue; /* * This should not happen, if we recored a full zone, it can't * be active. */ if (WARN_ON_ONCE(old_wp == ULLONG_MAX)) continue;
vfs_truncate(&file->f_path, (old_wp - zone->start) << SECTOR_SHIFT); }}
static int zloop_ctl_remove(struct zloop_options *opts){ struct zloop_device *zlo; int ret;
if (!(opts->mask & ZLOOP_OPT_ID)) { pr_err("No ID specified for remove\n"); return -EINVAL; }
if (opts->mask & ~ZLOOP_OPT_ID) { pr_err("Invalid option specified for remove\n"); return -EINVAL; }
ret = mutex_lock_killable(&zloop_ctl_mutex); if (ret) return ret;
zlo = idr_find(&zloop_index_idr, opts->id); if (!zlo || zlo->state == Zlo_creating) { ret = -ENODEV; } else if (zlo->state == Zlo_deleting) { ret = -EINVAL; } else { idr_remove(&zloop_index_idr, zlo->id); WRITE_ONCE(zlo->state, Zlo_deleting); }
mutex_unlock(&zloop_ctl_mutex); if (ret) return ret;
del_gendisk(zlo->disk);
if (zlo->discard_write_cache) zloop_forget_cache(zlo);
put_disk(zlo->disk);
pr_info("Removed device %d\n", opts->id);
module_put(THIS_MODULE);
return 0;}
static int zloop_parse_options(struct zloop_options *opts, const char *buf){ substring_t args[MAX_OPT_ARGS]; char *options, *o, *p; unsigned int token; int ret = 0;
/* Set defaults. */ opts->mask = 0; opts->id = ZLOOP_DEF_ID; opts->capacity = ZLOOP_DEF_ZONE_SIZE * ZLOOP_DEF_NR_ZONES; opts->zone_size = ZLOOP_DEF_ZONE_SIZE; opts->nr_conv_zones = ZLOOP_DEF_NR_CONV_ZONES; opts->max_open_zones = ZLOOP_DEF_MAX_OPEN_ZONES; opts->nr_queues = ZLOOP_DEF_NR_QUEUES; opts->queue_depth = ZLOOP_DEF_QUEUE_DEPTH; opts->buffered_io = ZLOOP_DEF_BUFFERED_IO; opts->zone_append = ZLOOP_DEF_ZONE_APPEND; opts->ordered_zone_append = ZLOOP_DEF_ORDERED_ZONE_APPEND;
if (!buf) return 0;
/* Skip leading spaces before the options. */ while (isspace(*buf)) buf++;
options = o = kstrdup(buf, GFP_KERNEL); if (!options) return -ENOMEM;
/* Parse the options, doing only some light invalid value checks. */ while ((p = strsep(&o, ",\n")) != NULL) { if (!*p) continue;
token = match_token(p, zloop_opt_tokens, args); opts->mask |= token; switch (token) { case ZLOOP_OPT_ID: if (match_int(args, &opts->id)) { ret = -EINVAL; goto out; } break; case ZLOOP_OPT_CAPACITY: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (!token) { pr_err("Invalid capacity\n"); ret = -EINVAL; goto out; } opts->capacity = ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; break; case ZLOOP_OPT_ZONE_SIZE: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (!token || token > ZLOOP_MAX_ZONE_SIZE_MB || !is_power_of_2(token)) { pr_err("Invalid zone size %u\n", token); ret = -EINVAL; goto out; } opts->zone_size = ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; break; case ZLOOP_OPT_ZONE_CAPACITY: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (!token) { pr_err("Invalid zone capacity\n"); ret = -EINVAL; goto out; } opts->zone_capacity = ((sector_t)token * SZ_1M) >> SECTOR_SHIFT; break; case ZLOOP_OPT_NR_CONV_ZONES: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } opts->nr_conv_zones = token; break; case ZLOOP_OPT_MAX_OPEN_ZONES: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } opts->max_open_zones = token; break; case ZLOOP_OPT_BASE_DIR: p = match_strdup(args); if (!p) { ret = -ENOMEM; goto out; } kfree(opts->base_dir); opts->base_dir = p; break; case ZLOOP_OPT_NR_QUEUES: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (!token) { pr_err("Invalid number of queues\n"); ret = -EINVAL; goto out; } opts->nr_queues = min(token, num_online_cpus()); break; case ZLOOP_OPT_QUEUE_DEPTH: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (!token) { pr_err("Invalid queue depth\n"); ret = -EINVAL; goto out; } opts->queue_depth = token; break; case ZLOOP_OPT_BUFFERED_IO: opts->buffered_io = true; break; case ZLOOP_OPT_ZONE_APPEND: if (match_uint(args, &token)) { ret = -EINVAL; goto out; } if (token != 0 && token != 1) { pr_err("Invalid zone_append value\n"); ret = -EINVAL; goto out; } opts->zone_append = token; break; case ZLOOP_OPT_ORDERED_ZONE_APPEND: opts->ordered_zone_append = true; break; case ZLOOP_OPT_DISCARD_WRITE_CACHE: opts->discard_write_cache = true; break; case ZLOOP_OPT_ERR: default: pr_warn("unknown parameter or missing value '%s'\n", p); ret = -EINVAL; goto out; } }
ret = -EINVAL; if (opts->capacity <= opts->zone_size) { pr_err("Invalid capacity\n"); goto out; }
if (opts->zone_capacity > opts->zone_size) { pr_err("Invalid zone capacity\n"); goto out; }
ret = 0;out: kfree(options); return ret;}
enum { ZLOOP_CTL_ADD, ZLOOP_CTL_REMOVE,};
static struct zloop_ctl_op { int code; const char *name;} zloop_ctl_ops[] = { { ZLOOP_CTL_ADD, "add" }, { ZLOOP_CTL_REMOVE, "remove" }, { -1, NULL },};
static ssize_t zloop_ctl_write(struct file *file, const char __user *ubuf, size_t count, loff_t *pos){ struct zloop_options opts = { }; struct zloop_ctl_op *op; const char *buf, *opts_buf; int i, ret;
if (count > PAGE_SIZE) return -ENOMEM;
buf = memdup_user_nul(ubuf, count); if (IS_ERR(buf)) return PTR_ERR(buf);
for (i = 0; i < ARRAY_SIZE(zloop_ctl_ops); i++) { op = &zloop_ctl_ops[i]; if (!op->name) { pr_err("Invalid operation\n"); ret = -EINVAL; goto out; } if (!strncmp(buf, op->name, strlen(op->name))) break; }
if (count <= strlen(op->name)) opts_buf = NULL; else opts_buf = buf + strlen(op->name);
ret = zloop_parse_options(&opts, opts_buf); if (ret) { pr_err("Failed to parse options\n"); goto out; }
switch (op->code) { case ZLOOP_CTL_ADD: ret = zloop_ctl_add(&opts); break; case ZLOOP_CTL_REMOVE: ret = zloop_ctl_remove(&opts); break; default: pr_err("Invalid operation\n"); ret = -EINVAL; goto out; }
out: kfree(opts.base_dir); kfree(buf); return ret ? ret : count;}
static int zloop_ctl_show(struct seq_file *seq_file, void *private){ const struct match_token *tok; int i;
/* Add operation */ seq_printf(seq_file, "%s ", zloop_ctl_ops[0].name); for (i = 0; i < ARRAY_SIZE(zloop_opt_tokens); i++) { tok = &zloop_opt_tokens[i]; if (!tok->pattern) break; if (i) seq_putc(seq_file, ','); seq_puts(seq_file, tok->pattern); } seq_putc(seq_file, '\n');
/* Remove operation */ seq_puts(seq_file, zloop_ctl_ops[1].name); seq_puts(seq_file, " id=%d\n");
return 0;}
static int zloop_ctl_open(struct inode *inode, struct file *file){ file->private_data = NULL; return single_open(file, zloop_ctl_show, NULL);}
static int zloop_ctl_release(struct inode *inode, struct file *file){ return single_release(inode, file);}
static const struct file_operations zloop_ctl_fops = { .owner = THIS_MODULE, .open = zloop_ctl_open, .release = zloop_ctl_release, .write = zloop_ctl_write, .read = seq_read,};
static struct miscdevice zloop_misc = { .minor = MISC_DYNAMIC_MINOR, .name = "zloop-control", .fops = &zloop_ctl_fops,};
static int __init zloop_init(void){ int ret;
ret = misc_register(&zloop_misc); if (ret) { pr_err("Failed to register misc device: %d\n", ret); return ret; } pr_info("Module loaded\n");
return 0;}
static void __exit zloop_exit(void){ misc_deregister(&zloop_misc); idr_destroy(&zloop_index_idr);}
module_init(zloop_init);module_exit(zloop_exit);
MODULE_DESCRIPTION("Zoned loopback device");MODULE_LICENSE("GPL");