Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858// SPDX-License-Identifier: GPL-2.0-only/* * Copyright (C) 2001-2002 Sistina Software (UK) Limited. * * This file is released under the GPL. */
#include <linux/blkdev.h>#include <linux/device-mapper.h>#include <linux/delay.h>#include <linux/fs.h>#include <linux/init.h>#include <linux/kdev_t.h>#include <linux/list.h>#include <linux/list_bl.h>#include <linux/mempool.h>#include <linux/module.h>#include <linux/slab.h>#include <linux/vmalloc.h>#include <linux/log2.h>#include <linux/dm-kcopyd.h>
#include "dm.h"
#include "dm-exception-store.h"
#define DM_MSG_PREFIX "snapshots"
static const char dm_snapshot_merge_target_name[] = "snapshot-merge";
#define dm_target_is_snapshot_merge(ti) \ ((ti)->type->name == dm_snapshot_merge_target_name)
/* * The size of the mempool used to track chunks in use. */#define MIN_IOS 256
#define DM_TRACKED_CHUNK_HASH_SIZE 16#define DM_TRACKED_CHUNK_HASH(x) ((unsigned long)(x) & \ (DM_TRACKED_CHUNK_HASH_SIZE - 1))
struct dm_hlist_head { struct hlist_head head; spinlock_t lock;};
struct dm_exception_table { uint32_t hash_mask; unsigned int hash_shift; struct dm_hlist_head *table;};
struct dm_snapshot { struct rw_semaphore lock;
struct dm_dev *origin; struct dm_dev *cow;
struct dm_target *ti;
/* List of snapshots per Origin */ struct list_head list;
/* * You can't use a snapshot if this is 0 (e.g. if full). * A snapshot-merge target never clears this. */ int valid;
/* * The snapshot overflowed because of a write to the snapshot device. * We don't have to invalidate the snapshot in this case, but we need * to prevent further writes. */ int snapshot_overflowed;
/* Origin writes don't trigger exceptions until this is set */ int active;
atomic_t pending_exceptions_count;
spinlock_t pe_allocation_lock;
/* Protected by "pe_allocation_lock" */ sector_t exception_start_sequence;
/* Protected by kcopyd single-threaded callback */ sector_t exception_complete_sequence;
/* * A list of pending exceptions that completed out of order. * Protected by kcopyd single-threaded callback. */ struct rb_root out_of_order_tree;
mempool_t pending_pool;
struct dm_exception_table pending; struct dm_exception_table complete;
/* * pe_lock protects all pending_exception operations and access * as well as the snapshot_bios list. */ spinlock_t pe_lock;
/* Chunks with outstanding reads */ spinlock_t tracked_chunk_lock; struct hlist_head tracked_chunk_hash[DM_TRACKED_CHUNK_HASH_SIZE];
/* The on disk metadata handler */ struct dm_exception_store *store;
unsigned int in_progress; struct wait_queue_head in_progress_wait;
struct dm_kcopyd_client *kcopyd_client;
/* Wait for events based on state_bits */ unsigned long state_bits;
/* Range of chunks currently being merged. */ chunk_t first_merging_chunk; int num_merging_chunks;
/* * The merge operation failed if this flag is set. * Failure modes are handled as follows: * - I/O error reading the header * => don't load the target; abort. * - Header does not have "valid" flag set * => use the origin; forget about the snapshot. * - I/O error when reading exceptions * => don't load the target; abort. * (We can't use the intermediate origin state.) * - I/O error while merging * => stop merging; set merge_failed; process I/O normally. */ bool merge_failed:1;
bool discard_zeroes_cow:1; bool discard_passdown_origin:1;
/* * Incoming bios that overlap with chunks being merged must wait * for them to be committed. */ struct bio_list bios_queued_during_merge;};
/* * state_bits: * RUNNING_MERGE - Merge operation is in progress. * SHUTDOWN_MERGE - Set to signal that merge needs to be stopped; * cleared afterwards. */#define RUNNING_MERGE 0#define SHUTDOWN_MERGE 1
/* * Maximum number of chunks being copied on write. * * The value was decided experimentally as a trade-off between memory * consumption, stalling the kernel's workqueues and maintaining a high enough * throughput. */#define DEFAULT_COW_THRESHOLD 2048
static unsigned int cow_threshold = DEFAULT_COW_THRESHOLD;module_param_named(snapshot_cow_threshold, cow_threshold, uint, 0644);MODULE_PARM_DESC(snapshot_cow_threshold, "Maximum number of chunks being copied on write");
DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(snapshot_copy_throttle, "A percentage of time allocated for copy on write");
struct dm_dev *dm_snap_origin(struct dm_snapshot *s){ return s->origin;}EXPORT_SYMBOL(dm_snap_origin);
struct dm_dev *dm_snap_cow(struct dm_snapshot *s){ return s->cow;}EXPORT_SYMBOL(dm_snap_cow);
static sector_t chunk_to_sector(struct dm_exception_store *store, chunk_t chunk){ return chunk << store->chunk_shift;}
static int bdev_equal(struct block_device *lhs, struct block_device *rhs){ /* * There is only ever one instance of a particular block * device so we can compare pointers safely. */ return lhs == rhs;}
struct dm_snap_pending_exception { struct dm_exception e;
/* * Origin buffers waiting for this to complete are held * in a bio list */ struct bio_list origin_bios; struct bio_list snapshot_bios;
/* Pointer back to snapshot context */ struct dm_snapshot *snap;
/* * 1 indicates the exception has already been sent to * kcopyd. */ int started;
/* There was copying error. */ int copy_error;
/* A sequence number, it is used for in-order completion. */ sector_t exception_sequence;
struct rb_node out_of_order_node;
/* * For writing a complete chunk, bypassing the copy. */ struct bio *full_bio; bio_end_io_t *full_bio_end_io;};
/* * Hash table mapping origin volumes to lists of snapshots and * a lock to protect it */static struct kmem_cache *exception_cache;static struct kmem_cache *pending_cache;
struct dm_snap_tracked_chunk { struct hlist_node node; chunk_t chunk;};
static void init_tracked_chunk(struct bio *bio){ struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
INIT_HLIST_NODE(&c->node);}
static bool is_bio_tracked(struct bio *bio){ struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
return !hlist_unhashed(&c->node);}
static void track_chunk(struct dm_snapshot *s, struct bio *bio, chunk_t chunk){ struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk));
c->chunk = chunk;
spin_lock_irq(&s->tracked_chunk_lock); hlist_add_head(&c->node, &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)]); spin_unlock_irq(&s->tracked_chunk_lock);}
static void stop_tracking_chunk(struct dm_snapshot *s, struct bio *bio){ struct dm_snap_tracked_chunk *c = dm_per_bio_data(bio, sizeof(struct dm_snap_tracked_chunk)); unsigned long flags;
spin_lock_irqsave(&s->tracked_chunk_lock, flags); hlist_del(&c->node); spin_unlock_irqrestore(&s->tracked_chunk_lock, flags);}
static int __chunk_is_tracked(struct dm_snapshot *s, chunk_t chunk){ struct dm_snap_tracked_chunk *c; int found = 0;
spin_lock_irq(&s->tracked_chunk_lock);
hlist_for_each_entry(c, &s->tracked_chunk_hash[DM_TRACKED_CHUNK_HASH(chunk)], node) { if (c->chunk == chunk) { found = 1; break; } }
spin_unlock_irq(&s->tracked_chunk_lock);
return found;}
/* * This conflicting I/O is extremely improbable in the caller, * so fsleep(1000) is sufficient and there is no need for a wait queue. */static void __check_for_conflicting_io(struct dm_snapshot *s, chunk_t chunk){ while (__chunk_is_tracked(s, chunk)) fsleep(1000);}
/* * One of these per registered origin, held in the snapshot_origins hash */struct origin { /* The origin device */ struct block_device *bdev;
struct list_head hash_list;
/* List of snapshots for this origin */ struct list_head snapshots;};
/* * This structure is allocated for each origin target */struct dm_origin { struct dm_dev *dev; struct dm_target *ti; unsigned int split_boundary; struct list_head hash_list;};
/* * Size of the hash table for origin volumes. If we make this * the size of the minors list then it should be nearly perfect */#define ORIGIN_HASH_SIZE 256#define ORIGIN_MASK 0xFFstatic struct list_head *_origins;static struct list_head *_dm_origins;static struct rw_semaphore _origins_lock;
static DECLARE_WAIT_QUEUE_HEAD(_pending_exceptions_done);static DEFINE_SPINLOCK(_pending_exceptions_done_spinlock);static uint64_t _pending_exceptions_done_count;
static int init_origin_hash(void){ int i;
_origins = kmalloc_objs(struct list_head, ORIGIN_HASH_SIZE); if (!_origins) { DMERR("unable to allocate memory for _origins"); return -ENOMEM; } for (i = 0; i < ORIGIN_HASH_SIZE; i++) INIT_LIST_HEAD(_origins + i);
_dm_origins = kmalloc_objs(struct list_head, ORIGIN_HASH_SIZE); if (!_dm_origins) { DMERR("unable to allocate memory for _dm_origins"); kfree(_origins); return -ENOMEM; } for (i = 0; i < ORIGIN_HASH_SIZE; i++) INIT_LIST_HEAD(_dm_origins + i);
init_rwsem(&_origins_lock);
return 0;}
static void exit_origin_hash(void){ kfree(_origins); kfree(_dm_origins);}
static unsigned int origin_hash(struct block_device *bdev){ return bdev->bd_dev & ORIGIN_MASK;}
static struct origin *__lookup_origin(struct block_device *origin){ struct list_head *ol; struct origin *o;
ol = &_origins[origin_hash(origin)]; list_for_each_entry(o, ol, hash_list) if (bdev_equal(o->bdev, origin)) return o;
return NULL;}
static void __insert_origin(struct origin *o){ struct list_head *sl = &_origins[origin_hash(o->bdev)];
list_add_tail(&o->hash_list, sl);}
static struct dm_origin *__lookup_dm_origin(struct block_device *origin){ struct list_head *ol; struct dm_origin *o;
ol = &_dm_origins[origin_hash(origin)]; list_for_each_entry(o, ol, hash_list) if (bdev_equal(o->dev->bdev, origin)) return o;
return NULL;}
static void __insert_dm_origin(struct dm_origin *o){ struct list_head *sl = &_dm_origins[origin_hash(o->dev->bdev)];
list_add_tail(&o->hash_list, sl);}
static void __remove_dm_origin(struct dm_origin *o){ list_del(&o->hash_list);}
/* * _origins_lock must be held when calling this function. * Returns number of snapshots registered using the supplied cow device, plus: * snap_src - a snapshot suitable for use as a source of exception handover * snap_dest - a snapshot capable of receiving exception handover. * snap_merge - an existing snapshot-merge target linked to the same origin. * There can be at most one snapshot-merge target. The parameter is optional. * * Possible return values and states of snap_src and snap_dest. * 0: NULL, NULL - first new snapshot * 1: snap_src, NULL - normal snapshot * 2: snap_src, snap_dest - waiting for handover * 2: snap_src, NULL - handed over, waiting for old to be deleted * 1: NULL, snap_dest - source got destroyed without handover */static int __find_snapshots_sharing_cow(struct dm_snapshot *snap, struct dm_snapshot **snap_src, struct dm_snapshot **snap_dest, struct dm_snapshot **snap_merge){ struct dm_snapshot *s; struct origin *o; int count = 0; int active;
o = __lookup_origin(snap->origin->bdev); if (!o) goto out;
list_for_each_entry(s, &o->snapshots, list) { if (dm_target_is_snapshot_merge(s->ti) && snap_merge) *snap_merge = s; if (!bdev_equal(s->cow->bdev, snap->cow->bdev)) continue;
down_read(&s->lock); active = s->active; up_read(&s->lock);
if (active) { if (snap_src) *snap_src = s; } else if (snap_dest) *snap_dest = s;
count++; }
out: return count;}
/* * On success, returns 1 if this snapshot is a handover destination, * otherwise returns 0. */static int __validate_exception_handover(struct dm_snapshot *snap){ struct dm_snapshot *snap_src = NULL, *snap_dest = NULL; struct dm_snapshot *snap_merge = NULL;
/* Does snapshot need exceptions handed over to it? */ if ((__find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, &snap_merge) == 2) || snap_dest) { snap->ti->error = "Snapshot cow pairing for exception table handover failed"; return -EINVAL; }
/* * If no snap_src was found, snap cannot become a handover * destination. */ if (!snap_src) return 0;
/* * Non-snapshot-merge handover? */ if (!dm_target_is_snapshot_merge(snap->ti)) return 1;
/* * Do not allow more than one merging snapshot. */ if (snap_merge) { snap->ti->error = "A snapshot is already merging."; return -EINVAL; }
if (!snap_src->store->type->prepare_merge || !snap_src->store->type->commit_merge) { snap->ti->error = "Snapshot exception store does not support snapshot-merge."; return -EINVAL; }
return 1;}
static void __insert_snapshot(struct origin *o, struct dm_snapshot *s){ struct dm_snapshot *l;
/* Sort the list according to chunk size, largest-first smallest-last */ list_for_each_entry(l, &o->snapshots, list) if (l->store->chunk_size < s->store->chunk_size) break; list_add_tail(&s->list, &l->list);}
/* * Make a note of the snapshot and its origin so we can look it * up when the origin has a write on it. * * Also validate snapshot exception store handovers. * On success, returns 1 if this registration is a handover destination, * otherwise returns 0. */static int register_snapshot(struct dm_snapshot *snap){ struct origin *o, *new_o = NULL; struct block_device *bdev = snap->origin->bdev; int r = 0;
new_o = kmalloc_obj(*new_o); if (!new_o) return -ENOMEM;
down_write(&_origins_lock);
r = __validate_exception_handover(snap); if (r < 0) { kfree(new_o); goto out; }
o = __lookup_origin(bdev); if (o) kfree(new_o); else { /* New origin */ o = new_o;
/* Initialise the struct */ INIT_LIST_HEAD(&o->snapshots); o->bdev = bdev;
__insert_origin(o); }
__insert_snapshot(o, snap);
out: up_write(&_origins_lock);
return r;}
/* * Move snapshot to correct place in list according to chunk size. */static void reregister_snapshot(struct dm_snapshot *s){ struct block_device *bdev = s->origin->bdev;
down_write(&_origins_lock);
list_del(&s->list); __insert_snapshot(__lookup_origin(bdev), s);
up_write(&_origins_lock);}
static void unregister_snapshot(struct dm_snapshot *s){ struct origin *o;
down_write(&_origins_lock); o = __lookup_origin(s->origin->bdev);
list_del(&s->list); if (o && list_empty(&o->snapshots)) { list_del(&o->hash_list); kfree(o); }
up_write(&_origins_lock);}
/* * Implementation of the exception hash tables. * The lowest hash_shift bits of the chunk number are ignored, allowing * some consecutive chunks to be grouped together. */static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk);
/* Lock to protect access to the completed and pending exception hash tables. */struct dm_exception_table_lock { spinlock_t *complete_slot; spinlock_t *pending_slot;};
static void dm_exception_table_lock_init(struct dm_snapshot *s, chunk_t chunk, struct dm_exception_table_lock *lock){ struct dm_exception_table *complete = &s->complete; struct dm_exception_table *pending = &s->pending;
lock->complete_slot = &complete->table[exception_hash(complete, chunk)].lock; lock->pending_slot = &pending->table[exception_hash(pending, chunk)].lock;}
static void dm_exception_table_lock(struct dm_exception_table_lock *lock){ spin_lock_nested(lock->complete_slot, 1); spin_lock_nested(lock->pending_slot, 2);}
static void dm_exception_table_unlock(struct dm_exception_table_lock *lock){ spin_unlock(lock->pending_slot); spin_unlock(lock->complete_slot);}
static int dm_exception_table_init(struct dm_exception_table *et, uint32_t size, unsigned int hash_shift){ unsigned int i;
et->hash_shift = hash_shift; et->hash_mask = size - 1; et->table = kvmalloc_objs(struct dm_hlist_head, size); if (!et->table) return -ENOMEM;
for (i = 0; i < size; i++) { INIT_HLIST_HEAD(&et->table[i].head); spin_lock_init(&et->table[i].lock); }
return 0;}
static void dm_exception_table_exit(struct dm_exception_table *et, struct kmem_cache *mem){ struct dm_hlist_head *slot; struct dm_exception *ex; struct hlist_node *pos; int i, size;
size = et->hash_mask + 1; for (i = 0; i < size; i++) { slot = et->table + i;
hlist_for_each_entry_safe(ex, pos, &slot->head, hash_list) { hlist_del(&ex->hash_list); kmem_cache_free(mem, ex); cond_resched(); } }
kvfree(et->table);}
static uint32_t exception_hash(struct dm_exception_table *et, chunk_t chunk){ return (chunk >> et->hash_shift) & et->hash_mask;}
static void dm_remove_exception(struct dm_exception *e){ hlist_del(&e->hash_list);}
/* * Return the exception data for a sector, or NULL if not * remapped. */static struct dm_exception *dm_lookup_exception(struct dm_exception_table *et, chunk_t chunk){ struct hlist_head *slot; struct dm_exception *e;
slot = &et->table[exception_hash(et, chunk)].head; hlist_for_each_entry(e, slot, hash_list) if (chunk >= e->old_chunk && chunk <= e->old_chunk + dm_consecutive_chunk_count(e)) return e;
return NULL;}
static struct dm_exception *alloc_completed_exception(gfp_t gfp){ struct dm_exception *e;
e = kmem_cache_alloc(exception_cache, gfp); if (!e && gfp == GFP_NOIO) e = kmem_cache_alloc(exception_cache, GFP_ATOMIC);
return e;}
static void free_completed_exception(struct dm_exception *e){ kmem_cache_free(exception_cache, e);}
static struct dm_snap_pending_exception *alloc_pending_exception(struct dm_snapshot *s){ struct dm_snap_pending_exception *pe = mempool_alloc(&s->pending_pool, GFP_NOIO);
atomic_inc(&s->pending_exceptions_count); pe->snap = s;
return pe;}
static void free_pending_exception(struct dm_snap_pending_exception *pe){ struct dm_snapshot *s = pe->snap;
mempool_free(pe, &s->pending_pool); smp_mb__before_atomic(); atomic_dec(&s->pending_exceptions_count);}
static void dm_insert_exception(struct dm_exception_table *eh, struct dm_exception *new_e){ struct hlist_head *l; struct dm_exception *e = NULL;
l = &eh->table[exception_hash(eh, new_e->old_chunk)].head;
/* Add immediately if this table doesn't support consecutive chunks */ if (!eh->hash_shift) goto out;
/* List is ordered by old_chunk */ hlist_for_each_entry(e, l, hash_list) { /* Insert after an existing chunk? */ if (new_e->old_chunk == (e->old_chunk + dm_consecutive_chunk_count(e) + 1) && new_e->new_chunk == (dm_chunk_number(e->new_chunk) + dm_consecutive_chunk_count(e) + 1)) { dm_consecutive_chunk_count_inc(e); free_completed_exception(new_e); return; }
/* Insert before an existing chunk? */ if (new_e->old_chunk == (e->old_chunk - 1) && new_e->new_chunk == (dm_chunk_number(e->new_chunk) - 1)) { dm_consecutive_chunk_count_inc(e); e->old_chunk--; e->new_chunk--; free_completed_exception(new_e); return; }
if (new_e->old_chunk < e->old_chunk) break; }
out: if (!e) { /* * Either the table doesn't support consecutive chunks or slot * l is empty. */ hlist_add_head(&new_e->hash_list, l); } else if (new_e->old_chunk < e->old_chunk) { /* Add before an existing exception */ hlist_add_before(&new_e->hash_list, &e->hash_list); } else { /* Add to l's tail: e is the last exception in this slot */ hlist_add_behind(&new_e->hash_list, &e->hash_list); }}
/* * Callback used by the exception stores to load exceptions when * initialising. */static int dm_add_exception(void *context, chunk_t old, chunk_t new){ struct dm_snapshot *s = context; struct dm_exception *e;
e = alloc_completed_exception(GFP_KERNEL); if (!e) return -ENOMEM;
e->old_chunk = old;
/* Consecutive_count is implicitly initialised to zero */ e->new_chunk = new;
dm_insert_exception(&s->complete, e);
return 0;}
/* * Return a minimum chunk size of all snapshots that have the specified origin. * Return zero if the origin has no snapshots. */static uint32_t __minimum_chunk_size(struct origin *o){ struct dm_snapshot *snap; unsigned int chunk_size = rounddown_pow_of_two(UINT_MAX);
if (o) list_for_each_entry(snap, &o->snapshots, list) chunk_size = min_not_zero(chunk_size, snap->store->chunk_size);
return (uint32_t) chunk_size;}
/* * Hard coded magic. */static int calc_max_buckets(void){ /* use a fixed size of 2MB */ unsigned long mem = 2 * 1024 * 1024;
mem /= sizeof(struct dm_hlist_head);
return mem;}
/* * Allocate room for a suitable hash table. */static int init_hash_tables(struct dm_snapshot *s){ sector_t hash_size, cow_dev_size, max_buckets;
/* * Calculate based on the size of the original volume or * the COW volume... */ cow_dev_size = get_dev_size(s->cow->bdev); max_buckets = calc_max_buckets();
hash_size = cow_dev_size >> s->store->chunk_shift; hash_size = min(hash_size, max_buckets);
if (hash_size < 64) hash_size = 64; hash_size = rounddown_pow_of_two(hash_size); if (dm_exception_table_init(&s->complete, hash_size, DM_CHUNK_CONSECUTIVE_BITS)) return -ENOMEM;
/* * Allocate hash table for in-flight exceptions * Make this smaller than the real hash table */ hash_size >>= 3; if (hash_size < 64) hash_size = 64;
if (dm_exception_table_init(&s->pending, hash_size, 0)) { dm_exception_table_exit(&s->complete, exception_cache); return -ENOMEM; }
return 0;}
static void merge_shutdown(struct dm_snapshot *s){ clear_bit_unlock(RUNNING_MERGE, &s->state_bits); smp_mb__after_atomic(); wake_up_bit(&s->state_bits, RUNNING_MERGE);}
static struct bio *__release_queued_bios_after_merge(struct dm_snapshot *s){ s->first_merging_chunk = 0; s->num_merging_chunks = 0;
return bio_list_get(&s->bios_queued_during_merge);}
/* * Remove one chunk from the index of completed exceptions. */static int __remove_single_exception_chunk(struct dm_snapshot *s, chunk_t old_chunk){ struct dm_exception *e;
e = dm_lookup_exception(&s->complete, old_chunk); if (!e) { DMERR("Corruption detected: exception for block %llu is on disk but not in memory", (unsigned long long)old_chunk); return -EINVAL; }
/* * If this is the only chunk using this exception, remove exception. */ if (!dm_consecutive_chunk_count(e)) { dm_remove_exception(e); free_completed_exception(e); return 0; }
/* * The chunk may be either at the beginning or the end of a * group of consecutive chunks - never in the middle. We are * removing chunks in the opposite order to that in which they * were added, so this should always be true. * Decrement the consecutive chunk counter and adjust the * starting point if necessary. */ if (old_chunk == e->old_chunk) { e->old_chunk++; e->new_chunk++; } else if (old_chunk != e->old_chunk + dm_consecutive_chunk_count(e)) { DMERR("Attempt to merge block %llu from the middle of a chunk range [%llu - %llu]", (unsigned long long)old_chunk, (unsigned long long)e->old_chunk, (unsigned long long) e->old_chunk + dm_consecutive_chunk_count(e)); return -EINVAL; }
dm_consecutive_chunk_count_dec(e);
return 0;}
static void flush_bios(struct bio *bio);
static int remove_single_exception_chunk(struct dm_snapshot *s){ struct bio *b = NULL; int r; chunk_t old_chunk = s->first_merging_chunk + s->num_merging_chunks - 1;
down_write(&s->lock);
/* * Process chunks (and associated exceptions) in reverse order * so that dm_consecutive_chunk_count_dec() accounting works. */ do { r = __remove_single_exception_chunk(s, old_chunk); if (r) goto out; } while (old_chunk-- > s->first_merging_chunk);
b = __release_queued_bios_after_merge(s);
out: up_write(&s->lock); if (b) flush_bios(b);
return r;}
static int origin_write_extent(struct dm_snapshot *merging_snap, sector_t sector, unsigned int chunk_size);
static void merge_callback(int read_err, unsigned long write_err, void *context);
static uint64_t read_pending_exceptions_done_count(void){ uint64_t pending_exceptions_done;
spin_lock(&_pending_exceptions_done_spinlock); pending_exceptions_done = _pending_exceptions_done_count; spin_unlock(&_pending_exceptions_done_spinlock);
return pending_exceptions_done;}
static void increment_pending_exceptions_done_count(void){ spin_lock(&_pending_exceptions_done_spinlock); _pending_exceptions_done_count++; spin_unlock(&_pending_exceptions_done_spinlock);
wake_up_all(&_pending_exceptions_done);}
static void snapshot_merge_next_chunks(struct dm_snapshot *s){ int i, linear_chunks; chunk_t old_chunk, new_chunk; struct dm_io_region src, dest; sector_t io_size; uint64_t previous_count;
BUG_ON(!test_bit(RUNNING_MERGE, &s->state_bits)); if (unlikely(test_bit(SHUTDOWN_MERGE, &s->state_bits))) goto shut;
/* * valid flag never changes during merge, so no lock required. */ if (!s->valid) { DMERR("Snapshot is invalid: can't merge"); goto shut; }
linear_chunks = s->store->type->prepare_merge(s->store, &old_chunk, &new_chunk); if (linear_chunks <= 0) { if (linear_chunks < 0) { DMERR("Read error in exception store: shutting down merge"); down_write(&s->lock); s->merge_failed = true; up_write(&s->lock); } goto shut; }
/* Adjust old_chunk and new_chunk to reflect start of linear region */ old_chunk = old_chunk + 1 - linear_chunks; new_chunk = new_chunk + 1 - linear_chunks;
/* * Use one (potentially large) I/O to copy all 'linear_chunks' * from the exception store to the origin */ io_size = linear_chunks * s->store->chunk_size;
dest.bdev = s->origin->bdev; dest.sector = chunk_to_sector(s->store, old_chunk); dest.count = min(io_size, get_dev_size(dest.bdev) - dest.sector);
src.bdev = s->cow->bdev; src.sector = chunk_to_sector(s->store, new_chunk); src.count = dest.count;
/* * Reallocate any exceptions needed in other snapshots then * wait for the pending exceptions to complete. * Each time any pending exception (globally on the system) * completes we are woken and repeat the process to find out * if we can proceed. While this may not seem a particularly * efficient algorithm, it is not expected to have any * significant impact on performance. */ previous_count = read_pending_exceptions_done_count(); while (origin_write_extent(s, dest.sector, io_size)) { wait_event(_pending_exceptions_done, (read_pending_exceptions_done_count() != previous_count)); /* Retry after the wait, until all exceptions are done. */ previous_count = read_pending_exceptions_done_count(); }
down_write(&s->lock); s->first_merging_chunk = old_chunk; s->num_merging_chunks = linear_chunks; up_write(&s->lock);
/* Wait until writes to all 'linear_chunks' drain */ for (i = 0; i < linear_chunks; i++) __check_for_conflicting_io(s, old_chunk + i);
dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, merge_callback, s); return;
shut: merge_shutdown(s);}
static void error_bios(struct bio *bio);
static void merge_callback(int read_err, unsigned long write_err, void *context){ struct dm_snapshot *s = context; struct bio *b = NULL;
if (read_err || write_err) { if (read_err) DMERR("Read error: shutting down merge."); else DMERR("Write error: shutting down merge."); goto shut; }
if (blkdev_issue_flush(s->origin->bdev) < 0) { DMERR("Flush after merge failed: shutting down merge"); goto shut; }
if (s->store->type->commit_merge(s->store, s->num_merging_chunks) < 0) { DMERR("Write error in exception store: shutting down merge"); goto shut; }
if (remove_single_exception_chunk(s) < 0) goto shut;
snapshot_merge_next_chunks(s);
return;
shut: down_write(&s->lock); s->merge_failed = true; b = __release_queued_bios_after_merge(s); up_write(&s->lock); error_bios(b);
merge_shutdown(s);}
static void start_merge(struct dm_snapshot *s){ if (!test_and_set_bit(RUNNING_MERGE, &s->state_bits)) snapshot_merge_next_chunks(s);}
/* * Stop the merging process and wait until it finishes. */static void stop_merge(struct dm_snapshot *s){ set_bit(SHUTDOWN_MERGE, &s->state_bits); wait_on_bit(&s->state_bits, RUNNING_MERGE, TASK_UNINTERRUPTIBLE); clear_bit(SHUTDOWN_MERGE, &s->state_bits);}
static int parse_snapshot_features(struct dm_arg_set *as, struct dm_snapshot *s, struct dm_target *ti){ int r; unsigned int argc; const char *arg_name;
static const struct dm_arg _args[] = { {0, 2, "Invalid number of feature arguments"}, };
/* * No feature arguments supplied. */ if (!as->argc) return 0;
r = dm_read_arg_group(_args, as, &argc, &ti->error); if (r) return -EINVAL;
while (argc && !r) { arg_name = dm_shift_arg(as); argc--;
if (!strcasecmp(arg_name, "discard_zeroes_cow")) s->discard_zeroes_cow = true;
else if (!strcasecmp(arg_name, "discard_passdown_origin")) s->discard_passdown_origin = true;
else { ti->error = "Unrecognised feature requested"; r = -EINVAL; break; } }
if (!s->discard_zeroes_cow && s->discard_passdown_origin) { /* * TODO: really these are disjoint.. but ti->num_discard_bios * and dm_bio_get_target_bio_nr() require rigid constraints. */ ti->error = "discard_passdown_origin feature depends on discard_zeroes_cow"; r = -EINVAL; }
return r;}
/* * Construct a snapshot mapping: * <origin_dev> <COW-dev> <p|po|n> <chunk-size> [<# feature args> [<arg>]*] */static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv){ struct dm_snapshot *s; struct dm_arg_set as; int i; int r = -EINVAL; char *origin_path, *cow_path; unsigned int args_used, num_flush_bios = 1; blk_mode_t origin_mode = BLK_OPEN_READ;
if (argc < 4) { ti->error = "requires 4 or more arguments"; r = -EINVAL; goto bad; }
if (dm_target_is_snapshot_merge(ti)) { num_flush_bios = 2; origin_mode = BLK_OPEN_WRITE; }
s = kzalloc_obj(*s); if (!s) { ti->error = "Cannot allocate private snapshot structure"; r = -ENOMEM; goto bad; }
as.argc = argc; as.argv = argv; dm_consume_args(&as, 4); r = parse_snapshot_features(&as, s, ti); if (r) goto bad_features;
origin_path = argv[0]; argv++; argc--;
r = dm_get_device(ti, origin_path, origin_mode, &s->origin); if (r) { ti->error = "Cannot get origin device"; goto bad_origin; }
cow_path = argv[0]; argv++; argc--;
r = dm_get_device(ti, cow_path, dm_table_get_mode(ti->table), &s->cow); if (r) { ti->error = "Cannot get COW device"; goto bad_cow; } if (s->cow->bdev && s->cow->bdev == s->origin->bdev) { ti->error = "COW device cannot be the same as origin device"; r = -EINVAL; goto bad_store; }
r = dm_exception_store_create(ti, argc, argv, s, &args_used, &s->store); if (r) { ti->error = "Couldn't create exception store"; r = -EINVAL; goto bad_store; }
argv += args_used; argc -= args_used;
s->ti = ti; s->valid = 1; s->snapshot_overflowed = 0; s->active = 0; atomic_set(&s->pending_exceptions_count, 0); spin_lock_init(&s->pe_allocation_lock); s->exception_start_sequence = 0; s->exception_complete_sequence = 0; s->out_of_order_tree = RB_ROOT; init_rwsem(&s->lock); INIT_LIST_HEAD(&s->list); spin_lock_init(&s->pe_lock); s->state_bits = 0; s->merge_failed = false; s->first_merging_chunk = 0; s->num_merging_chunks = 0; bio_list_init(&s->bios_queued_during_merge);
/* Allocate hash table for COW data */ if (init_hash_tables(s)) { ti->error = "Unable to allocate hash table space"; r = -ENOMEM; goto bad_hash_tables; }
init_waitqueue_head(&s->in_progress_wait);
s->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle); if (IS_ERR(s->kcopyd_client)) { r = PTR_ERR(s->kcopyd_client); ti->error = "Could not create kcopyd client"; goto bad_kcopyd; }
r = mempool_init_slab_pool(&s->pending_pool, MIN_IOS, pending_cache); if (r) { ti->error = "Could not allocate mempool for pending exceptions"; goto bad_pending_pool; }
for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) INIT_HLIST_HEAD(&s->tracked_chunk_hash[i]);
spin_lock_init(&s->tracked_chunk_lock);
ti->private = s; ti->num_flush_bios = num_flush_bios; if (s->discard_zeroes_cow) ti->num_discard_bios = (s->discard_passdown_origin ? 2 : 1); ti->per_io_data_size = sizeof(struct dm_snap_tracked_chunk);
/* Add snapshot to the list of snapshots for this origin */ /* Exceptions aren't triggered till snapshot_resume() is called */ r = register_snapshot(s); if (r == -ENOMEM) { ti->error = "Snapshot origin struct allocation failed"; goto bad_load_and_register; } else if (r < 0) { /* invalid handover, register_snapshot has set ti->error */ goto bad_load_and_register; }
/* * Metadata must only be loaded into one table at once, so skip this * if metadata will be handed over during resume. * Chunk size will be set during the handover - set it to zero to * ensure it's ignored. */ if (r > 0) { s->store->chunk_size = 0; return 0; }
r = s->store->type->read_metadata(s->store, dm_add_exception, (void *)s); if (r < 0) { ti->error = "Failed to read snapshot metadata"; goto bad_read_metadata; } else if (r > 0) { s->valid = 0; DMWARN("Snapshot is marked invalid."); }
if (!s->store->chunk_size) { ti->error = "Chunk size not set"; r = -EINVAL; goto bad_read_metadata; }
r = dm_set_target_max_io_len(ti, s->store->chunk_size); if (r) goto bad_read_metadata;
return 0;
bad_read_metadata: unregister_snapshot(s);bad_load_and_register: mempool_exit(&s->pending_pool);bad_pending_pool: dm_kcopyd_client_destroy(s->kcopyd_client);bad_kcopyd: dm_exception_table_exit(&s->pending, pending_cache); dm_exception_table_exit(&s->complete, exception_cache);bad_hash_tables: dm_exception_store_destroy(s->store);bad_store: dm_put_device(ti, s->cow);bad_cow: dm_put_device(ti, s->origin);bad_origin:bad_features: kfree(s);bad: return r;}
static void __free_exceptions(struct dm_snapshot *s){ dm_kcopyd_client_destroy(s->kcopyd_client); s->kcopyd_client = NULL;
dm_exception_table_exit(&s->pending, pending_cache); dm_exception_table_exit(&s->complete, exception_cache);}
static void __handover_exceptions(struct dm_snapshot *snap_src, struct dm_snapshot *snap_dest){ union { struct dm_exception_table table_swap; struct dm_exception_store *store_swap; } u;
/* * Swap all snapshot context information between the two instances. */ u.table_swap = snap_dest->complete; snap_dest->complete = snap_src->complete; snap_src->complete = u.table_swap;
u.store_swap = snap_dest->store; snap_dest->store = snap_src->store; snap_dest->store->userspace_supports_overflow = u.store_swap->userspace_supports_overflow; snap_src->store = u.store_swap;
snap_dest->store->snap = snap_dest; snap_src->store->snap = snap_src;
snap_dest->ti->max_io_len = snap_dest->store->chunk_size; snap_dest->valid = snap_src->valid; snap_dest->snapshot_overflowed = snap_src->snapshot_overflowed;
/* * Set source invalid to ensure it receives no further I/O. */ snap_src->valid = 0;}
static void snapshot_dtr(struct dm_target *ti){#ifdef CONFIG_DM_DEBUG int i;#endif struct dm_snapshot *s = ti->private; struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
down_read(&_origins_lock); /* Check whether exception handover must be cancelled */ (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); if (snap_src && snap_dest && (s == snap_src)) { down_write(&snap_dest->lock); snap_dest->valid = 0; up_write(&snap_dest->lock); DMERR("Cancelling snapshot handover."); } up_read(&_origins_lock);
if (dm_target_is_snapshot_merge(ti)) stop_merge(s);
/* Prevent further origin writes from using this snapshot. */ /* After this returns there can be no new kcopyd jobs. */ unregister_snapshot(s);
while (atomic_read(&s->pending_exceptions_count)) fsleep(1000); /* * Ensure instructions in mempool_exit aren't reordered * before atomic_read. */ smp_mb();
#ifdef CONFIG_DM_DEBUG for (i = 0; i < DM_TRACKED_CHUNK_HASH_SIZE; i++) BUG_ON(!hlist_empty(&s->tracked_chunk_hash[i]));#endif
__free_exceptions(s);
mempool_exit(&s->pending_pool);
dm_exception_store_destroy(s->store);
dm_put_device(ti, s->cow);
dm_put_device(ti, s->origin);
WARN_ON(s->in_progress);
kfree(s);}
static void account_start_copy(struct dm_snapshot *s){ spin_lock(&s->in_progress_wait.lock); s->in_progress++; spin_unlock(&s->in_progress_wait.lock);}
static void account_end_copy(struct dm_snapshot *s){ spin_lock(&s->in_progress_wait.lock); BUG_ON(!s->in_progress); s->in_progress--; if (likely(s->in_progress <= cow_threshold) && unlikely(waitqueue_active(&s->in_progress_wait))) wake_up_locked(&s->in_progress_wait); spin_unlock(&s->in_progress_wait.lock);}
static bool wait_for_in_progress(struct dm_snapshot *s, bool unlock_origins){ if (unlikely(s->in_progress > cow_threshold)) { spin_lock(&s->in_progress_wait.lock); if (likely(s->in_progress > cow_threshold)) { /* * NOTE: this throttle doesn't account for whether * the caller is servicing an IO that will trigger a COW * so excess throttling may result for chunks not required * to be COW'd. But if cow_threshold was reached, extra * throttling is unlikely to negatively impact performance. */ DECLARE_WAITQUEUE(wait, current);
__add_wait_queue(&s->in_progress_wait, &wait); __set_current_state(TASK_UNINTERRUPTIBLE); spin_unlock(&s->in_progress_wait.lock); if (unlock_origins) up_read(&_origins_lock); io_schedule(); remove_wait_queue(&s->in_progress_wait, &wait); return false; } spin_unlock(&s->in_progress_wait.lock); } return true;}
/* * Flush a list of buffers. */static void flush_bios(struct bio *bio){ struct bio *n;
while (bio) { n = bio->bi_next; bio->bi_next = NULL; submit_bio_noacct(bio); bio = n; }}
static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit);
/* * Flush a list of buffers. */static void retry_origin_bios(struct dm_snapshot *s, struct bio *bio){ struct bio *n; int r;
while (bio) { n = bio->bi_next; bio->bi_next = NULL; r = do_origin(s->origin, bio, false); if (r == DM_MAPIO_REMAPPED) submit_bio_noacct(bio); bio = n; }}
/* * Error a list of buffers. */static void error_bios(struct bio *bio){ struct bio *n;
while (bio) { n = bio->bi_next; bio->bi_next = NULL; bio_io_error(bio); bio = n; }}
static void __invalidate_snapshot(struct dm_snapshot *s, int err){ if (!s->valid) return;
if (err == -EIO) DMERR("Invalidating snapshot: Error reading/writing."); else if (err == -ENOMEM) DMERR("Invalidating snapshot: Unable to allocate exception.");
if (s->store->type->drop_snapshot) s->store->type->drop_snapshot(s->store);
s->valid = 0;
dm_table_event(s->ti->table);}
static void invalidate_snapshot(struct dm_snapshot *s, int err){ down_write(&s->lock); __invalidate_snapshot(s, err); up_write(&s->lock);}
static void pending_complete(void *context, int success){ struct dm_snap_pending_exception *pe = context; struct dm_exception *e; struct dm_snapshot *s = pe->snap; struct bio *origin_bios = NULL; struct bio *snapshot_bios = NULL; struct bio *full_bio = NULL; struct dm_exception_table_lock lock; int error = 0;
dm_exception_table_lock_init(s, pe->e.old_chunk, &lock);
if (!success) { /* Read/write error - snapshot is unusable */ invalidate_snapshot(s, -EIO); error = 1;
dm_exception_table_lock(&lock); goto out; }
e = alloc_completed_exception(GFP_NOIO); if (!e) { invalidate_snapshot(s, -ENOMEM); error = 1;
dm_exception_table_lock(&lock); goto out; } *e = pe->e;
down_read(&s->lock); dm_exception_table_lock(&lock); if (!s->valid) { up_read(&s->lock); free_completed_exception(e); error = 1;
goto out; }
/* * Add a proper exception. After inserting the completed exception all * subsequent snapshot reads to this chunk will be redirected to the * COW device. This ensures that we do not starve. Moreover, as long * as the pending exception exists, neither origin writes nor snapshot * merging can overwrite the chunk in origin. */ dm_insert_exception(&s->complete, e); up_read(&s->lock);
/* Wait for conflicting reads to drain */ if (__chunk_is_tracked(s, pe->e.old_chunk)) { dm_exception_table_unlock(&lock); __check_for_conflicting_io(s, pe->e.old_chunk); dm_exception_table_lock(&lock); }
out: /* Remove the in-flight exception from the list */ dm_remove_exception(&pe->e);
dm_exception_table_unlock(&lock);
snapshot_bios = bio_list_get(&pe->snapshot_bios); origin_bios = bio_list_get(&pe->origin_bios); full_bio = pe->full_bio; if (full_bio) full_bio->bi_end_io = pe->full_bio_end_io; increment_pending_exceptions_done_count();
/* Submit any pending write bios */ if (error) { if (full_bio) bio_io_error(full_bio); error_bios(snapshot_bios); } else { if (full_bio) bio_endio(full_bio); flush_bios(snapshot_bios); }
retry_origin_bios(s, origin_bios);
free_pending_exception(pe);}
static void complete_exception(struct dm_snap_pending_exception *pe){ struct dm_snapshot *s = pe->snap;
/* Update the metadata if we are persistent */ s->store->type->commit_exception(s->store, &pe->e, !pe->copy_error, pending_complete, pe);}
/* * Called when the copy I/O has finished. kcopyd actually runs * this code so don't block. */static void copy_callback(int read_err, unsigned long write_err, void *context){ struct dm_snap_pending_exception *pe = context; struct dm_snapshot *s = pe->snap;
pe->copy_error = read_err || write_err;
if (pe->exception_sequence == s->exception_complete_sequence) { struct rb_node *next;
s->exception_complete_sequence++; complete_exception(pe);
next = rb_first(&s->out_of_order_tree); while (next) { pe = rb_entry(next, struct dm_snap_pending_exception, out_of_order_node); if (pe->exception_sequence != s->exception_complete_sequence) break; next = rb_next(next); s->exception_complete_sequence++; rb_erase(&pe->out_of_order_node, &s->out_of_order_tree); complete_exception(pe); cond_resched(); } } else { struct rb_node *parent = NULL; struct rb_node **p = &s->out_of_order_tree.rb_node; struct dm_snap_pending_exception *pe2;
while (*p) { pe2 = rb_entry(*p, struct dm_snap_pending_exception, out_of_order_node); parent = *p;
BUG_ON(pe->exception_sequence == pe2->exception_sequence); if (pe->exception_sequence < pe2->exception_sequence) p = &((*p)->rb_left); else p = &((*p)->rb_right); }
rb_link_node(&pe->out_of_order_node, parent, p); rb_insert_color(&pe->out_of_order_node, &s->out_of_order_tree); } account_end_copy(s);}
/* * Dispatches the copy operation to kcopyd. */static void start_copy(struct dm_snap_pending_exception *pe){ struct dm_snapshot *s = pe->snap; struct dm_io_region src, dest; struct block_device *bdev = s->origin->bdev; sector_t dev_size;
dev_size = get_dev_size(bdev);
src.bdev = bdev; src.sector = chunk_to_sector(s->store, pe->e.old_chunk); src.count = min((sector_t)s->store->chunk_size, dev_size - src.sector);
dest.bdev = s->cow->bdev; dest.sector = chunk_to_sector(s->store, pe->e.new_chunk); dest.count = src.count;
/* Hand over to kcopyd */ account_start_copy(s); dm_kcopyd_copy(s->kcopyd_client, &src, 1, &dest, 0, copy_callback, pe);}
static void full_bio_end_io(struct bio *bio){ void *callback_data = bio->bi_private;
dm_kcopyd_do_callback(callback_data, 0, bio->bi_status ? 1 : 0);}
static void start_full_bio(struct dm_snap_pending_exception *pe, struct bio *bio){ struct dm_snapshot *s = pe->snap; void *callback_data;
pe->full_bio = bio; pe->full_bio_end_io = bio->bi_end_io;
account_start_copy(s); callback_data = dm_kcopyd_prepare_callback(s->kcopyd_client, copy_callback, pe);
bio->bi_end_io = full_bio_end_io; bio->bi_private = callback_data;
submit_bio_noacct(bio);}
static struct dm_snap_pending_exception *__lookup_pending_exception(struct dm_snapshot *s, chunk_t chunk){ struct dm_exception *e = dm_lookup_exception(&s->pending, chunk);
if (!e) return NULL;
return container_of(e, struct dm_snap_pending_exception, e);}
/* * Inserts a pending exception into the pending table. * * NOTE: a write lock must be held on the chunk's pending exception table slot * before calling this. */static struct dm_snap_pending_exception *__insert_pending_exception(struct dm_snapshot *s, struct dm_snap_pending_exception *pe, chunk_t chunk){ pe->e.old_chunk = chunk; bio_list_init(&pe->origin_bios); bio_list_init(&pe->snapshot_bios); pe->started = 0; pe->full_bio = NULL;
spin_lock(&s->pe_allocation_lock); if (s->store->type->prepare_exception(s->store, &pe->e)) { spin_unlock(&s->pe_allocation_lock); free_pending_exception(pe); return NULL; }
pe->exception_sequence = s->exception_start_sequence++; spin_unlock(&s->pe_allocation_lock);
dm_insert_exception(&s->pending, &pe->e);
return pe;}
/* * Looks to see if this snapshot already has a pending exception * for this chunk, otherwise it allocates a new one and inserts * it into the pending table. * * NOTE: a write lock must be held on the chunk's pending exception table slot * before calling this. */static struct dm_snap_pending_exception *__find_pending_exception(struct dm_snapshot *s, struct dm_snap_pending_exception *pe, chunk_t chunk){ struct dm_snap_pending_exception *pe2;
pe2 = __lookup_pending_exception(s, chunk); if (pe2) { free_pending_exception(pe); return pe2; }
return __insert_pending_exception(s, pe, chunk);}
static void remap_exception(struct dm_snapshot *s, struct dm_exception *e, struct bio *bio, chunk_t chunk){ bio_set_dev(bio, s->cow->bdev); bio->bi_iter.bi_sector = chunk_to_sector(s->store, dm_chunk_number(e->new_chunk) + (chunk - e->old_chunk)) + (bio->bi_iter.bi_sector & s->store->chunk_mask);}
static void zero_callback(int read_err, unsigned long write_err, void *context){ struct bio *bio = context; struct dm_snapshot *s = bio->bi_private;
account_end_copy(s); bio->bi_status = write_err ? BLK_STS_IOERR : 0; bio_endio(bio);}
static void zero_exception(struct dm_snapshot *s, struct dm_exception *e, struct bio *bio, chunk_t chunk){ struct dm_io_region dest;
dest.bdev = s->cow->bdev; dest.sector = bio->bi_iter.bi_sector; dest.count = s->store->chunk_size;
account_start_copy(s); WARN_ON_ONCE(bio->bi_private); bio->bi_private = s; dm_kcopyd_zero(s->kcopyd_client, 1, &dest, 0, zero_callback, bio);}
static bool io_overlaps_chunk(struct dm_snapshot *s, struct bio *bio){ return bio->bi_iter.bi_size == (s->store->chunk_size << SECTOR_SHIFT);}
static int snapshot_map(struct dm_target *ti, struct bio *bio){ struct dm_exception *e; struct dm_snapshot *s = ti->private; int r = DM_MAPIO_REMAPPED; chunk_t chunk; struct dm_snap_pending_exception *pe = NULL; struct dm_exception_table_lock lock;
init_tracked_chunk(bio);
if (bio->bi_opf & REQ_PREFLUSH) { bio_set_dev(bio, s->cow->bdev); return DM_MAPIO_REMAPPED; }
chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector); dm_exception_table_lock_init(s, chunk, &lock);
/* Full snapshots are not usable */ /* To get here the table must be live so s->active is always set. */ if (!s->valid) return DM_MAPIO_KILL;
if (bio_data_dir(bio) == WRITE) { while (unlikely(!wait_for_in_progress(s, false))) ; /* wait_for_in_progress() has slept */ }
down_read(&s->lock); dm_exception_table_lock(&lock);
if (!s->valid || (unlikely(s->snapshot_overflowed) && bio_data_dir(bio) == WRITE)) { r = DM_MAPIO_KILL; goto out_unlock; }
if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { if (s->discard_passdown_origin && dm_bio_get_target_bio_nr(bio)) { /* * passdown discard to origin (without triggering * snapshot exceptions via do_origin; doing so would * defeat the goal of freeing space in origin that is * implied by the "discard_passdown_origin" feature) */ bio_set_dev(bio, s->origin->bdev); track_chunk(s, bio, chunk); goto out_unlock; } /* discard to snapshot (target_bio_nr == 0) zeroes exceptions */ }
/* If the block is already remapped - use that, else remap it */ e = dm_lookup_exception(&s->complete, chunk); if (e) { remap_exception(s, e, bio, chunk); if (unlikely(bio_op(bio) == REQ_OP_DISCARD) && io_overlaps_chunk(s, bio)) { dm_exception_table_unlock(&lock); up_read(&s->lock); zero_exception(s, e, bio, chunk); r = DM_MAPIO_SUBMITTED; /* discard is not issued */ goto out; } goto out_unlock; }
if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { /* * If no exception exists, complete discard immediately * otherwise it'll trigger copy-out. */ bio_endio(bio); r = DM_MAPIO_SUBMITTED; goto out_unlock; }
/* * Write to snapshot - higher level takes care of RW/RO * flags so we should only get this if we are * writable. */ if (bio_data_dir(bio) == WRITE) { pe = __lookup_pending_exception(s, chunk); if (!pe) { dm_exception_table_unlock(&lock); pe = alloc_pending_exception(s); dm_exception_table_lock(&lock);
e = dm_lookup_exception(&s->complete, chunk); if (e) { free_pending_exception(pe); remap_exception(s, e, bio, chunk); goto out_unlock; }
pe = __find_pending_exception(s, pe, chunk); if (!pe) { dm_exception_table_unlock(&lock); up_read(&s->lock);
down_write(&s->lock);
if (s->store->userspace_supports_overflow) { if (s->valid && !s->snapshot_overflowed) { s->snapshot_overflowed = 1; DMERR("Snapshot overflowed: Unable to allocate exception."); } } else __invalidate_snapshot(s, -ENOMEM); up_write(&s->lock);
r = DM_MAPIO_KILL; goto out; } }
remap_exception(s, &pe->e, bio, chunk);
r = DM_MAPIO_SUBMITTED;
if (!pe->started && io_overlaps_chunk(s, bio)) { pe->started = 1;
dm_exception_table_unlock(&lock); up_read(&s->lock);
start_full_bio(pe, bio); goto out; }
bio_list_add(&pe->snapshot_bios, bio);
if (!pe->started) { /* this is protected by the exception table lock */ pe->started = 1;
dm_exception_table_unlock(&lock); up_read(&s->lock);
start_copy(pe); goto out; } } else { bio_set_dev(bio, s->origin->bdev); track_chunk(s, bio, chunk); }
out_unlock: dm_exception_table_unlock(&lock); up_read(&s->lock);out: return r;}
/* * A snapshot-merge target behaves like a combination of a snapshot * target and a snapshot-origin target. It only generates new * exceptions in other snapshots and not in the one that is being * merged. * * For each chunk, if there is an existing exception, it is used to * redirect I/O to the cow device. Otherwise I/O is sent to the origin, * which in turn might generate exceptions in other snapshots. * If merging is currently taking place on the chunk in question, the * I/O is deferred by adding it to s->bios_queued_during_merge. */static int snapshot_merge_map(struct dm_target *ti, struct bio *bio){ struct dm_exception *e; struct dm_snapshot *s = ti->private; int r = DM_MAPIO_REMAPPED; chunk_t chunk;
init_tracked_chunk(bio);
if (bio->bi_opf & REQ_PREFLUSH) { if (!dm_bio_get_target_bio_nr(bio)) bio_set_dev(bio, s->origin->bdev); else bio_set_dev(bio, s->cow->bdev); return DM_MAPIO_REMAPPED; }
if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) { /* Once merging, discards no longer effect change */ bio_endio(bio); return DM_MAPIO_SUBMITTED; }
chunk = sector_to_chunk(s->store, bio->bi_iter.bi_sector);
down_write(&s->lock);
/* Full merging snapshots are redirected to the origin */ if (!s->valid) goto redirect_to_origin;
/* If the block is already remapped - use that */ e = dm_lookup_exception(&s->complete, chunk); if (e) { /* Queue writes overlapping with chunks being merged */ if (bio_data_dir(bio) == WRITE && chunk >= s->first_merging_chunk && chunk < (s->first_merging_chunk + s->num_merging_chunks)) { bio_set_dev(bio, s->origin->bdev); bio_list_add(&s->bios_queued_during_merge, bio); r = DM_MAPIO_SUBMITTED; goto out_unlock; }
remap_exception(s, e, bio, chunk);
if (bio_data_dir(bio) == WRITE) track_chunk(s, bio, chunk); goto out_unlock; }
redirect_to_origin: bio_set_dev(bio, s->origin->bdev);
if (bio_data_dir(bio) == WRITE) { up_write(&s->lock); return do_origin(s->origin, bio, false); }
out_unlock: up_write(&s->lock);
return r;}
static int snapshot_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *error){ struct dm_snapshot *s = ti->private;
if (is_bio_tracked(bio)) stop_tracking_chunk(s, bio);
return DM_ENDIO_DONE;}
static void snapshot_merge_presuspend(struct dm_target *ti){ struct dm_snapshot *s = ti->private;
stop_merge(s);}
static int snapshot_preresume(struct dm_target *ti){ int r = 0; struct dm_snapshot *s = ti->private; struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
down_read(&_origins_lock); (void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); if (snap_src && snap_dest) { down_read(&snap_src->lock); if (s == snap_src) { DMERR("Unable to resume snapshot source until handover completes."); r = -EINVAL; } else if (!dm_suspended(snap_src->ti)) { DMERR("Unable to perform snapshot handover until source is suspended."); r = -EINVAL; } up_read(&snap_src->lock); } up_read(&_origins_lock);
return r;}
static void snapshot_resume(struct dm_target *ti){ struct dm_snapshot *s = ti->private; struct dm_snapshot *snap_src = NULL, *snap_dest = NULL, *snap_merging = NULL; struct dm_origin *o; struct mapped_device *origin_md = NULL; bool must_restart_merging = false;
down_read(&_origins_lock);
o = __lookup_dm_origin(s->origin->bdev); if (o) origin_md = dm_table_get_md(o->ti->table); if (!origin_md) { (void) __find_snapshots_sharing_cow(s, NULL, NULL, &snap_merging); if (snap_merging) origin_md = dm_table_get_md(snap_merging->ti->table); } if (origin_md == dm_table_get_md(ti->table)) origin_md = NULL; if (origin_md) { if (dm_hold(origin_md)) origin_md = NULL; }
up_read(&_origins_lock);
if (origin_md) { dm_internal_suspend_fast(origin_md); if (snap_merging && test_bit(RUNNING_MERGE, &snap_merging->state_bits)) { must_restart_merging = true; stop_merge(snap_merging); } }
down_read(&_origins_lock);
(void) __find_snapshots_sharing_cow(s, &snap_src, &snap_dest, NULL); if (snap_src && snap_dest) { down_write(&snap_src->lock); down_write_nested(&snap_dest->lock, SINGLE_DEPTH_NESTING); __handover_exceptions(snap_src, snap_dest); up_write(&snap_dest->lock); up_write(&snap_src->lock); }
up_read(&_origins_lock);
if (origin_md) { if (must_restart_merging) start_merge(snap_merging); dm_internal_resume_fast(origin_md); dm_put(origin_md); }
/* Now we have correct chunk size, reregister */ reregister_snapshot(s);
down_write(&s->lock); s->active = 1; up_write(&s->lock);}
static uint32_t get_origin_minimum_chunksize(struct block_device *bdev){ uint32_t min_chunksize;
down_read(&_origins_lock); min_chunksize = __minimum_chunk_size(__lookup_origin(bdev)); up_read(&_origins_lock);
return min_chunksize;}
static void snapshot_merge_resume(struct dm_target *ti){ struct dm_snapshot *s = ti->private;
/* * Handover exceptions from existing snapshot. */ snapshot_resume(ti);
/* * snapshot-merge acts as an origin, so set ti->max_io_len */ ti->max_io_len = get_origin_minimum_chunksize(s->origin->bdev);
start_merge(s);}
static void snapshot_status(struct dm_target *ti, status_type_t type, unsigned int status_flags, char *result, unsigned int maxlen){ unsigned int sz = 0; struct dm_snapshot *snap = ti->private; unsigned int num_features;
switch (type) { case STATUSTYPE_INFO:
down_write(&snap->lock);
if (!snap->valid) DMEMIT("Invalid"); else if (snap->merge_failed) DMEMIT("Merge failed"); else if (snap->snapshot_overflowed) DMEMIT("Overflow"); else { if (snap->store->type->usage) { sector_t total_sectors, sectors_allocated, metadata_sectors; snap->store->type->usage(snap->store, &total_sectors, §ors_allocated, &metadata_sectors); DMEMIT("%llu/%llu %llu", (unsigned long long)sectors_allocated, (unsigned long long)total_sectors, (unsigned long long)metadata_sectors); } else DMEMIT("Unknown"); }
up_write(&snap->lock);
break;
case STATUSTYPE_TABLE: /* * kdevname returns a static pointer so we need * to make private copies if the output is to * make sense. */ DMEMIT("%s %s", snap->origin->name, snap->cow->name); sz += snap->store->type->status(snap->store, type, result + sz, maxlen - sz); num_features = snap->discard_zeroes_cow + snap->discard_passdown_origin; if (num_features) { DMEMIT(" %u", num_features); if (snap->discard_zeroes_cow) DMEMIT(" discard_zeroes_cow"); if (snap->discard_passdown_origin) DMEMIT(" discard_passdown_origin"); } break;
case STATUSTYPE_IMA: DMEMIT_TARGET_NAME_VERSION(ti->type); DMEMIT(",snap_origin_name=%s", snap->origin->name); DMEMIT(",snap_cow_name=%s", snap->cow->name); DMEMIT(",snap_valid=%c", snap->valid ? 'y' : 'n'); DMEMIT(",snap_merge_failed=%c", snap->merge_failed ? 'y' : 'n'); DMEMIT(",snapshot_overflowed=%c", snap->snapshot_overflowed ? 'y' : 'n'); DMEMIT(";"); break; }}
static int snapshot_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data){ struct dm_snapshot *snap = ti->private; int r;
r = fn(ti, snap->origin, 0, ti->len, data);
if (!r) r = fn(ti, snap->cow, 0, get_dev_size(snap->cow->bdev), data);
return r;}
static void snapshot_io_hints(struct dm_target *ti, struct queue_limits *limits){ struct dm_snapshot *snap = ti->private;
if (snap->discard_zeroes_cow) { struct dm_snapshot *snap_src = NULL, *snap_dest = NULL;
down_read(&_origins_lock);
(void) __find_snapshots_sharing_cow(snap, &snap_src, &snap_dest, NULL); if (snap_src && snap_dest) snap = snap_src;
/* All discards are split on chunk_size boundary */ limits->discard_granularity = snap->store->chunk_size; limits->max_hw_discard_sectors = snap->store->chunk_size;
up_read(&_origins_lock); }}
/* *--------------------------------------------------------------- * Origin methods *--------------------------------------------------------------- *//* * If no exceptions need creating, DM_MAPIO_REMAPPED is returned and any * supplied bio was ignored. The caller may submit it immediately. * (No remapping actually occurs as the origin is always a direct linear * map.) * * If further exceptions are required, DM_MAPIO_SUBMITTED is returned * and any supplied bio is added to a list to be submitted once all * the necessary exceptions exist. */static int __origin_write(struct list_head *snapshots, sector_t sector, struct bio *bio){ int r = DM_MAPIO_REMAPPED; struct dm_snapshot *snap; struct dm_exception *e; struct dm_snap_pending_exception *pe, *pe2; struct dm_snap_pending_exception *pe_to_start_now = NULL; struct dm_snap_pending_exception *pe_to_start_last = NULL; struct dm_exception_table_lock lock; chunk_t chunk;
/* Do all the snapshots on this origin */ list_for_each_entry(snap, snapshots, list) { /* * Don't make new exceptions in a merging snapshot * because it has effectively been deleted */ if (dm_target_is_snapshot_merge(snap->ti)) continue;
/* Nothing to do if writing beyond end of snapshot */ if (sector >= dm_table_get_size(snap->ti->table)) continue;
/* * Remember, different snapshots can have * different chunk sizes. */ chunk = sector_to_chunk(snap->store, sector); dm_exception_table_lock_init(snap, chunk, &lock);
down_read(&snap->lock); dm_exception_table_lock(&lock);
/* Only deal with valid and active snapshots */ if (!snap->valid || !snap->active) goto next_snapshot;
pe = __lookup_pending_exception(snap, chunk); if (!pe) { /* * Check exception table to see if block is already * remapped in this snapshot and trigger an exception * if not. */ e = dm_lookup_exception(&snap->complete, chunk); if (e) goto next_snapshot;
dm_exception_table_unlock(&lock); pe = alloc_pending_exception(snap); dm_exception_table_lock(&lock);
pe2 = __lookup_pending_exception(snap, chunk);
if (!pe2) { e = dm_lookup_exception(&snap->complete, chunk); if (e) { free_pending_exception(pe); goto next_snapshot; }
pe = __insert_pending_exception(snap, pe, chunk); if (!pe) { dm_exception_table_unlock(&lock); up_read(&snap->lock);
invalidate_snapshot(snap, -ENOMEM); continue; } } else { free_pending_exception(pe); pe = pe2; } }
r = DM_MAPIO_SUBMITTED;
/* * If an origin bio was supplied, queue it to wait for the * completion of this exception, and start this one last, * at the end of the function. */ if (bio) { bio_list_add(&pe->origin_bios, bio); bio = NULL;
if (!pe->started) { pe->started = 1; pe_to_start_last = pe; } }
if (!pe->started) { pe->started = 1; pe_to_start_now = pe; }
next_snapshot: dm_exception_table_unlock(&lock); up_read(&snap->lock);
if (pe_to_start_now) { start_copy(pe_to_start_now); pe_to_start_now = NULL; } }
/* * Submit the exception against which the bio is queued last, * to give the other exceptions a head start. */ if (pe_to_start_last) start_copy(pe_to_start_last);
return r;}
/* * Called on a write from the origin driver. */static int do_origin(struct dm_dev *origin, struct bio *bio, bool limit){ struct origin *o; int r = DM_MAPIO_REMAPPED;
again: down_read(&_origins_lock); o = __lookup_origin(origin->bdev); if (o) { if (limit) { struct dm_snapshot *s;
list_for_each_entry(s, &o->snapshots, list) if (unlikely(!wait_for_in_progress(s, true))) goto again; }
r = __origin_write(&o->snapshots, bio->bi_iter.bi_sector, bio); } up_read(&_origins_lock);
return r;}
/* * Trigger exceptions in all non-merging snapshots. * * The chunk size of the merging snapshot may be larger than the chunk * size of some other snapshot so we may need to reallocate multiple * chunks in other snapshots. * * We scan all the overlapping exceptions in the other snapshots. * Returns 1 if anything was reallocated and must be waited for, * otherwise returns 0. * * size must be a multiple of merging_snap's chunk_size. */static int origin_write_extent(struct dm_snapshot *merging_snap, sector_t sector, unsigned int size){ int must_wait = 0; sector_t n; struct origin *o;
/* * The origin's __minimum_chunk_size() got stored in max_io_len * by snapshot_merge_resume(). */ down_read(&_origins_lock); o = __lookup_origin(merging_snap->origin->bdev); for (n = 0; n < size; n += merging_snap->ti->max_io_len) if (__origin_write(&o->snapshots, sector + n, NULL) == DM_MAPIO_SUBMITTED) must_wait = 1; up_read(&_origins_lock);
return must_wait;}
/* * Origin: maps a linear range of a device, with hooks for snapshotting. */
/* * Construct an origin mapping: <dev_path> * The context for an origin is merely a 'struct dm_dev *' * pointing to the real device. */static int origin_ctr(struct dm_target *ti, unsigned int argc, char **argv){ int r; struct dm_origin *o;
if (argc != 1) { ti->error = "origin: incorrect number of arguments"; return -EINVAL; }
o = kmalloc_obj(struct dm_origin); if (!o) { ti->error = "Cannot allocate private origin structure"; r = -ENOMEM; goto bad_alloc; }
r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &o->dev); if (r) { ti->error = "Cannot get target device"; goto bad_open; }
o->ti = ti; ti->private = o; ti->num_flush_bios = 1;
return 0;
bad_open: kfree(o);bad_alloc: return r;}
static void origin_dtr(struct dm_target *ti){ struct dm_origin *o = ti->private;
dm_put_device(ti, o->dev); kfree(o);}
static int origin_map(struct dm_target *ti, struct bio *bio){ struct dm_origin *o = ti->private; unsigned int available_sectors;
bio_set_dev(bio, o->dev->bdev);
if (unlikely(bio->bi_opf & REQ_PREFLUSH)) return DM_MAPIO_REMAPPED;
if (bio_data_dir(bio) != WRITE) return DM_MAPIO_REMAPPED;
available_sectors = o->split_boundary - ((unsigned int)bio->bi_iter.bi_sector & (o->split_boundary - 1));
if (bio_sectors(bio) > available_sectors) dm_accept_partial_bio(bio, available_sectors);
/* Only tell snapshots if this is a write */ return do_origin(o->dev, bio, true);}
/* * Set the target "max_io_len" field to the minimum of all the snapshots' * chunk sizes. */static void origin_resume(struct dm_target *ti){ struct dm_origin *o = ti->private;
o->split_boundary = get_origin_minimum_chunksize(o->dev->bdev);
down_write(&_origins_lock); __insert_dm_origin(o); up_write(&_origins_lock);}
static void origin_postsuspend(struct dm_target *ti){ struct dm_origin *o = ti->private;
down_write(&_origins_lock); __remove_dm_origin(o); up_write(&_origins_lock);}
static void origin_status(struct dm_target *ti, status_type_t type, unsigned int status_flags, char *result, unsigned int maxlen){ struct dm_origin *o = ti->private;
switch (type) { case STATUSTYPE_INFO: result[0] = '\0'; break;
case STATUSTYPE_TABLE: snprintf(result, maxlen, "%s", o->dev->name); break; case STATUSTYPE_IMA: result[0] = '\0'; break; }}
static int origin_iterate_devices(struct dm_target *ti, iterate_devices_callout_fn fn, void *data){ struct dm_origin *o = ti->private;
return fn(ti, o->dev, 0, ti->len, data);}
static struct target_type origin_target = { .name = "snapshot-origin", .version = {1, 9, 0}, .module = THIS_MODULE, .ctr = origin_ctr, .dtr = origin_dtr, .map = origin_map, .resume = origin_resume, .postsuspend = origin_postsuspend, .status = origin_status, .iterate_devices = origin_iterate_devices,};
static struct target_type snapshot_target = { .name = "snapshot", .version = {1, 16, 0}, .module = THIS_MODULE, .ctr = snapshot_ctr, .dtr = snapshot_dtr, .map = snapshot_map, .end_io = snapshot_end_io, .preresume = snapshot_preresume, .resume = snapshot_resume, .status = snapshot_status, .iterate_devices = snapshot_iterate_devices, .io_hints = snapshot_io_hints,};
static struct target_type merge_target = { .name = dm_snapshot_merge_target_name, .version = {1, 5, 0}, .module = THIS_MODULE, .ctr = snapshot_ctr, .dtr = snapshot_dtr, .map = snapshot_merge_map, .end_io = snapshot_end_io, .presuspend = snapshot_merge_presuspend, .preresume = snapshot_preresume, .resume = snapshot_merge_resume, .status = snapshot_status, .iterate_devices = snapshot_iterate_devices, .io_hints = snapshot_io_hints,};
static int __init dm_snapshot_init(void){ int r;
r = dm_exception_store_init(); if (r) { DMERR("Failed to initialize exception stores"); return r; }
r = init_origin_hash(); if (r) { DMERR("init_origin_hash failed."); goto bad_origin_hash; }
exception_cache = KMEM_CACHE(dm_exception, 0); if (!exception_cache) { DMERR("Couldn't create exception cache."); r = -ENOMEM; goto bad_exception_cache; }
pending_cache = KMEM_CACHE(dm_snap_pending_exception, 0); if (!pending_cache) { DMERR("Couldn't create pending cache."); r = -ENOMEM; goto bad_pending_cache; }
r = dm_register_target(&snapshot_target); if (r < 0) goto bad_register_snapshot_target;
r = dm_register_target(&origin_target); if (r < 0) goto bad_register_origin_target;
r = dm_register_target(&merge_target); if (r < 0) goto bad_register_merge_target;
return 0;
bad_register_merge_target: dm_unregister_target(&origin_target);bad_register_origin_target: dm_unregister_target(&snapshot_target);bad_register_snapshot_target: kmem_cache_destroy(pending_cache);bad_pending_cache: kmem_cache_destroy(exception_cache);bad_exception_cache: exit_origin_hash();bad_origin_hash: dm_exception_store_exit();
return r;}
static void __exit dm_snapshot_exit(void){ dm_unregister_target(&snapshot_target); dm_unregister_target(&origin_target); dm_unregister_target(&merge_target);
exit_origin_hash(); kmem_cache_destroy(pending_cache); kmem_cache_destroy(exception_cache);
dm_exception_store_exit();}
/* Module hooks */module_init(dm_snapshot_init);module_exit(dm_snapshot_exit);
MODULE_DESCRIPTION(DM_NAME " snapshot target");MODULE_AUTHOR("Joe Thornber");MODULE_LICENSE("GPL");MODULE_ALIAS("dm-snapshot-origin");MODULE_ALIAS("dm-snapshot-merge");