看了ceph的关于snap的一些代码（snap.cc、snaprealm.cc、snapserver.cc、snapmapper.cc），作了标注，在此记录。
snap.cc主要是通过encode（序列化），将数据结构表示为二进制流的方式，decode（反序列化），以便通过网络传输或保存在磁盘等存储介质上。
以下是snaprealm.h的一些结构体的声明和注释
1 struct SnapRealm {
2 // realm state
3
4 sr_t srnode;
5
6 // in-memory state
7 MDCache *mdcache;
8 CInode *inode;
9
10 bool open; // set to true once all past_parents are opened
11 SnapRealm *parent;
12 set<SnapRealm*> open_children; // active children that are currently open
13 map<inodeno_t,SnapRealm*> open_past_parents; // these are explicitly pinned.
14
15 // cache
16 snapid_t cached_seq; // max seq over self and all past+present parents.
17 snapid_t cached_last_created; // max last_created over all past+present parents
18 snapid_t cached_last_destroyed;
19 set<snapid_t> cached_snaps;
20 SnapContext cached_snap_context;
21
22 bufferlist cached_snap_trace;
23
24 elist<CInode*> inodes_with_caps; // for efficient realm splits
25 map<client_t, xlist<Capability*>* > client_caps; // to identify clients who need snap notificationssnaprealm.cc：
1 #include "SnapRealm.h"
2 #include "MDCache.h"
3 #include "MDS.h"
4
5 #include "messages/MClientSnap.h"
6
7
8 /*
9 * SnapRealm
10 */
11
12 #define dout_subsys ceph_subsys_mds
13 #undef dout_prefix
14 #define dout_prefix _prefix(_dout, mdcache->mds->get_nodeid(), inode, srnode.seq, this)
15 static ostream& _prefix(std::ostream *_dout, int whoami, CInode *inode,
16 uint64_t seq, SnapRealm *realm) {
17 return *_dout << " mds." << whoami
18 << ".cache.snaprealm(" << inode->ino()
19 << " seq " << seq << " " << realm << ") ";
20 }
21
22 ostream& operator<<(ostream& out, const SnapRealm& realm)
23 {
24 out << "snaprealm(" << realm.inode->ino()
25 << " seq " << realm.srnode.seq
26 << " lc " << realm.srnode.last_created
27 << " cr " << realm.srnode.created;
28 if (realm.srnode.created != realm.srnode.current_parent_since)
29 out << " cps " << realm.srnode.current_parent_since;
30 out << " snaps=" << realm.srnode.snaps;
31 if (realm.srnode.past_parents.size()) {
32 out << " past_parents=(";
33 for (map<snapid_t, snaplink_t>::const_iterator p = realm.srnode.past_parents.begin();
34 p != realm.srnode.past_parents.end();
35 ++p) {
36 if (p != realm.srnode.past_parents.begin()) out << ",";
37 out << p->second.first << "-" << p->first
38 << "=" << p->second.ino;
39 }
40 out << ")";
41 }
42 out << " " << &realm << ")";
43 return out;
44 }
45
46 //添加parent到open_past_parents的map容器中
47 void SnapRealm::add_open_past_parent(SnapRealm *parent)
48 {
49 open_past_parents[parent->inode->ino()] = parent;
50 parent->inode->get(CInode::PIN_PASTSNAPPARENT); //标记为CInode::PIN_PASTSNAPPARENT
51 }
52
53 struct C_SR_RetryOpenParents : public MDSInternalContextBase {
54 SnapRealm *sr;
55 snapid_t first, last, parent_last;
56 inodeno_t parent;
57 MDSInternalContextBase* fin;
58 C_SR_RetryOpenParents(SnapRealm *s, snapid_t f, snapid_t l, snapid_t pl,
59 inodeno_t p, MDSInternalContextBase *c) :
60 sr(s), first(f), last(l), parent_last(pl), parent(p), fin(c) {}
61 MDS *get_mds() { return sr->mdcache->mds; }
62 void finish(int r) {
63 if (r < 0)
64 sr->_remove_missing_parent(parent_last, parent, r);
65 if (sr->_open_parents(fin, first, last))
66 fin->complete(0);
67 }
68 };
69
70 //删除找不到snapid的parent
71 void SnapRealm::_remove_missing_parent(snapid_t snapid, inodeno_t parent, int err)
72 {
73 map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.find(snapid);
74 if (p != srnode.past_parents.end()) {
75 dout(10) << __func__ << " " << parent << " [" << p->second.first << ","
76 << p->first << "] errno " << err << dendl;
77 srnode.past_parents.erase(p);
78 } else {
79 dout(10) << __func__ << " " << parent << " not found" << dendl;
80 }
81 }
82
83 //判定parents是否open，返回true or false
84 bool SnapRealm::_open_parents(MDSInternalContextBase *finish, snapid_t first, snapid_t last)
85 {
86 dout(10) << "open_parents [" << first << "," << last << "]" << dendl;
87 if (open)
88 return true;
89
90 // make sure my current parents' parents are open...
91 if (parent) {
92 dout(10) << " current parent [" << srnode.current_parent_since << ",head] is " << *parent
93 << " on " << *parent->inode << dendl;
94 if (last >= srnode.current_parent_since &&
95 !parent->_open_parents(finish, MAX(first, srnode.current_parent_since), last))
96 return false;
97 }
98
99 // and my past parents too!
100 assert(srnode.past_parents.size() >= open_past_parents.size());
101 if (srnode.past_parents.size() > open_past_parents.size()) {
102 for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
103 p != srnode.past_parents.end();
104 ++p) {
105 dout(10) << " past_parent [" << p->second.first << "," << p->first << "] is "
106 << p->second.ino << dendl;
107 CInode *parent = mdcache->get_inode(p->second.ino);
108 if (!parent) {
109 C_SR_RetryOpenParents *fin = new C_SR_RetryOpenParents(this, first, last, p->first,
110 p->second.ino, finish);
111 mdcache->open_ino(p->second.ino, mdcache->mds->mdsmap->get_metadata_pool(), fin);
112 return false;
113 }
114 assert(parent->snaprealm); // hmm!
115 if (!parent->snaprealm->_open_parents(finish, p->second.first, p->first))
116 return false;
117 if (!open_past_parents.count(p->second.ino)) {
118 add_open_past_parent(parent->snaprealm);
119 }
120 }
121 }
122
123 open = true;
124 return true;
125 }
126
127 //判断从first到last的snap的past_parents是否open
128 bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last)
129 {
130 dout(10) << "have_past_parents_open [" << first << "," << last << "]" << dendl;
131 if (open)
132 return true;
133
134 for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
135 p != srnode.past_parents.end();
136 ++p) {
137 if (p->second.first > last)
138 break;
139 dout(10) << " past parent [" << p->second.first << "," << p->first << "] was "
140 << p->second.ino << dendl;
141 if (open_past_parents.count(p->second.ino) == 0) {
142 dout(10) << " past parent " << p->second.ino << " is not open" << dendl;
143 return false;
144 }
145 if (!open_past_parents[p->second.ino]->have_past_parents_open(MAX(first, p->second.first),
146 MIN(last, p->first)))
147 return false;
148 }
149
150 open = true;
151 return true;
152 }
153
154
155 void SnapRealm::close_parents()
156 {
157 for (map<inodeno_t,SnapRealm*>::iterator p = open_past_parents.begin();
158 p != open_past_parents.end();
159 ++p)
160 p->second->inode->put(CInode::PIN_PASTSNAPPARENT); //解除标记CInode::PIN_PASTSNAPPARENT
161 open_past_parents.clear(); //清空
162 }
163
164
165 /*
166 * get list of snaps for this realm. we must include parents' snaps
167 * for the intervals during which they were our parent.
168 */
169 //将realm中的snaps取出并存入第一个参数set容器中
170 void SnapRealm::build_snap_set(set<snapid_t> &s,
171 snapid_t& max_seq, snapid_t& max_last_created, snapid_t& max_last_destroyed,
172 snapid_t first, snapid_t last)
173 {
174 dout(10) << "build_snap_set [" << first << "," << last << "] on " << *this << dendl;
175
176 if (srnode.seq > max_seq)
177 max_seq = srnode.seq;
178 if (srnode.last_created > max_last_created)
179 max_last_created = srnode.last_created;
180 if (srnode.last_destroyed > max_last_destroyed)
181 max_last_destroyed = srnode.last_destroyed;
182
183 // include my snaps within interval [first,last]
184 for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
185 p != srnode.snaps.end() && p->first <= last;
186 ++p)
187 s.insert(p->first);
188
189 // include snaps for parents during intervals that intersect [first,last]
190 for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
191 p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
192 ++p) {
193 CInode *oldparent = mdcache->get_inode(p->second.ino);
194 assert(oldparent); // call open_parents first!
195 assert(oldparent->snaprealm);
196 oldparent->snaprealm->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
197 MAX(first, p->second.first),
198 MIN(last, p->first)); //递归，将past_parent的snapid加入set容器
199 }
200 if (srnode.current_parent_since <= last && parent)
201 parent->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
202 MAX(first, srnode.current_parent_since), last); //递归，将current_parent的snapid加入set容器
203 }
204
205
206 //检查cache中的数据结构并更新cached_snaps和cached_snap_trace
207 void SnapRealm::check_cache()
208 {
209 if (cached_seq >= srnode.seq)
210 return;
211
212 cached_snaps.clear();
213 cached_snap_context.clear();
214
215 cached_last_created = srnode.last_created;
216 cached_last_destroyed = srnode.last_destroyed;
217 cached_seq = srnode.seq;
218 build_snap_set(cached_snaps, cached_seq, cached_last_created, cached_last_destroyed,
219 0, CEPH_NOSNAP);
220
221 cached_snap_trace.clear();
222 build_snap_trace(cached_snap_trace);
223
224 dout(10) << "check_cache rebuilt " << cached_snaps
225 << " seq " << srnode.seq
226 << " cached_seq " << cached_seq
227 << " cached_last_created " << cached_last_created
228 << " cached_last_destroyed " << cached_last_destroyed
229 << ")" << dendl;
230 }
231
232
233 const set<snapid_t>& SnapRealm::get_snaps()
234 {
235 check_cache();
236 dout(10) << "get_snaps " << cached_snaps
237 << " (seq " << srnode.seq << " cached_seq " << cached_seq << ")"
238 << dendl;
239 return cached_snaps;
240 }
241
242 /*
243 * build vector in reverse sorted order
244 */
245 //循环将存储snapid的set中的内容放入返回值cached_snap_context中
246 const SnapContext& SnapRealm::get_snap_context()
247 {
248 check_cache();
249
250 if (!cached_snap_context.seq) {
251 cached_snap_context.seq = cached_seq;
252 cached_snap_context.snaps.resize(cached_snaps.size());
253 unsigned i = 0;
254 for (set<snapid_t>::reverse_iterator p = cached_snaps.rbegin();
255 p != cached_snaps.rend();
256 ++p)
257 cached_snap_context.snaps[i++] = *p;
258 }
259
260 return cached_snap_context;
261 }
262
263 //得到cache_snap，并将其info存入infomap的map容器中
264 void SnapRealm::get_snap_info(map<snapid_t,SnapInfo*>& infomap, snapid_t first, snapid_t last)
265 {
266 const set<snapid_t>& snaps = get_snaps();
267 dout(10) << "get_snap_info snaps " << snaps << dendl;
268
269 // include my snaps within interval [first,last]
270 for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
271 p != srnode.snaps.end() && p->first <= last;
272 ++p)
273 infomap[p->first] = &p->second; //令键为p->first的infomap的值等于&p->second
274
275 // include snaps for parents during intervals that intersect [first,last]
276 for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
277 p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
278 ++p) {
279 CInode *oldparent = mdcache->get_inode(p->second.ino);
280 assert(oldparent); // call open_parents first!
281 assert(oldparent->snaprealm);
282 oldparent->snaprealm->get_snap_info(infomap,
283 MAX(first, p->second.first),
284 MIN(last, p->first));
285 }
286 if (srnode.current_parent_since <= last && parent)
287 parent->get_snap_info(infomap, MAX(first, srnode.current_parent_since), last);
288 }
289
290 const string& SnapRealm::get_snapname(snapid_t snapid, inodeno_t atino)
291 {
292 if (srnode.snaps.count(snapid)) {
293 if (atino == inode->ino())
294 return srnode.snaps[snapid].name;
295 else
296 return srnode.snaps[snapid].get_long_name();
297 }
298
299 map<snapid_t,snaplink_t>::iterator p = srnode.past_parents.lower_bound(snapid);
300 if (p != srnode.past_parents.end() && p->second.first <= snapid) {
301 CInode *oldparent = mdcache->get_inode(p->second.ino);
302 assert(oldparent); // call open_parents first!
303 assert(oldparent->snaprealm);
304 return oldparent->snaprealm->get_snapname(snapid, atino);
305 }
306
307 assert(srnode.current_parent_since <= snapid);
308 assert(parent);
309 return parent->get_snapname(snapid, atino);
310 }
311
312 //判断第一个参数n，在srnode.snaps从first到last范围内找出对应的snapid
313 snapid_t SnapRealm::resolve_snapname(const string& n, inodeno_t atino, snapid_t first, snapid_t last)
314 {
315 // first try me
316 dout(10) << "resolve_snapname '" << n << "' in [" << first << "," << last << "]" << dendl;
317
318 //snapid_t num;
319 //if (n[0] == '~') num = atoll(n.c_str()+1);
320
321 bool actual = (atino == inode->ino());
322 string pname;
323 inodeno_t pino;
324 if (!actual) {
325 if (!n.length() ||
326 n[0] != '_') return 0;
327 int next_ = n.find('_', 1);
328 if (next_ < 0) return 0;
329 pname = n.substr(1, next_ - 1);
330 pino = atoll(n.c_str() + next_ + 1);
331 dout(10) << " " << n << " parses to name '" << pname << "' dirino " << pino << dendl;
332 }
333
334 //根据snapinfo得到snapid
335 for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
336 p != srnode.snaps.end() && p->first <= last;
337 ++p) {
338 dout(15) << " ? " << p->second << dendl;
339 //if (num && p->second.snapid == num)
340 //return p->first;
341 if (actual && p->second.name == n)
342 return p->first;
343 if (!actual && p->second.name == pname && p->second.ino == pino)
344 return p->first;
345 }
346
347 // include snaps for parents during intervals that intersect [first,last]
348 for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
349 p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
350 ++p) {
351 CInode *oldparent = mdcache->get_inode(p->second.ino);
352 assert(oldparent); // call open_parents first!
353 assert(oldparent->snaprealm);
354 snapid_t r = oldparent->snaprealm->resolve_snapname(n, atino,
355 MAX(first, p->second.first),
356 MIN(last, p->first));
357 if (r)
358 return r;
359 }
360 if (parent && srnode.current_parent_since <= last)
361 return parent->resolve_snapname(n, atino, MAX(first, srnode.current_parent_since), last);
362 return 0;
363 }
364
365
366 //调整parent令变量parent等于当前parent
367 void SnapRealm::adjust_parent()
368 {
369 SnapRealm *newparent = inode->get_parent_dn()->get_dir()->get_inode()->find_snaprealm();
370 if (newparent != parent) {
371 dout(10) << "adjust_parent " << parent << " -> " << newparent << dendl;
372 if (parent)
373 parent->open_children.erase(this);
374 parent = newparent;
375 if (parent)
376 parent->open_children.insert(this);
377
378 invalidate_cached_snaps();
379 }
380 }
381
382 //将child分离出当前realm
383 void SnapRealm::split_at(SnapRealm *child)
384 {
385 dout(10) << "split_at " << *child
386 << " on " << *child->inode << dendl;
387
388 if (inode->is_mdsdir() || !child->inode->is_dir()) {
389 // it's not a dir.
390 if (child->inode->containing_realm) {
391 // - no open children.
392 // - only need to move this child's inode's caps.
393 child->inode->move_to_realm(child); //脱离containing_realm，加入到child realm
394 } else {
395 // no caps, nothing to move/split.
396 dout(20) << " split no-op, no caps to move on file " << *child->inode << dendl;
397 assert(!child->inode->is_any_caps());
398 }
399 return;
400 }
401
402 // it's a dir.
403
404 // split open_children
405 dout(10) << " open_children are " << open_children << dendl;
406 for (set<SnapRealm*>::iterator p = open_children.begin();
407 p != open_children.end(); ) {
408 SnapRealm *realm = *p;
409 if (realm != child &&
410 child->inode->is_projected_ancestor_of(realm->inode)) {
411 dout(20) << " child gets child realm " << *realm << " on " << *realm->inode << dendl;
412 realm->parent = child;
413 child->open_children.insert(realm);
414 open_children.erase(p++);
415 } else {
416 dout(20) << " keeping child realm " << *realm << " on " << *realm->inode << dendl;
417 ++p;
418 }
419 }
420
421 // split inodes_with_caps
422 elist<CInode*>::iterator p = inodes_with_caps.begin(member_offset(CInode, item_caps));
423 while (!p.end()) {
424 CInode *in = *p;
425 ++p;
426
427 // does inode fall within the child realm?
428 bool under_child = false;
429
430 if (in == child->inode) {
431 under_child = true;
432 } else {
433 CInode *t = in;
434 while (t->get_parent_dn()) {
435 t = t->get_parent_dn()->get_dir()->get_inode();
436 if (t == child->inode) {
437 under_child = true;
438 break;
439 }
440 if (t == in)
441 break;
442 }
443 }
444 if (under_child) {
445 dout(20) << " child gets " << *in << dendl;
446 in->move_to_realm(child); //脱离containing_realm，加入到child realm
447 } else {
448 dout(20) << " keeping " << *in << dendl;
449 }
450 }
451
452 }
453
454 const bufferlist& SnapRealm::get_snap_trace()
455 {
456 check_cache();
457 return cached_snap_trace;
458 }
459
460 void SnapRealm::build_snap_trace(bufferlist& snapbl) //将snaps和所有parents存储在snapbl中
461 {
462 SnapRealmInfo info(inode->ino(), srnode.created, srnode.seq, srnode.current_parent_since);
463
464 if (parent) {
465 info.h.parent = parent->inode->ino();
466 if (!srnode.past_parents.empty()) {
467 snapid_t last = srnode.past_parents.rbegin()->first;
468 set<snapid_t> past;
469 snapid_t max_seq, max_last_created, max_last_destroyed;
470 build_snap_set(past, max_seq, max_last_created, max_last_destroyed, 0, last);
471 info.prior_parent_snaps.reserve(past.size());
472 for (set<snapid_t>::reverse_iterator p = past.rbegin(); p != past.rend(); ++p)
473 info.prior_parent_snaps.push_back(*p);
474 dout(10) << "build_snap_trace prior_parent_snaps from [1," << last << "] "
475 << info.prior_parent_snaps << dendl;
476 }
477 } else
478 info.h.parent = 0;
479
480 info.my_snaps.reserve(srnode.snaps.size());
481 for (map<snapid_t,SnapInfo>::reverse_iterator p = srnode.snaps.rbegin();
482 p != srnode.snaps.rend();
483 ++p)
484 info.my_snaps.push_back(p->first);
485 dout(10) << "build_snap_trace my_snaps " << info.my_snaps << dendl;
486
487 ::encode(info, snapbl);
488
489 if (parent)
490 parent->build_snap_trace(snapbl);
491 }
492
493
494 //删除past_parents中有但cached_snaps中没有的
495 void SnapRealm::prune_past_parents()
496 {
497 dout(10) << "prune_past_parents" << dendl;
498 check_cache();
499 assert(open);
500
501 map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
502 while (p != srnode.past_parents.end()) {
503 set<snapid_t>::iterator q = cached_snaps.lower_bound(p->second.first);
504 if (q == cached_snaps.end() || //若cached_snaps中没有p指向的值则erase
505 *q > p->first) {
506 dout(10) << "prune_past_parents pruning [" << p->second.first << "," << p->first
507 << "] " << p->second.ino << dendl;
508 srnode.past_parents.erase(p++);
509 } else {
510 dout(10) << "prune_past_parents keeping [" << p->second.first << "," << p->first
511 << "] " << p->second.ino << dendl;
512 ++p;
513 }
514 }
515 }snapserver.cc：
1 #include "SnapServer.h"
2 #include "MDS.h"
3 #include "osd/OSDMap.h"
4 #include "osdc/Objecter.h"
5 #include "mon/MonClient.h"
6
7 #include "include/types.h"
8 #include "messages/MMDSTableRequest.h"
9 #include "messages/MRemoveSnaps.h"
10
11 #include "msg/Messenger.h"
12
13 #include "common/config.h"
14 #include "include/assert.h"
15
16 #define dout_subsys ceph_subsys_mds
17 #undef dout_prefix
18 #define dout_prefix *_dout << "mds." << rank << ".snap "
19
20
21 void SnapServer::reset_state() //重置状态,将snaps和need_to_purge中的内容清空
22 {
23 last_snap = 1; /* snapid 1 reserved for initial root snaprealm */
24 snaps.clear();
25 need_to_purge.clear();
26
27 // find any removed snapshot in data pools
28 snapid_t first_free = 0;
29 const OSDMap *osdmap = mds->objecter->get_osdmap_read();
30 for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
31 p != mds->mdsmap->get_data_pools().end();
32 ++p) {
33 const pg_pool_t *pi = osdmap->get_pg_pool(*p);
34 if (!pi->removed_snaps.empty() &&
35 pi->removed_snaps.range_end() > first_free)
36 first_free = pi->removed_snaps.range_end();
37 }
38 mds->objecter->put_osdmap_read();
39 if (first_free > last_snap)
40 last_snap = first_free; //扩展last_snap范围
41 }
42
43
44 // SERVER
45 //根据操作执行:创建-将bl中的内容加入到pending_creat中，销毁-将bl中的内容加入到pending_destroy中
46 void SnapServer::_prepare(bufferlist &bl, uint64_t reqid, mds_rank_t bymds)
47 {
48 bufferlist::iterator p = bl.begin();
49 __u32 op;
50 ::decode(op, p);
51
52 switch (op) {
53 case TABLE_OP_CREATE:
54 {
55 version++;
56
57 SnapInfo info;
58 ::decode(info.ino, p);
59 if (!p.end()) {
60 ::decode(info.name, p);
61 ::decode(info.stamp, p);
62 info.snapid = ++last_snap;
63 pending_create[version] = info;
64 dout(10) << "prepare v" << version << " create " << info << dendl;
65 } else {
66 pending_noop.insert(version);
67 dout(10) << "prepare v" << version << " noop" << dendl;
68 }
69 bl.clear();
70 ::encode(last_snap, bl);
71 }
72 break;
73
74 case TABLE_OP_DESTROY:
75 {
76 inodeno_t ino;
77 snapid_t snapid;
78 ::decode(ino, p); // not used, currently.
79 ::decode(snapid, p);
80 version++;
81
82 // bump last_snap... we use it as a version value on the snaprealm.
83 ++last_snap;
84
85 pending_destroy[version] = pair<snapid_t,snapid_t>(snapid, last_snap);
86 dout(10) << "prepare v" << version << " destroy " << snapid << " seq " << last_snap << dendl;
87
88 bl.clear();
89 ::encode(last_snap, bl);
90 }
91 break;
92
93 default:
94 assert(0);
95 }
96 //dump();
97 }
98
99 //返回容器中此id出现的次数
100 bool SnapServer::_is_prepared(version_t tid)
101 {
102 return
103 pending_create.count(tid) ||
104 pending_destroy.count(tid);
105 }
106
107 //将操作pending_create或pending_destroy的内容存入对应的结构中
108 bool SnapServer::_commit(version_t tid, MMDSTableRequest *req)
109 {
110 if (pending_create.count(tid)) { //若此tid在pending_creat中有值，清空并将此结构中对应snapid中的内容存入snaps中
111 dout(7) << "commit " << tid << " create " << pending_create[tid] << dendl;
112 snaps[pending_create[tid].snapid] = pending_create[tid];
113 pending_create.erase(tid);
114 }
115
116 else if (pending_destroy.count(tid)) { //若此tid在pending_creat中有值，清空并将对应内容加入到need_to_purge
117 snapid_t sn = pending_destroy[tid].first; //removed_snap
118 snapid_t seq = pending_destroy[tid].second; //seq
119 dout(7) << "commit " << tid << " destroy " << sn << " seq " << seq << dendl;
120 snaps.erase(sn);
121
122 for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
123 p != mds->mdsmap->get_data_pools().end();
124 ++p) {
125 need_to_purge[*p].insert(sn);
126 need_to_purge[*p].insert(seq);
127 }
128
129 pending_destroy.erase(tid);
130 }
131 else if (pending_noop.count(tid)) { //清空pending_noop
132 dout(7) << "commit " << tid << " noop" << dendl;
133 pending_noop.erase(tid);
134 }
135 else
136 assert(0);
137
138 // bump version.
139 version++;
140 //dump();
141 return true;
142 }
143
144 //输出要执行的回滚命令并清空pending_create等结构中的内容，不执行具体操作
145 void SnapServer::_rollback(version_t tid)
146 {
147 if (pending_create.count(tid)) {
148 dout(7) << "rollback " << tid << " create " << pending_create[tid] << dendl;
149 pending_create.erase(tid);
150 }
151
152 else if (pending_destroy.count(tid)) {
153 dout(7) << "rollback " << tid << " destroy " << pending_destroy[tid] << dendl;
154 pending_destroy.erase(tid);
155 }
156
157 else if (pending_noop.count(tid)) {
158 dout(7) << "rollback " << tid << " noop" << dendl;
159 pending_noop.erase(tid);
160 }
161
162 else
163 assert(0);
164
165 // bump version.
166 version++;
167 //dump();
168 }
169
170 //将传入参数bl中的项在need_to_purge的对应项擦除
171 void SnapServer::_server_update(bufferlist& bl)
172 {
173 bufferlist::iterator p = bl.begin();
174 map<int, vector<snapid_t> > purge;
175 ::decode(purge, p);
176
177 dout(7) << "_server_update purged " << purge << dendl;
178 for (map<int, vector<snapid_t> >::iterator p = purge.begin();
179 p != purge.end();
180 ++p) {
181 for (vector<snapid_t>::iterator q = p->second.begin();
182 q != p->second.end();
183 ++q)
184 need_to_purge[p->first].erase(*q);
185 if (need_to_purge[p->first].empty())
186 need_to_purge.erase(p->first);
187 }
188
189 version++;
190 }
191
192 void SnapServer::handle_query(MMDSTableRequest *req)
193 {
194 req->put();
195 }
196
197
198 //遍历need_to_purge，执行remove的操作
199 void SnapServer::check_osd_map(bool force)
200 {
201 if (!force && version == last_checked_osdmap) {
202 dout(10) << "check_osd_map - version unchanged" << dendl;
203 return;
204 }
205 dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl;
206
207 map<int, vector<snapid_t> > all_purge;
208 map<int, vector<snapid_t> > all_purged;
209
210 const OSDMap *osdmap = mds->objecter->get_osdmap_read();
211 for (map<int, set<snapid_t> >::iterator p = need_to_purge.begin(); //遍历need_to_purge
212 p != need_to_purge.end();
213 ++p) {
214 int id = p->first;
215 const pg_pool_t *pi = osdmap->get_pg_pool(id);
216 for (set<snapid_t>::iterator q = p->second.begin();
217 q != p->second.end();
218 ++q) {
219 if (pi->is_removed_snap(*q)) { //已经remove，存入all_purged，还没有remove则存入all_purge
220 dout(10) << " osdmap marks " << *q << " as removed" << dendl;
221 all_purged[id].push_back(*q);
222 } else {
223 all_purge[id].push_back(*q);
224 }
225 }
226 }
227 mds->objecter->put_osdmap_read();
228
229 if (!all_purged.empty()) { //已经purge，调用do_server_update更新mdlog
230 // prepare to remove from need_to_purge list
231 bufferlist bl;
232 ::encode(all_purged, bl);
233 do_server_update(bl);
234 }
235
236 if (!all_purge.empty()) { //需要purge，创建消息存储all_purge并发送给mon
237 dout(10) << "requesting removal of " << all_purge << dendl;
238 MRemoveSnaps *m = new MRemoveSnaps(all_purge);
239 mds->monc->send_mon_message(m);
240 }
241
242 last_checked_osdmap = version;
243 }
244
245
246
247 void SnapServer::dump(Formatter *f) const
248 {
249 f->open_object_section("snapserver");
250
251 f->dump_int("last_snap", last_snap.val);
252
253 f->open_array_section("pending_noop");
254 for(set<version_t>::const_iterator i = pending_noop.begin(); i != pending_noop.end(); ++i) {
255 f->dump_unsigned("version", *i);
256 }
257 f->close_section();
258
259 f->open_array_section("snaps");
260 for (map<snapid_t, SnapInfo>::const_iterator i = snaps.begin(); i != snaps.end(); ++i) {
261 f->open_object_section("snap");
262 i->second.dump(f);
263 f->close_section();
264 }
265 f->close_section();
266
267 f->open_object_section("need_to_purge");
268 for (map<int, set<snapid_t> >::const_iterator i = need_to_purge.begin(); i != need_to_purge.end(); ++i) {
269 stringstream pool_id;
270 pool_id << i->first;
271 f->open_array_section(pool_id.str().c_str());
272 for (set<snapid_t>::const_iterator s = i->second.begin(); s != i->second.end(); ++s) {
273 f->dump_unsigned("snapid", s->val);
274 }
275 f->close_section();
276 }
277 f->close_section();
278
279 f->open_array_section("pending_create");
280 for(map<version_t, SnapInfo>::const_iterator i = pending_create.begin(); i != pending_create.end(); ++i) {
281 f->open_object_section("snap");
282 f->dump_unsigned("version", i->first);
283 f->open_object_section("snapinfo");
284 i->second.dump(f);
285 f->close_section();
286 f->close_section();
287 }
288 f->close_section();
289
290 f->open_array_section("pending_destroy");
291 for(map<version_t, pair<snapid_t, snapid_t> >::const_iterator i = pending_destroy.begin(); i != pending_destroy.end(); ++i) {
292 f->open_object_section("snap");
293 f->dump_unsigned("version", i->first);
294 f->dump_unsigned("removed_snap", i->second.first);
295 f->dump_unsigned("seq", i->second.second);
296 f->close_section();
297 }
298 f->close_section();
299
300 f->close_section();
301 }
302
303 void SnapServer::generate_test_instances(list<SnapServer*>& ls)
304 {
305 list<SnapInfo*> snapinfo_instances;
306 SnapInfo::generate_test_instances(snapinfo_instances);
307 SnapInfo populated_snapinfo = *(snapinfo_instances.back());
308 for (list<SnapInfo*>::iterator i = snapinfo_instances.begin(); i != snapinfo_instances.end(); ++i) {
309 delete *i;
310 }
311
312 SnapServer *blank = new SnapServer();
313 ls.push_back(blank);
314 SnapServer *populated = new SnapServer();
315 populated->last_snap = 123;
316 populated->snaps[456] = populated_snapinfo;
317 populated->need_to_purge[2].insert(012);
318 populated->pending_create[234] = populated_snapinfo;
319 populated->pending_destroy[345].first = 567;
320 populated->pending_destroy[345].second = 768;
321 populated->pending_noop.insert(890);
322
323 ls.push_back(populated);
324
325 }snapmapper.h中有一段注释如下，
/**
* SnapMapper
*
* Manages two mappings:
* 1) hobject_t -> {snapid} //为每个克隆对象存储快照集
* 2) snapid -> {hobject_t} //存储快照集的对象作为其快照
*
* We accomplish this using two sets of keys:
* 1) OBJECT_PREFIX + obj.str() -> encoding of object_snaps
* 2) MAPPING_PREFIX + snapid_t + obj.str() -> encoding of pair<snapid_t, obj>
*
* The on disk strings and encodings are implemented in to_raw, to_raw_key,
* from_raw, to_object_key.
*
* The object -> {snapid} mapping is primarily included so that the
* SnapMapper state can be verified against the external PG state during
* scrub etc.
*
* The 2) mapping is arranged such that all objects in a particular
* snap will sort together, and so that all objects in a pg for a
* particular snap will group under up to 8 prefixes. //最多8前缀需要进行检查以确定用于特定pg在一个特定的单元的所有对象
*/
snapmapper.cc:
1 #include "SnapMapper.h"
2
3 #define dout_subsys ceph_subsys_osd
4 #undef dout_prefix
5 #define dout_prefix *_dout << "snap_mapper."
6
7 using std::string;
8
9 const string SnapMapper::MAPPING_PREFIX = "MAP_";
10 const string SnapMapper::OBJECT_PREFIX = "OBJ_";
11
12 int OSDriver::get_keys(
13 const std::set<std::string> &keys,
14 std::map<std::string, bufferlist> *out)
15 {
16 return os->omap_get_values(cid, hoid, keys, out);
17 }
18
19 int OSDriver::get_next(
20 const std::string &key,
21 pair<std::string, bufferlist> *next)
22 {
23 ObjectMap::ObjectMapIterator iter =
24 os->get_omap_iterator(cid, hoid);
25 if (!iter) {
26 assert(0);
27 return -EINVAL;
28 }
29 iter->upper_bound(key);
30 if (iter->valid()) {
31 if (next)
32 *next = make_pair(iter->key(), iter->value());
33 return 0;
34 } else {
35 return -ENOENT;
36 }
37 }
38
39 struct Mapping {
40 snapid_t snap;
41 hobject_t hoid;
42 Mapping(const pair<snapid_t, hobject_t> &in)
43 : snap(in.first), hoid(in.second) {}
44 Mapping() : snap(0) {}
45 void encode(bufferlist &bl) const {
46 ENCODE_START(1, 1, bl);
47 ::encode(snap, bl);
48 ::encode(hoid, bl);
49 ENCODE_FINISH(bl);
50 }
51 void decode(bufferlist::iterator &bl) {
52 DECODE_START(1, bl);
53 ::decode(snap, bl);
54 ::decode(hoid, bl);
55 DECODE_FINISH(bl);
56 }
57 };
58 WRITE_CLASS_ENCODER(Mapping)
59
60 string SnapMapper::get_prefix(snapid_t snap)
61 {
62 char buf[100];
63 int len = snprintf(
64 buf, sizeof(buf),
65 "%.*X_", (int)(sizeof(snap)*2),
66 static_cast<unsigned>(snap));
67 return MAPPING_PREFIX + string(buf, len);
68 }
69
70 string SnapMapper::to_raw_key( //实现mapping有两种key，此为raw_key
71 const pair<snapid_t, hobject_t> &in)
72 {
73 return get_prefix(in.first) + shard_prefix + in.second.to_str();
74 }
75
76 pair<string, bufferlist> SnapMapper::to_raw( //返回键值和对应map
77 const pair<snapid_t, hobject_t> &in)
78 {
79 bufferlist bl;
80 ::encode(Mapping(in), bl);
81 return make_pair(
82 to_raw_key(in),
83 bl);
84 }
85
86 pair<snapid_t, hobject_t> SnapMapper::from_raw( //根据map返回snapid和hobject对应的snap和hoid 与to_raw相反
87 const pair<std::string, bufferlist> &image)
88 {
89 Mapping map;
90 bufferlist bl(image.second);
91 bufferlist::iterator bp(bl.begin());
92 ::decode(map, bp);
93 return make_pair(map.snap, map.hoid);
94 }
95
96 bool SnapMapper::is_mapping(const string &to_test)
97 {
98 return to_test.substr(0, MAPPING_PREFIX.size()) == MAPPING_PREFIX;
99 }
100
101 string SnapMapper::to_object_key(const hobject_t &hoid) //实现mapping有两种key，此为object_key
102 {
103 return OBJECT_PREFIX + shard_prefix + hoid.to_str();
104 }
105
106 void SnapMapper::object_snaps::encode(bufferlist &bl) const
107 {
108 ENCODE_START(1, 1, bl);
109 ::encode(oid, bl);
110 ::encode(snaps, bl);
111 ENCODE_FINISH(bl);
112 }
113
114 void SnapMapper::object_snaps::decode(bufferlist::iterator &bl)
115 {
116 DECODE_START(1, bl);
117 ::decode(oid, bl);
118 ::decode(snaps, bl);
119 DECODE_FINISH(bl);
120 }
121
122 int SnapMapper::get_snaps( //根据iod从backend中找到对应的snaps并输出
123 const hobject_t &oid,
124 object_snaps *out)
125 {
126 assert(check(oid));
127 set<string> keys;
128 map<string, bufferlist> got;
129 keys.insert(to_object_key(oid));
130 int r = backend.get_keys(keys, &got);
131 if (r < 0)
132 return r;
133 if (got.empty())
134 return -ENOENT;
135 if (out) {
136 bufferlist::iterator bp = got.begin()->second.begin();
137 ::decode(*out, bp);
138 dout(20) << __func__ << " " << oid << " " << out->snaps << dendl;
139 assert(!out->snaps.empty());
140 } else {
141 dout(20) << __func__ << " " << oid << " (out == NULL)" << dendl;
142 }
143 return 0;
144 }
145
146 void SnapMapper::clear_snaps( //remove to_object_key
147 const hobject_t &oid,
148 MapCacher::Transaction<std::string, bufferlist> *t)
149 {
150 assert(check(oid));
151 set<string> to_remove;
152 to_remove.insert(to_object_key(oid));
153 backend.remove_keys(to_remove, t);
154 }
155
156 void SnapMapper::set_snaps( //将object_key和snap设置为一个map，并调用set_keys加入到Transaction结构中
157 const hobject_t &oid,
158 const object_snaps &in,
159 MapCacher::Transaction<std::string, bufferlist> *t)
160 {
161 assert(check(oid));
162 map<string, bufferlist> to_set;
163 bufferlist bl;
164 ::encode(in, bl);
165 to_set[to_object_key(oid)] = bl;
166 backend.set_keys(to_set, t);
167 }
168
169 int SnapMapper::update_snaps( //将new_snaps更新加入backend中
170 const hobject_t &oid,
171 const set<snapid_t> &new_snaps,
172 const set<snapid_t> *old_snaps_check,
173 MapCacher::Transaction<std::string, bufferlist> *t)
174 {
175 dout(20) << __func__ << " " << oid << " " << new_snaps
176 << " was " << (old_snaps_check ? *old_snaps_check : set<snapid_t>())
177 << dendl;
178 assert(check(oid));
179 if (new_snaps.empty())
180 return remove_oid(oid, t);
181
182 object_snaps out;
183 int r = get_snaps(oid, &out);
184 if (r < 0)
185 return r;
186 if (old_snaps_check)
187 assert(out.snaps == *old_snaps_check);
188
189 object_snaps in(oid, new_snaps);
190 set_snaps(oid, in, t);
191
192 set<string> to_remove;
193 for (set<snapid_t>::iterator i = out.snaps.begin();
194 i != out.snaps.end();
195 ++i) {
196 if (!new_snaps.count(*i)) {
197 to_remove.insert(to_raw_key(make_pair(*i, oid)));
198 }
199 }
200 backend.remove_keys(to_remove, t);
201 return 0;
202 }
203
204 void SnapMapper::add_oid(
205 const hobject_t &oid,
206 const set<snapid_t>& snaps,
207 MapCacher::Transaction<std::string, bufferlist> *t)
208 {
209 dout(20) << __func__ << " " << oid << " " << snaps << dendl;
210 assert(check(oid));
211 {
212 object_snaps out;
213 int r = get_snaps(oid, &out);
214 assert(r == -ENOENT);
215 }
216
217 object_snaps _snaps(oid, snaps);
218 set_snaps(oid, _snaps, t);
219
220 map<string, bufferlist> to_add;
221 for (set<snapid_t>::iterator i = snaps.begin();
222 i != snaps.end();
223 ++i) {
224 to_add.insert(to_raw(make_pair(*i, oid)));
225 }
226 backend.set_keys(to_add, t); //将to_add和t打包加入backend
227 }
228
229 int SnapMapper::get_next_object_to_trim(
230 snapid_t snap,
231 hobject_t *hoid)
232 {
233 for (set<string>::iterator i = prefixes.begin();
234 i != prefixes.end();
235 ++i) {
236 string list_after(get_prefix(snap) + *i);
237
238 pair<string, bufferlist> next;
239 int r = backend.get_next(list_after, &next);
240 if (r < 0) {
241 break; // Done
242 }
243
244 if (next.first.substr(0, list_after.size()) !=
245 list_after) {
246 continue; // Done with this prefix
247 }
248
249 assert(is_mapping(next.first));
250
251 pair<snapid_t, hobject_t> next_decoded(from_raw(next));
252 assert(next_decoded.first == snap);
253 assert(check(next_decoded.second));
254
255 if (hoid)
256 *hoid = next_decoded.second;
257 return 0;
258 }
259 return -ENOENT;
260 }
261
262
263 int SnapMapper::remove_oid(
264 const hobject_t &oid,
265 MapCacher::Transaction<std::string, bufferlist> *t)
266 {
267 dout(20) << __func__ << " " << oid << dendl;
268 assert(check(oid));
269 return _remove_oid(oid, t);
270 }
271
272 int SnapMapper::_remove_oid(
273 const hobject_t &oid,
274 MapCacher::Transaction<std::string, bufferlist> *t)
275 {
276 object_snaps out;
277 int r = get_snaps(oid, &out);
278 if (r < 0)
279 return r;
280
281 clear_snaps(oid, t); //remove to_object_key
282
283 set<string> to_remove;
284 for (set<snapid_t>::iterator i = out.snaps.begin();
285 i != out.snaps.end();
286 ++i) {
287 to_remove.insert(to_raw_key(make_pair(*i, oid)));
288 }
289 backend.remove_keys(to_remove, t); //remove to_raw_key
290 return 0;
291 }
292
293 int SnapMapper::get_snaps( //根据oid找到对应snaps并存在set容器snaps中
294 const hobject_t &oid,
295 std::set<snapid_t> *snaps)
296 {
297 assert(check(oid));
298 object_snaps out;
299 int r = get_snaps(oid, &out);
300 if (r < 0)
301 return r;
302 if (snaps)
303 snaps->swap(out.snaps);
304 return 0;
305 }
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
