Coverage for drivers/linstorcowutil.py : 24%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr
4#
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
17from sm_typing import List, override
19from linstorjournaler import LinstorJournaler
20from linstorvolumemanager import LinstorVolumeManager
22from concurrent.futures import ThreadPoolExecutor
23import base64
24import errno
25import json
26import socket
27import threading
28import time
30from cowutil import CowImageInfo, CowUtil, getCowUtil
31import util
32import xs_errors
34from vditype import VdiType
36MANAGER_PLUGIN = 'linstor-manager'
39def call_remote_method(session, host_ref, method, args):
40 try:
41 response = session.xenapi.host.call_plugin(
42 host_ref, MANAGER_PLUGIN, method, args
43 )
44 except Exception as e:
45 util.SMlog('call-plugin on {} ({} with {}) exception: {}'.format(
46 host_ref, method, args, e
47 ))
48 raise util.SMException(str(e))
50 util.SMlog('call-plugin on {} ({} with {}) returned: {}'.format(
51 host_ref, method, args, response
52 ))
54 return response
57class LinstorCallException(util.SMException):
58 def __init__(self, cmd_err):
59 self.cmd_err = cmd_err
61 @override
62 def __str__(self) -> str:
63 return str(self.cmd_err)
66class ErofsLinstorCallException(LinstorCallException):
67 pass
70class NoPathLinstorCallException(LinstorCallException):
71 pass
73def log_successful_call(target_host, device_path, vdi_uuid, remote_method, response):
74 util.SMlog('Successful access on {} for device {} ({}): `{}` => {}'.format(
75 target_host, device_path, vdi_uuid, remote_method, str(response)
76 ), priority=util.LOG_DEBUG)
78def log_failed_call(target_host, next_target, device_path, vdi_uuid, remote_method, e):
79 util.SMlog('Failed to call method on {} for device {} ({}): {}. Trying accessing on {}... (cause: {})'.format(
80 target_host, device_path, vdi_uuid, remote_method, next_target, e
81 ), priority=util.LOG_DEBUG)
83def linstorhostcall(local_method, remote_method=None):
84 if not remote_method:
85 remote_method = local_method
87 def decorated(response_parser):
88 def wrapper(*args, **kwargs):
89 self = args[0]
90 vdi_uuid = args[1]
92 device_path = self._linstor.build_device_path(
93 self._linstor.get_volume_name(vdi_uuid)
94 )
96 if not self._session:
97 return self._call_local_method(local_method, device_path, *args[2:], **kwargs)
99 remote_args = {
100 'devicePath': device_path,
101 'groupName': self._linstor.group_name,
102 'vdiType': self._vdi_type
103 }
104 remote_args.update(**kwargs)
105 remote_args = {str(key): str(value) for key, value in remote_args.items()}
107 this_host_ref = util.get_this_host_ref(self._session)
108 def call_method(host_label, host_ref):
109 if host_ref == this_host_ref:
110 return self._call_local_method(local_method, device_path, *args[2:], **kwargs)
111 response = call_remote_method(self._session, host_ref, remote_method, remote_args)
112 log_successful_call(host_label, device_path, vdi_uuid, remote_method, response)
113 return response_parser(self, vdi_uuid, response)
115 # 1. Try on attached host.
116 try:
117 host_ref_attached = next(iter(util.get_hosts_attached_on(self._session, [vdi_uuid])), None)
118 if host_ref_attached:
119 return call_method('attached host', host_ref_attached)
120 except Exception as e:
121 log_failed_call('attached host', 'master', device_path, vdi_uuid, remote_method, e)
123 # 2. Try on master host.
124 try:
125 return call_method('master', util.get_master_ref(self._session))
126 except Exception as e:
127 log_failed_call('master', 'primary', device_path, vdi_uuid, remote_method, e)
129 # 3. Try on a primary.
130 hosts = self._get_hosts(remote_method, device_path)
132 nodes, primary_hostname = self._linstor.find_up_to_date_diskful_nodes(vdi_uuid)
133 if primary_hostname:
134 try:
135 return call_method('primary', self._find_host_ref_from_hostname(hosts, primary_hostname))
136 except Exception as remote_e:
137 self._raise_openers_exception(device_path, remote_e)
139 log_failed_call('primary', 'another node', device_path, vdi_uuid, remote_method, 'no primary')
141 # 4. Try on any host with local data.
142 try:
143 return call_method('another node', next(filter(None,
144 (self._find_host_ref_from_hostname(hosts, hostname) for hostname in nodes)
145 ), None))
146 except Exception as remote_e:
147 self._raise_openers_exception(device_path, remote_e)
149 return wrapper
150 return decorated
153def linstormodifier():
154 def decorated(func):
155 def wrapper(*args, **kwargs):
156 self = args[0]
158 ret = func(*args, **kwargs)
159 self._linstor.invalidate_resource_cache()
160 return ret
161 return wrapper
162 return decorated
165class LinstorCowUtil(object):
166 def __init__(self, session, linstor, vdi_type: str):
167 self._session = session
168 self._linstor = linstor
169 self._cowutil = getCowUtil(vdi_type)
170 self._vdi_type = vdi_type
172 @property
173 def cowutil(self) -> CowUtil:
174 return self._cowutil
176 def create_chain_paths(self, vdi_uuid, readonly=False):
177 # OPTIMIZE: Add a limit_to_first_allocated_block param to limit cowutil calls.
178 # Useful for the snapshot code algorithm.
180 leaf_vdi_path = self._linstor.get_device_path(vdi_uuid)
181 path = leaf_vdi_path
182 while True:
183 if not util.pathexists(path):
184 raise xs_errors.XenError(
185 'VDIUnavailable', opterr='Could not find: {}'.format(path)
186 )
188 # Diskless path can be created on the fly, ensure we can open it.
189 def check_volume_usable():
190 while True:
191 try:
192 with open(path, 'r' if readonly else 'r+'):
193 pass
194 except IOError as e:
195 if e.errno == errno.ENODATA:
196 time.sleep(2)
197 continue
198 if e.errno == errno.EROFS or e.errno == errno.EMEDIUMTYPE:
199 util.SMlog('Volume not attachable because used. Openers: {}'.format(
200 self._linstor.get_volume_openers(vdi_uuid)
201 ))
202 raise
203 break
204 util.retry(check_volume_usable, 15, 2)
206 vdi_uuid = self.get_info(vdi_uuid).parentUuid
207 if not vdi_uuid:
208 break
209 path = self._linstor.get_device_path(vdi_uuid)
210 readonly = True # Non-leaf is always readonly.
212 return leaf_vdi_path
214 # --------------------------------------------------------------------------
215 # Getters: read locally and try on another host in case of failure.
216 # --------------------------------------------------------------------------
218 def check(self, vdi_uuid, ignore_missing_footer=False, fast=False):
219 kwargs = {
220 'ignoreMissingFooter': ignore_missing_footer,
221 'fast': fast
222 }
223 return self._check(vdi_uuid, **kwargs)
225 @linstorhostcall('check')
226 def _check(self, vdi_uuid, response):
227 return CowUtil.CheckResult(response)
229 def get_info(self, vdi_uuid, include_parent=True):
230 kwargs = {
231 'includeParent': include_parent,
232 'resolveParent': False
233 }
234 return self._get_info(vdi_uuid, self._extract_uuid, **kwargs)
236 @linstorhostcall('getInfo')
237 def _get_info(self, vdi_uuid, response):
238 obj = json.loads(response)
240 image_info = CowImageInfo(vdi_uuid)
241 image_info.sizeVirt = obj['sizeVirt']
242 image_info.sizePhys = obj['sizePhys']
243 if 'parentPath' in obj:
244 image_info.parentPath = obj['parentPath']
245 image_info.parentUuid = obj['parentUuid']
246 image_info.hidden = obj['hidden']
247 image_info.path = obj['path']
249 return image_info
251 @linstorhostcall('hasParent')
252 def has_parent(self, vdi_uuid, response):
253 return util.strtobool(response)
255 def get_parent(self, vdi_uuid):
256 return self._get_parent(vdi_uuid, self._extract_uuid)
258 @linstorhostcall('getParent')
259 def _get_parent(self, vdi_uuid, response):
260 return response
262 @linstorhostcall('getSizeVirt')
263 def get_size_virt(self, vdi_uuid, response):
264 return int(response)
266 @linstorhostcall('getMaxResizeSize')
267 def get_max_resize_size(self, vdi_uuid, response):
268 return int(response)
270 @linstorhostcall('getSizePhys')
271 def get_size_phys(self, vdi_uuid, response):
272 return int(response)
274 @linstorhostcall('getAllocatedSize')
275 def get_allocated_size(self, vdi_uuid, response):
276 return int(response)
278 @linstorhostcall('getDepth')
279 def get_depth(self, vdi_uuid, response):
280 return int(response)
282 @linstorhostcall('getKeyHash')
283 def get_key_hash(self, vdi_uuid, response):
284 return response or None
286 @linstorhostcall('getBlockBitmap')
287 def get_block_bitmap(self, vdi_uuid, response):
288 return base64.b64decode(response)
290 @linstorhostcall('_get_drbd_size', 'getDrbdSize')
291 def get_drbd_size(self, vdi_uuid, response):
292 return int(response)
294 def _get_drbd_size(self, path):
295 (ret, stdout, stderr) = util.doexec(['blockdev', '--getsize64', path])
296 if ret == 0:
297 return int(stdout.strip())
298 raise util.SMException('Failed to get DRBD size: {}'.format(stderr))
300 # --------------------------------------------------------------------------
301 # Setters: only used locally.
302 # --------------------------------------------------------------------------
304 @linstormodifier()
305 def create(self, path, size, static, msize=0):
306 return self._call_local_method_or_fail(self._cowutil.create, path, size, static, msize)
308 @linstormodifier()
309 def set_size_phys(self, path, size, debug=True):
310 return self._call_local_method_or_fail(self._cowutil.setSizePhys, path, size, debug)
312 @linstormodifier()
313 def set_parent(self, path, parentPath, parentRaw=False):
314 return self._call_local_method_or_fail(self._cowutil.setParent, path, parentPath, parentRaw)
316 @linstormodifier()
317 def set_hidden(self, path, hidden=True):
318 return self._call_local_method_or_fail(self._cowutil.setHidden, path, hidden)
320 @linstormodifier()
321 def set_key(self, path, key_hash):
322 return self._call_local_method_or_fail(self._cowutil.setKey, path, key_hash)
324 @linstormodifier()
325 def kill_data(self, path):
326 return self._call_local_method_or_fail(self._cowutil.killData, path)
328 @linstormodifier()
329 def snapshot(self, path, parent, parentRaw, msize=0, checkEmpty=True):
330 return self._call_local_method_or_fail(self._cowutil.snapshot, path, parent, parentRaw, msize, checkEmpty)
332 def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size):
333 # Only inflate if the LINSTOR volume capacity is not enough.
334 new_size = LinstorVolumeManager.round_up_volume_size(new_size)
335 if new_size <= old_size:
336 return
338 util.SMlog(
339 'Inflate {} (size={}, previous={})'
340 .format(vdi_path, new_size, old_size)
341 )
343 journaler.create(
344 LinstorJournaler.INFLATE, vdi_uuid, old_size
345 )
346 self._linstor.resize_volume(vdi_uuid, new_size)
348 result_size = self.get_drbd_size(vdi_uuid)
349 if result_size < new_size:
350 util.SMlog(
351 'WARNING: Cannot inflate volume to {}B, result size: {}B'
352 .format(new_size, result_size)
353 )
355 self._zeroize(vdi_path, result_size - self._cowutil.getFooterSize())
356 self.set_size_phys(vdi_path, result_size, False)
357 journaler.remove(LinstorJournaler.INFLATE, vdi_uuid)
359 def deflate(self, vdi_path, new_size, old_size, zeroize=False):
360 if zeroize:
361 assert old_size > self._cowutil.getFooterSize()
362 self._zeroize(vdi_path, old_size - self._cowutil.getFooterSize())
364 new_size = LinstorVolumeManager.round_up_volume_size(new_size)
365 if new_size >= old_size:
366 return
368 util.SMlog(
369 'Deflate {} (new size={}, previous={})'
370 .format(vdi_path, new_size, old_size)
371 )
373 self.set_size_phys(vdi_path, new_size)
374 # TODO: Change the LINSTOR volume size using linstor.resize_volume.
376 # --------------------------------------------------------------------------
377 # Remote setters: write locally and try on another host in case of failure.
378 # --------------------------------------------------------------------------
380 @linstormodifier()
381 def set_size_virt(self, path, size, jFile):
382 kwargs = {
383 'size': size,
384 'jFile': jFile
385 }
386 return self._call_method(self._cowutil.setSizeVirt, 'setSizeVirt', path, use_parent=False, **kwargs)
388 @linstormodifier()
389 def set_size_virt_fast(self, path, size):
390 kwargs = {
391 'size': size
392 }
393 return self._call_method(self._cowutil.setSizeVirtFast, 'setSizeVirtFast', path, use_parent=False, **kwargs)
395 @linstormodifier()
396 def force_parent(self, path, parentPath, parentRaw=False):
397 kwargs = {
398 'parentPath': str(parentPath),
399 'parentRaw': parentRaw
400 }
401 return self._call_method(self._cowutil.setParent, 'setParent', path, use_parent=False, **kwargs)
403 @linstormodifier()
404 def force_coalesce(self, path):
405 return int(self._call_method(self._cowutil.coalesce, 'coalesce', path, use_parent=True))
407 @linstormodifier()
408 def force_repair(self, path):
409 return self._call_method(self._cowutil.repair, 'repair', path, use_parent=False)
411 @linstormodifier()
412 def force_deflate(self, path, newSize, oldSize, zeroize):
413 kwargs = {
414 'newSize': newSize,
415 'oldSize': oldSize,
416 'zeroize': zeroize
417 }
418 return self._call_method('_force_deflate', 'deflate', path, use_parent=False, **kwargs)
420 def _force_deflate(self, path, newSize, oldSize, zeroize):
421 self.deflate(path, newSize, oldSize, zeroize)
423 # --------------------------------------------------------------------------
424 # Helpers.
425 # --------------------------------------------------------------------------
427 def compute_volume_size(self, virtual_size: int) -> int:
428 if VdiType.isCowImage(self._vdi_type):
429 # All LINSTOR VDIs have the metadata area preallocated for
430 # the maximum possible virtual size (for fast online VDI.resize).
431 meta_overhead = self._cowutil.calcOverheadEmpty(
432 max(virtual_size, self._cowutil.getDefaultPreallocationSizeVirt())
433 )
434 bitmap_overhead = self._cowutil.calcOverheadBitmap(virtual_size)
435 virtual_size += meta_overhead + bitmap_overhead
436 else:
437 raise Exception('Invalid image type: {}'.format(self._vdi_type))
439 return LinstorVolumeManager.round_up_volume_size(virtual_size)
441 def _extract_uuid(self, device_path):
442 # TODO: Remove new line in the vhdutil module. Not here.
443 return self._linstor.get_volume_uuid_from_device_path(
444 device_path.rstrip('\n')
445 )
447 def _get_hosts(self, remote_method, device_path):
448 try:
449 return self._session.xenapi.host.get_all_records()
450 except Exception as e:
451 raise xs_errors.XenError(
452 'VDIUnavailable',
453 opterr='Unable to get host list to run cowutil command `{}` (path={}): {}'
454 .format(remote_method, device_path, e)
455 )
457 # --------------------------------------------------------------------------
459 @staticmethod
460 def _find_host_ref_from_hostname(hosts, hostname):
461 return next((ref for ref, rec in hosts.items() if rec['hostname'] == hostname), None)
463 def _raise_openers_exception(self, device_path, e):
464 if isinstance(e, util.CommandException):
465 e_str = 'cmd: `{}`, code: `{}`, reason: `{}`'.format(e.cmd, e.code, e.reason)
466 else:
467 e_str = str(e)
469 try:
470 volume_uuid = self._linstor.get_volume_uuid_from_device_path(
471 device_path
472 )
473 e_wrapper = Exception(
474 e_str + ' (openers: {})'.format(
475 self._linstor.get_volume_openers(volume_uuid)
476 )
477 )
478 except Exception as illformed_e:
479 e_wrapper = Exception(
480 e_str + ' (unable to get openers: {})'.format(illformed_e)
481 )
482 util.SMlog('raise opener exception: {}'.format(e_wrapper))
483 raise e_wrapper # pylint: disable = E0702
485 def _sanitize_local_method(self, local_method):
486 if isinstance(local_method, str):
487 return getattr(self if local_method.startswith('_') else self._cowutil, local_method)
488 return local_method
490 def _call_local_method(self, local_method, device_path, *args, **kwargs):
491 local_method = self._sanitize_local_method(local_method)
493 try:
494 def local_call():
495 try:
496 return local_method(device_path, *args, **kwargs)
497 except util.CommandException as e:
498 if e.code == errno.EROFS or e.code == errno.EMEDIUMTYPE:
499 raise ErofsLinstorCallException(e) # Break retry calls.
500 if e.code == errno.ENOENT:
501 raise NoPathLinstorCallException(e)
502 raise e
503 # Retry only locally if it's not an EROFS exception.
504 return util.retry(local_call, 5, 2, exceptions=[util.CommandException])
505 except util.CommandException as e:
506 util.SMlog('failed to execute locally CowUtil (sys {})'.format(e.code))
507 raise e
509 def _call_local_method_or_fail(self, local_method, device_path, *args, **kwargs):
510 try:
511 return self._call_local_method(local_method, device_path, *args, **kwargs)
512 except ErofsLinstorCallException as e:
513 # Volume is locked on a host, find openers.
514 self._raise_openers_exception(device_path, e.cmd_err)
516 def _call_method(self, local_method, remote_method, device_path, use_parent, *args, **kwargs):
517 # Note: `use_parent` exists to know if the COW image parent is used by the local/remote method.
518 # Normally in case of failure, if the parent is unused we try to execute the method on
519 # another host using the DRBD opener list. In the other case, if the parent is required,
520 # we must check where this last one is open instead of the child.
522 local_method = self._sanitize_local_method(local_method)
524 # A. Try to write locally...
525 try:
526 return self._call_local_method(local_method, device_path, *args, **kwargs)
527 except Exception:
528 pass
530 util.SMlog('unable to execute `{}` locally, retry using a writable host...'.format(remote_method))
532 # B. Execute the command on another host.
533 # B.1. Get host list.
534 hosts = self._get_hosts(remote_method, device_path)
536 # B.2. Prepare remote args.
537 remote_args = {
538 'devicePath': device_path,
539 'groupName': self._linstor.group_name,
540 'vdiType': self._vdi_type
541 }
542 remote_args.update(**kwargs)
543 remote_args = {str(key): str(value) for key, value in remote_args.items()}
545 volume_uuid = self._linstor.get_volume_uuid_from_device_path(
546 device_path
547 )
548 parent_volume_uuid = None
549 if use_parent:
550 parent_volume_uuid = self.get_parent(volume_uuid)
552 openers_uuid = parent_volume_uuid if use_parent else volume_uuid
554 # B.3. Call!
555 def remote_call():
556 try:
557 all_openers = self._linstor.get_volume_openers(openers_uuid)
558 except Exception as e:
559 raise xs_errors.XenError(
560 'VDIUnavailable',
561 opterr='Unable to get DRBD openers to run CowUtil command `{}` (path={}): {}'
562 .format(remote_method, device_path, e)
563 )
565 no_host_found = True
566 for hostname, openers in all_openers.items():
567 if not openers:
568 continue
570 host_ref = self._find_host_ref_from_hostname(hosts, hostname)
571 if not host_ref:
572 continue
574 no_host_found = False
575 try:
576 return call_remote_method(self._session, host_ref, remote_method, remote_args)
577 except Exception:
578 pass
580 if no_host_found:
581 try:
582 return local_method(device_path, *args, **kwargs)
583 except Exception as e:
584 self._raise_openers_exception(device_path, e)
586 raise xs_errors.XenError(
587 'VDIUnavailable',
588 opterr='No valid host found to run CowUtil command `{}` (path=`{}`, openers=`{}`)'
589 .format(remote_method, device_path, openers)
590 )
591 return util.retry(remote_call, 5, 2)
593 def _zeroize(self, path, size):
594 if not util.zeroOut(path, size, self._cowutil.getFooterSize()):
595 raise xs_errors.XenError(
596 'EIO',
597 opterr='Failed to zero out COW image footer {}'.format(path)
598 )
600class MultiLinstorCowUtil:
601 class ExecutorData(threading.local):
602 def __init__(self):
603 self.clear()
605 def clear(self):
606 self.session = None
607 self.linstor = None
608 self.vdi_type_to_cowutil = {}
610 class Load:
611 def __init__(self, session):
612 self.session = session
614 def cleanup(self):
615 if self.session:
616 self.session.xenapi.session.logout()
617 self.session = None
619 def __init__(self, uri, group_name) -> None:
620 self._uri = uri
621 self._group_name = group_name
622 self._loads: List[MultiLinstorCowUtil.Load] = []
623 self._executor_data = self.ExecutorData()
625 def __del__(self):
626 self._cleanup()
628 def run(self, func, user_data_list):
629 def wrapper(func, user_data):
630 if not self._executor_data.session:
631 self._init_executor_thread()
632 return func(user_data, self)
634 with ThreadPoolExecutor(thread_name_prefix="CowUtil") as executor:
635 return executor.map(lambda user_data: wrapper(func, user_data), user_data_list)
637 def get_local_cowutil(self, vdi_type):
638 instance = self._executor_data.vdi_type_to_cowutil.get(vdi_type)
639 if not instance:
640 instance = LinstorCowUtil(
641 self._executor_data.session,
642 self._executor_data.linstor,
643 vdi_type
644 )
645 self._executor_data.vdi_type_to_cowutil[vdi_type] = instance
646 return instance
648 def _init_executor_thread(self):
649 session = util.get_localAPI_session()
650 load = self.Load(session)
651 try:
652 linstor = LinstorVolumeManager(
653 self._uri,
654 self._group_name,
655 repair=False,
656 logger=util.SMlog
657 )
658 self._executor_data.linstor = linstor
659 self._executor_data.session = session
660 except:
661 self._executor_data.clear()
662 load.cleanup()
663 raise
665 self._loads.append(load)
667 def _cleanup(self):
668 for load in self._loads:
669 try:
670 load.cleanup()
671 except Exception as e:
672 util.SMlog(f"Failed to clean load executor: {e}")
673 self._loads.clear()