Tools part of PV PCI hotplug, applied to xen-unstable.hg cs17042. Signed-off-by: Yosuke Iwamatsu diff -r e1dde6f8bc87 tools/python/xen/xend/XendDomainInfo.py --- a/tools/python/xen/xend/XendDomainInfo.py Wed Feb 13 18:30:12 2008 +0000 +++ b/tools/python/xen/xend/XendDomainInfo.py Thu Feb 21 19:30:30 2008 +0900 @@ -546,6 +546,95 @@ class XendDomainInfo: xen.xend.XendDomain.instance().managed_config_save(self) return self.getDeviceController(dev_type).sxpr(devid) + def pci_device_configure(self, dev_sxp, devid = 0): + """Configure an existing pci device. + + @param dev_config: device configuration + @type dev_config: SXP object (parsed config) + @param devid: device id + @type devid: int + @return: Returns True if successfully updated device + @rtype: boolean + """ + + dev_class = sxp.name(dev_sxp) + dev_config = {} + + if dev_class != 'pci': + return False + + if self.info.is_hvm(): + return False + + # If pci platform does not exist, create and exit. + found = False + for existing_type, existing_info in self.info.all_devices_sxpr(): + if existing_type == 'pci': + found = True + break + if not found: + for pci_state in sxp.children(dev_sxp, 'state'): + try: + if pci_state[1] != 'Initialising': # not attach + raise XendError("Cannot detach when pci platform " + "not exist") + except IndexError: + raise XendError("Error reading state") + self.device_create(dev_sxp) + return True + + # Convert device sxp to a dict. + # In reconfigure case, the config of PCI device looks like below: + # + # sxp: + # [device, [pci, [dev, [domain, 0], [bus, 0], [slot, 0], [func, 0]], + # [state, 'Initialising']]] + # + # dict: + # {devs: [{slot: 0, bus: 0, domain: 0, func: 0}], + # states: ['Initialising']} + # + # 'state' parameter takes either 'Initialising' or 'Closing', + # which indicates attach or detach. + pci_devs = [] + for pci_dev in sxp.children(dev_sxp, 'dev'): + pci_dev_info = {} + for opt_val in pci_dev[1:]: + try: + opt, val = opt_val + pci_dev_info[opt] = val + except TypeError: + pass + pci_devs.append(pci_dev_info) + dev_config['devs'] = pci_devs + pci_states = [] + for pci_state in sxp.children(dev_sxp, 'state'): + try: + pci_states.append(pci_state[1]) + except IndexError: + raise XendError("Error reading state") + dev_config['states'] = pci_states + + # use DevController.reconfigureDevice to change device config + dev_control = self.getDeviceController(dev_class) + dev_uuid = dev_control.reconfigureDevice(devid, dev_config) + + dev_control.waitForDevice_reconfigure(devid) + num_devs = dev_control.cleanupDevice(devid) + + # update XendConfig with new device info + if dev_uuid: + new_dev_sxp = dev_control.configuration(devid) + self.info.device_update(dev_uuid, new_dev_sxp) + + # If there's no device connected, destroy pci here. + # To remove config of pci, self.destroyDevice() should be modified. + if num_devs == 0: + self.destroyDevice('pci', devid) + # self.destroyDevice('pci', devid, False, True) + + return True + def device_configure(self, dev_sxp, devid = None): """Configure an existing device. @@ -560,6 +649,10 @@ class XendDomainInfo: # convert device sxp to a dict dev_class = sxp.name(dev_sxp) dev_config = {} + + if dev_class == 'pci': + return self.pci_device_configure(dev_sxp) + for opt_val in dev_sxp[1:]: try: dev_config[opt_val[0]] = opt_val[1] diff -r e1dde6f8bc87 tools/python/xen/xend/server/DevController.py --- a/tools/python/xen/xend/server/DevController.py Wed Feb 13 18:30:12 2008 +0000 +++ b/tools/python/xen/xend/server/DevController.py Thu Feb 21 19:30:30 2008 +0900 @@ -51,6 +51,8 @@ xenbusState = { 'Connected' : 4, 'Closing' : 5, 'Closed' : 6, + 'Reconfiguring': 7, + 'Reconfigured' : 8, } xoptions = XendOptions.instance() @@ -88,6 +90,8 @@ class DevController: (devid, back, front) = self.getDeviceDetails(config) if devid is None: return 0 + + self.setupDevice(config) (backpath, frontpath) = self.addStoreEntries(config, devid, back, front) @@ -198,6 +202,16 @@ class DevController: if status == Timeout: raise VmError("Device %s (%s) could not be disconnected. " % + (devid, self.deviceClass)) + + + def waitForDevice_reconfigure(self, devid): + log.debug("Waiting for %s - reconfigureDevice.", devid) + + (status, err) = self.waitForBackend_reconfigure(devid) + + if status == Timeout: + raise VmError("Device %s (%s) could not be reconfigured. " % (devid, self.deviceClass)) @@ -325,6 +339,10 @@ class DevController: """ raise NotImplementedError() + + def setupDevice(self, config): + """ Setup device from config. """ + return def migrate(self, deviceConfig, network, dst, step, domName): """ Migration of a device. The 'network' parameter indicates @@ -412,6 +430,14 @@ class DevController: return result + def removeBackend(self, devid, *args): + frontpath = self.frontendPath(devid) + backpath = xstransact.Read(frontpath, "backend") + if backpath: + return xstransact.Remove(backpath, *args) + else: + raise VmError("Device %s not connected" % devid) + def readBackend(self, devid, *args): frontpath = self.frontendPath(devid) backpath = xstransact.Read(frontpath, "backend") @@ -562,6 +588,23 @@ class DevController: return result['status'] + def waitForBackend_reconfigure(self, devid): + frontpath = self.frontendPath(devid) + backpath = xstransact.Read(frontpath, "backend") + if backpath: + statusPath = backpath + '/' + "state" + ev = Event() + result = { 'status': Timeout } + + xswatch(statusPath, xenbusStatusCallback, ev, result) + + ev.wait(DEVICE_CREATE_TIMEOUT) + + return (result['status'], None) + else: + return (Missing, None) + + def backendPath(self, backdom, devid): """Construct backend path given the backend domain and device id. @@ -626,3 +669,19 @@ def deviceDestroyCallback(statusPath, ev ev.set() return 0 + + +def xenbusStatusCallback(statusPath, ev, result): + log.debug("xenbusStatusCallback %s.", statusPath) + + status = xstransact.Read(statusPath) + + if status == str(xenbusState['Connected']): + result['status'] = Connected + else: + return 1 + + log.debug("xenbusStatusCallback %d.", result['status']) + + ev.set() + return 0 diff -r e1dde6f8bc87 tools/python/xen/xend/server/pciif.py --- a/tools/python/xen/xend/server/pciif.py Wed Feb 13 18:30:12 2008 +0000 +++ b/tools/python/xen/xend/server/pciif.py Thu Feb 21 19:30:30 2008 +0900 @@ -23,7 +23,7 @@ from xen.xend.XendError import VmError from xen.xend.XendError import VmError from xen.xend.XendLogging import log -from xen.xend.server.DevController import DevController +from xen.xend.server.DevController import DevController, xenbusState import xen.lowlevel.xc @@ -43,6 +43,15 @@ while not (t&1): t>>=1 PAGE_SHIFT+=1 +def parse_hex(val): + try: + if isinstance(val, types.StringTypes): + return int(val, 16) + else: + return val + except ValueError: + return None + class PciController(DevController): def __init__(self, vm): @@ -51,15 +60,6 @@ class PciController(DevController): def getDeviceDetails(self, config): """@see DevController.getDeviceDetails""" - def parse_hex(val): - try: - if isinstance(val, types.StringTypes): - return int(val, 16) - else: - return val - except ValueError: - return None - back = {} pcidevid = 0 for pci_config in config.get('devs', []): @@ -67,7 +67,6 @@ class PciController(DevController): bus = parse_hex(pci_config.get('bus', 0)) slot = parse_hex(pci_config.get('slot', 0)) func = parse_hex(pci_config.get('func', 0)) - self.setupDevice(domain, bus, slot, func) back['dev-%i' % pcidevid] = "%04x:%02x:%02x.%02x" % \ (domain, bus, slot, func) pcidevid += 1 @@ -127,7 +126,7 @@ class PciController(DevController): return sxpr - def setupDevice(self, domain, bus, slot, func): + def setupOneDevice(self, domain, bus, slot, func): """ Attach I/O resources for device to frontend domain """ fe_domid = self.getDomid() @@ -181,6 +180,167 @@ class PciController(DevController): raise VmError(('pci: failed to configure irq on device '+ '%s - errno=%d')%(dev.name,rc)) + def setupDevice(self, config): + """Setup devices form config""" + pcidevid = 0 + for pci_config in config.get('devs', []): + domain = parse_hex(pci_config.get('domain', 0)) + bus = parse_hex(pci_config.get('bus', 0)) + slot = parse_hex(pci_config.get('slot', 0)) + func = parse_hex(pci_config.get('func', 0)) + self.setupOneDevice(domain, bus, slot, func) + pcidevid += 1 + + return + + def reconfigureDevice(self, _, config): + """@see DevController.reconfigureDevice""" + (devid, back, front) = self.getDeviceDetails(config) + num_devs = int(back['num_devs']) + states = config.get('states', []) + + num_olddevs = int(self.readBackend(devid, 'num_devs')) + + for i in range(num_devs): + try: + dev = back['dev-%i' % i] + state = states[i] + except: + raise XendError('Error reading config') + + if state == 'Initialising': # Attach + # Check if device exists. + for j in range(num_olddevs): + if dev == self.readBackend(devid, 'dev-%i' % j): + raise XendError('Device %s is already connected.' % dev) + log.debug('Attaching PCI device %s.' % dev) + (domain, bus, slotfunc) = dev.split(':') + (slot, func) = slotfunc.split('.') + domain = parse_hex(domain) + bus = parse_hex(bus) + slot = parse_hex(slot) + func = parse_hex(func) + self.setupOneDevice(domain, bus, slot, func) + + self.writeBackend(devid, 'dev-%i' % (num_olddevs + i), dev) + self.writeBackend(devid, 'state-%i' % (num_olddevs + i), + str(xenbusState['Initialising'])) + self.writeBackend(devid, 'num_devs', str(num_olddevs + i + 1)) + + elif state == 'Closing': # Detach + # Check if device exists. + found = False + for j in range(num_olddevs): + if dev == self.readBackend(devid, 'dev-%i' % j): + found = True + log.debug('Detaching device %s' % dev) + self.writeBackend(devid, 'state-%i' % j, + str(xenbusState['Closing'])) + if not found: + raise XendError('Device %s is not connected' % dev) + + else: + raise XendError('Error reconfiguring device %s' % dev) + + self.writeBackend(devid, 'state', str(xenbusState['Reconfiguring'])) + + return self.readBackend(devid, 'uuid') + + def cleanupOneDevice(self, domain, bus, slot, func): + """ Detach I/O resources for device from frontend domain + """ + fe_domid = self.getDomid() + + try: + dev = PciDevice(domain, bus, slot, func) + except Exception, e: + raise VmError("pci: failed to locate device and "+ + "parse it's resources - "+str(e)) + + if dev.driver!='pciback': + raise VmError(("pci: PCI Backend does not own device "+ \ + "%s\n"+ \ + "See the pciback.hide kernel "+ \ + "command-line parameter or\n"+ \ + "bind your slot/device to the PCI backend using sysfs" \ + )%(dev.name)) + + for (start, size) in dev.ioports: + log.debug('pci: disabling ioport 0x%x/0x%x'%(start,size)) + rc = xc.domain_ioport_permission(domid = fe_domid, first_port = start, + nr_ports = size, allow_access = False) + if rc<0: + raise VmError(('pci: failed to configure I/O ports on device '+ + '%s - errno=%d')%(dev.name,rc)) + + for (start, size) in dev.iomem: + # Convert start/size from bytes to page frame sizes + start_pfn = start>>PAGE_SHIFT + # Round number of pages up to nearest page boundary (if not on one) + nr_pfns = (size+(PAGE_SIZE-1))>>PAGE_SHIFT + + log.debug('pci: disabling iomem 0x%x/0x%x pfn 0x%x/0x%x'% \ + (start,size,start_pfn,nr_pfns)) + rc = xc.domain_iomem_permission(domid = fe_domid, + first_pfn = start_pfn, + nr_pfns = nr_pfns, + allow_access = False) + if rc<0: + raise VmError(('pci: failed to configure I/O memory on device '+ + '%s - errno=%d')%(dev.name,rc)) + + if dev.irq>0: + log.debug('pci: disabling irq %d'%dev.irq) + rc = xc.domain_irq_permission(domid = fe_domid, pirq = dev.irq, + allow_access = False) + if rc<0: + raise VmError(('pci: failed to configure irq on device '+ + '%s - errno=%d')%(dev.name,rc)) + + def cleanupDevice(self, devid): + """ Detach I/O resources for device and cleanup xenstore entries + after reconfigure. + + @param devid: The device ID + @type devid: int + @return: Return the number of devices connected + @rtype: int + """ + num_devs = int(self.readBackend(devid, 'num_devs')) + new_num_devs = 0 + for i in range(num_devs): + state = int(self.readBackend(devid, 'state-%i' % i)) + if state == xenbusState['Closing']: + # Detach I/O resources. + dev = self.readBackend(devid, 'dev-%i' % i) + (domain, bus, slotfunc) = dev.split(':') ### + (slot, func) = slotfunc.split('.') ### + domain = parse_hex(domain) + bus = parse_hex(bus) + slot = parse_hex(slot) + func = parse_hex(func) + self.cleanupOneDevice(domain, bus, slot, func) + # Remove xenstore entries. + self.removeBackend(devid, 'dev-%i' % i) + self.removeBackend(devid, 'vdev-%i' % i) + self.removeBackend(devid, 'state-%i' % i) + else: + if new_num_devs != i: + tmpdev = self.readBackend(devid, 'dev-%i' % i) + self.writeBackend(devid, 'dev-%i' % new_num_devs, tmpdev) + self.removeBackend(devid, 'dev-%i' % i) + tmpvdev = self.readBackend(devid, 'vdev-%i' % i) + self.writeBackend(devid, 'vdev-%i' % new_num_devs, tmpvdev) + self.removeBackend(devid, 'vdev-%i' % i) + tmpstate = self.readBackend(devid, 'state-%i' % i) + self.writeBackend(devid, 'state-%i' % new_num_devs, tmpstate) + self.removeBackend(devid, 'state-%i' % i) + new_num_devs = new_num_devs + 1 + + self.writeBackend(devid, 'num_devs', str(new_num_devs)) + + return new_num_devs + def waitForBackend(self,devid): return (0, "ok - no hotplug") diff -r e1dde6f8bc87 tools/python/xen/xm/main.py --- a/tools/python/xen/xm/main.py Wed Feb 13 18:30:12 2008 +0000 +++ b/tools/python/xen/xm/main.py Thu Feb 21 19:30:30 2008 +0900 @@ -175,6 +175,10 @@ SUBCOMMAND_HELP = { 'vnet-delete' : ('', 'Delete a Vnet.'), 'vnet-list' : ('[-l|--long]', 'List Vnets.'), 'vtpm-list' : (' [--long]', 'List virtual TPM devices.'), + 'pci-attach' : (' ', + 'Create a new pass-through pci device.'), + 'pci-detach' : (' ', + 'Destroy a domain\'s pass-through pci device.'), # security @@ -335,6 +339,8 @@ device_commands = [ "network-detach", "network-list", "vtpm-list", + "pci-attach", + "pci-detach", ] vnet_commands = [ @@ -2197,6 +2203,39 @@ def xm_network_attach(args): usage('network-attach') vif.append(vif_param) server.xend.domain.device_create(dom, vif) + + +def parse_pci_configuration(args, state): + dom = args[0] + pci_dev_str = args[1] + pci=['pci'] + pci_match = re.match(r"((?P[0-9a-fA-F]{1,4})[:,])?" + \ + r"(?P[0-9a-fA-F]{1,2})[:,]" + \ + r"(?P[0-9a-fA-F]{1,2})[.,]" + \ + r"(?P[0-9a-fA-F])", pci_dev_str) + if pci_match == None: + raise OptionError("Invalid argument: %s" % pci_dev_str) + pci_dev_info = pci_match.groupdict('0') + try: + pci.append(['dev', ['domain', '0x'+ pci_dev_info['domain']], \ + ['bus', '0x'+ pci_dev_info['bus']], + ['slot', '0x'+ pci_dev_info['slot']], + ['func', '0x'+ pci_dev_info['func']]]) + except: + raise OptionError("Invalid argument: %s" % pci_dev_str) + pci.append(['state', state]) + + return (dom, pci) + +def xm_pci_attach(args): + arg_check(args, 'pci-attach', 2) + (dom, pci) = parse_pci_configuration(args, 'Initialising') + server.xend.domain.device_configure(dom, pci) + +def xm_pci_detach(args): + arg_check(args, 'pci-detach', 2) + (dom, pci) = parse_pci_configuration(args, 'Closing') + server.xend.domain.device_configure(dom, pci) def detach(args, deviceClass): @@ -2452,6 +2491,9 @@ commands = { "vnet-delete": xm_vnet_delete, # vtpm "vtpm-list": xm_vtpm_list, + # pci + "pci-attach": xm_pci_attach, + "pci-detach": xm_pci_detach, } ## The commands supported by a separate argument parser in xend.xm.