"""
Module that allows to work with Plesk backup XML files from Plesk backup archive of different versions:
- read Plesk backup XML as an object model
- modify Plesk backup XML files and store them in a new Plesk backup archive
"""
import tarfile
import zipfile
import gzip
import logging
import re
import itertools
from itertools import imap
from contextlib import closing
from xml.etree import ElementTree

from parallels.utils import if_not_none, parse_bool, obj, group_by_id, \
	exists as utils_exists
from parallels.utils import unique_merge
from parallels.utils.xml import elem, text_elem, create_xml_stream
from parallels.utils.xml import multiple_find
from parallels.utils.xml import multiple_findall
from parallels.utils.xml import multiple_attrib_get
from parallels.common.plesk_backup.data_model import \
	Client, AuxiliaryUser, AuxiliaryUserRole, DatabaseServer, \
	DnsRecord, Subscription, HostingPlan, AddonDomain, Subdomain, DomainAlias, \
	parse_subscription_ips, Mailbox, Mailbox8, DnsSettings, \
	Reseller, ResellerPlan, get_hosting_type, get_maintenance_mode, Subscription8, Subdomain8, \
	parse_php_settings_node, parse_logrotation_node, read_password, parse_ip, \
	parse_system_ip, CustomPlanItem
from parallels.common.plesk_backup.file_adapters import TarFileAdapter, ZipFileAdapter, Plesk8UnixBackupFileAdapter, \
	PlainFileAdapter
from parallels.common.utils import unique_name_generator

logger = logging.getLogger(__name__)


def load_backup(
	path,
	migration_list_data=None,
	is_expand_mode=False,
	discard_mailsystem=False,
):
	"""
	is_expand_node defines how resellers filtering should work:
	- While migrating from Plesk standalone, Plesk reseller is migrated to PPA reseller.
	Reseller is migrated only if it is in migration list.
	- While migrating from Plesk in Expand cluster, Plesk reseller is migrated to PPA client.
	Reseller is migrated only if it has at least one own subscription.

	Implementation detail: additionally in Expand mode we iterate over reseller if its clients have subscriptions
	no matter if it has own subscriptions.
	"""
	container = PleskBackupContainer(path, discard_mailsystem)
	if isinstance(container.archive, PlainFileAdapter):
		# For single XML file, use Plesk 10 reader,
		# to read all objects from single file,
		return PleskBackupSource10(container, migration_list_data, is_expand_mode)
	elif container.dump_version >= (11, 1):
		return PleskBackupSource11(container, migration_list_data, is_expand_mode)
	elif container.dump_version[0] in (10, 11):
		return PleskBackupSource10(container, migration_list_data, is_expand_mode)
	elif container.dump_version[0] == 9:
		return PleskBackupSource9(container, migration_list_data, is_expand_mode)
	elif container.dump_version[0] == 8:
		return PleskBackupSource8(container, migration_list_data)
	else:
		raise Exception(u"Unsupported backup dump version: %r" % (container.dump_version,))


class PleskBackupContainer(object):
	logger = logging.getLogger(__name__)

	def __init__(self, file_name, discard_mailsystem=False):
		if file_name.endswith('.xml'):
			self._load_from_plain_xml_file(file_name)
		else:
			self._load_from_archive(discard_mailsystem, file_name)

		self._filename = file_name
		self.dump_version = tuple(map(int, self.backup_info.getroot().attrib['dump-version'].split('.')))
		self.agent_name = self.backup_info.getroot().attrib['agent-name']

	@property
	def filename(self):
		"""Name of a file on migrator server where this backup container is located"""
		return self._filename

	def _load_from_plain_xml_file(self, file_name):
		logger.debug(u"Opening XML dump file: %s", file_name)
		self.archive = PlainFileAdapter(file_name)
		with open(file_name, 'rb') as fp:
			self.backup_info = ElementTree.ElementTree(file=fp)
		self.backup_info_file = file_name
		self.suffix = None
		self.backup_id = None

	def _load_from_archive(self, discard_mailsystem, file_name):
		# Don't use zipfile.is_zipfile to detect ZIP archive, because it will
		# find ZIP archive within uncompressed TAR archive and happily report
		# that file is ZIP arhive.

		if tarfile.is_tarfile(file_name):
			self.logger.debug("Reading dump from a TAR archive %s" % file_name)
			self.archive = TarFileAdapter(file_name)
			backup_info_re = re.compile(r'(?:\S*info_)?(\d*)\.xml')
		elif zipfile.is_zipfile(file_name):
			self.logger.debug("Reading dump from a ZIP archive %s" % file_name)
			self.archive = ZipFileAdapter(file_name)
			backup_info_re = re.compile(r'(?:\S*info_)?(\d*)\.xml')
		else:
			self.logger.debug("Reading dump from Plesk 8 backup file %s" % file_name)
			self.archive = Plesk8UnixBackupFileAdapter(file_name)
			backup_info_re = re.compile(r'dump()().xml')

		# Find main backup info file.
		xml_files = {
			m.group(0): m.group(1)
			for m in map(backup_info_re.match, self.archive.get_names())
			if m is not None
		}
		"""
		xml_files contains all xml-files from archive:

		{
			'resellers/ivanov/clients/bob/backup_bob_info_1201180356.xml':
				'1201180356',
			'clients/cat/backup_cat_info_1201180356.xml':
				'1201180356',
			'clients/tom/domains/tom.abaturin.lp.plesk.ru/backup_tom.abaturin.lp.plesk.ru_info_1201180356.xml':
				'1201180356',
			'resellers/ivanov/backup_ivanov_info_1201180356.xml':
				'1201180356',
			'clients/tom/backup_tom_info_1201180356.xml':
				'1201180356',
			'clients/cat/domains/cat.abaturin.lp.plesk.ru/backup_cat.abaturin.lp.plesk.ru_info_1201180356.xml':
				'1201180356',
			'backup_info_1201180356.xml':
				'1201180356',
			'clients/john/domains/john.tld/backup_john.tld_info_1201180356.xml':
				'1201180356',
			'resellers/ivanov/clients/bob/domains/bob.tld/backup_bob.tld_info_1201180356.xml':
				'1201180356',
			'clients/john/backup_john_info_1201180356.xml':
				'1201180356'
		}

		We need to select top level backup. It always has shortest name (e.g. backup_info_1201180356.xml) because it is
		located in achive's root but all other XML is placed in subfolders
		"""
		self.backup_info_file = min(xml_files.keys(), key=lambda x: len(x))
		self.backup_id = xml_files[self.backup_info_file]

		self.backup_info = self._load_xml(self.backup_info_file)
		if discard_mailsystem:
			for mailsystem_parent_node in self.backup_info.findall('.//mailsystem/..'):
				mailsystem_parent_node.remove(mailsystem_parent_node.find('mailsystem'))

	def close(self):
		self.archive.close()

	def _load_xml(self, path):
		with closing(self.archive.extract(path)) as source:
			tree = self._cleanup_tree(ElementTree.ElementTree(file=source))
		self.archive.close()
		return tree

	def _cleanup_tree(self, tree):
		if tree.getroot().tag == '{urn:envelope}Envelope':
			data = tree.getroot().find('{urn:envelope}Data/{urn:envelope}migration-dump')
			self._remove_namespace(data, '{urn:envelope}')
			return ElementTree.ElementTree(data)
		else:
			return tree

	def _remove_namespace(self, root_node, namespace):
		for node in root_node.iter():
			if node.tag.startswith(namespace):
				node.tag = node.tag[len(namespace):]


class PleskBackupSourceBase(object):
	def iter_all_subscriptions_with_owner_and_reseller(self):
		"""Get all subscriptions - subscriptions owned by admin, resellers and customers.

		:rtype: list[(
			parallels.common.plesk_backup.data_model.Subscription,
			parallels.common.plesk_backup.data_model.Client | None,
			parallels.common.plesk_backup.data_model.Reseller | None
		)]
		"""
		for subscription in self.iter_admin_subscriptions():
			yield subscription, None, None
		
		for client in self.iter_clients():
			for subscription in client.subscriptions:
				yield subscription, client, None

		for reseller in self.iter_resellers():
			for subscription in reseller.subscriptions:
				yield subscription, reseller, reseller  # reseller subscribes himself on his own plans

			for client in reseller.clients:
				for subscription in client.subscriptions:
					yield subscription, client, reseller

	def iter_all_subscriptions(self):
		"""Get all subscriptions - subscriptions owned by admin, resellers and customers.

		:rtype: list[parallels.common.plesk_backup.data_model.Subscription]
		"""
		for subscription, _, _ in self.iter_all_subscriptions_with_owner_and_reseller():
			yield subscription

	def iter_all_clients_with_owner(self):
		"""
		:rtype: list[(
			parallels.common.plesk_backup.data_model.Client,
			parallels.common.plesk_backup.data_model.Reseller | None
		)]
		"""
		for client in self.iter_clients():
			yield client, None

		for reseller in self.iter_resellers():
			for client in reseller.clients:
				yield client, reseller

	def iter_all_clients(self):
		"""
		:rtype: list[parallels.common.plesk_backup.data_model.Client]
		"""
		for client, _ in self.iter_all_clients_with_owner():
			yield client

	def iter_resellers(self):
		"""
		:rtype: list[parallels.common.plesk_backup.data_model.Reseller]
		"""
		return []

	def iter_clients(self):
		"""
		:rtype: list[parallels.common.plesk_backup.data_model.Client]
		"""
		return []

	def iter_admin_subscriptions(self):
		"""
		:rtype: list[parallels.common.plesk_backup.data_model.Subscription]
		"""
		return []

	def get_system_ips(self):
		"""
		:rtype: list
		"""
		return []

	def get_default_ip(self):
		"""
		:rtype: basestring | None
		"""
		return None


class PleskBackupSource(PleskBackupSourceBase):
	logger = logging.getLogger(__name__)

	def __init__(self, container, migration_list_data=None, is_expand_mode=False):
		self.container = container
		self.backup_info = self.container.backup_info
		self.is_expand_mode = is_expand_mode
		# if None is passed - nothing is filtered out
		self.migration_list_data = _extract_migration_list_data(migration_list_data)

		self.resellers_info, self.clients_info, self.domains_info = self._index_entities()
		self.resellers_to_save = self.resellers_info.keys()
		self.clients_to_save = self.clients_info.keys()
		self.domains_to_save = self.domains_info.keys()
		self.unique_plan_names = unique_name_generator.UniqueNameGenerator("%d")

	@property
	def backup_info_file(self):
		return self.container.backup_info_file

	def get_indexed(self):
		return self._indexed

	def _index_entities(self):
		"""Get XML node for all resellers, clients and domains.
		
		Returns (resellers_map, clients_map, domains_map) where each map is dictionary
		from entity name to root XML node with entity info.

		Override this method in ancestors.
		"""
		raise NotImplementedError()

	def _iter_xml_files(self, pattern):
		for member in self.container.archive.get_members():
			if member.is_file \
				and member.name.endswith('.xml') \
				and member.name != self.container.backup_info_file \
				and re.match(pattern, member.name):

				yield member.name, self.container._load_xml(member.name)

	def _iter_reseller_files(self):
		for path, xml in self._iter_xml_files(re.compile(r'^resellers/[^/]+/[^/]+\.xml$')):
			node = xml.find('reseller')
			name = node.attrib['name']
			yield path, xml, node, name

	def _iter_client_files(self):
		for path, xml in self._iter_xml_files(re.compile(r'^(.*/)?clients/[^/]+/[^/]+\.xml$')):
			node = xml.find('client')
			name = node.attrib['name']
			yield path, xml, node, name

	def _iter_domain_files(self):
		for path, xml in self._iter_xml_files(re.compile(r'^(.*/)?domains/[^/]+/[^/]+\.xml$')):
			node = xml.find('domain')
			name = node.attrib['name']
			yield path, xml, node, name

		for node in self.backup_info.findall('./domain'):
			name = node.attrib['name']
			yield self.backup_info_file, self.backup_info, node, name

	def close(self):
		self.container.close()

	def _is_backup_archive_member_relevant(self, member):
		"""Determine, if the given file should be extracted from archive.
		
		When saving the backup back into the tar file, we need to know whether
		a file of the source backup was relevant to migration list. If it's
		not, it won't be saved into the tar file.
		"""
		# having the paths like:
		# clients/john/domains/domain.tld/info.xml
		# clients/john/domains/domain.tld/.discovered/info/dump_part
		# we want to extract from them the directory of the corresponding info.xml file:
		# clients/john/domains/domain.tld/
		# by calling get_dirname with offset=1 and offset=3, respectively.

		def get_dirname(path, offset=1):
			return '/'.join(path.split('/')[:-offset]) + '/'

		def get_filename(path):
			return path.rstrip('/').split('/').pop()

		filename_blacklist = ('dump-status.xml',)
		if get_filename(member.name) in filename_blacklist:
			logger.debug(u"Discarding a file '%s' (blacklisted)." % member.name)
			return False

		indexed_by_directory = group_by_id(self._indexed, lambda i: get_dirname(i.path.lstrip('./')))
		if '.discovered' in member.name.split('/'):
			member_directory = get_dirname(member.name.lstrip('./'), offset=3)
		else:
			member_directory = get_dirname(member.name.lstrip('./'))

		if member_directory == '/':
			return True  # backup's root info xml is always relevant
		else:
			indexed_entry = indexed_by_directory.get(member_directory)

			# There can be more paths - like one below - in some backups (e.g. PfW 11.x):
			# clients/customer1/domains/customer-domain.com/databases/
			# customer1_mssql_2/.discovered/backup_customer1_mssql_2_info_1404151534/GUID_
			#
			# These files were added with intent to allow partial restore of some part of subscription:
			# its databases, hosting or mailnames.
			# However, these files aren't used even in Plesk scenarios. So, we are safe to discard these
			# paths from backup.
			if indexed_entry is None:
				self.logger.debug("Discard path %r as having unknown type/purpose and being not needed." % member.name)
				return False

			if indexed_entry.obj_type == 'reseller':
				if indexed_entry.obj_name not in self.resellers_to_save:
					return False
			elif indexed_entry.obj_type == 'client':
				if indexed_entry.obj_name not in self.clients_to_save:
					return False
			elif indexed_entry.obj_type == 'domain':
				if indexed_entry.obj_name not in self.domains_to_save:
					return False
			else:
				assert False, "Backup contains a path (%r), associated with an object of unknown type (%s)" % (
					member.name, indexed_entry.obj_type
				)
			return True

	def _get_reseller_names_filtered(self):
		backup_reseller_names = set(self.resellers_info.keys())

		def subscriptions_count(reseller):
			return sum(
				[len(self._get_reseller_domain_names_filtered(reseller))] + 
				[
					len(subscriptions)
					for client in self._get_reseller_client_names_filtered(reseller)
					for subscriptions in self._get_client_domain_names_filtered(client) 
				]
			)

		if self.migration_list_data is not None:
			if self.is_expand_mode:
				return [reseller for reseller in self.resellers_info.keys() if subscriptions_count(reseller) > 0]
			else:
				return [reseller for reseller in self.resellers_info.keys()
					if subscriptions_count(reseller) > 0
					or reseller in self.migration_list_data.resellers
				]
		else:
			return backup_reseller_names

	def _get_admin_plan_names_filtered(self):
		backup_plan_names = set([
			plan_node.attrib['name']
			for plan_node in self.backup_info.findall('server/account-templates/domain-template')
		])
		if self.migration_list_data is not None:
			return backup_plan_names & self.migration_list_data.plans[None]	
		else:
			return backup_plan_names

	def _get_reseller_plan_names_filtered(self, reseller_name):
		backup_plan_names = set([
			plan_node.attrib['name']
			for plan_node in self.resellers_info[reseller_name].findall('preferences/domain-template')
		])
		if self.migration_list_data is not None:
			return backup_plan_names & self.migration_list_data.plans.get(reseller_name, set())
		else:
			return backup_plan_names

	def _get_admin_domain_names_filtered(self):
		if self.migration_list_data is not None:
			return set(self._get_admin_domain_names()) & self.migration_list_data.subscriptions
		else:
			return self._get_admin_domain_names()

	def _get_client_domain_names_filtered(self, client_name):
		if self.migration_list_data is not None:
			return set(self._get_client_domain_names(client_name)) & self.migration_list_data.subscriptions
		else:
			return self._get_client_domain_names(client_name)

	def _get_reseller_domain_names_filtered(self, reseller_name):
		if self.migration_list_data is not None:
			return set(self._get_reseller_domain_names(reseller_name)) & self.migration_list_data.subscriptions
		else:
			return self._get_reseller_domain_names(reseller_name)

	def _get_admin_client_names_filtered(self):
		if self.migration_list_data is not None:
			return [
				client_name for client_name in self._get_admin_client_names() 
				if len(self._get_client_domain_names_filtered(client_name)) > 0
				or client_name in self.migration_list_data.customers
			]
		else:
			return self._get_admin_client_names()

	def _get_reseller_client_names_filtered(self, reseller_name):
		if self.migration_list_data is not None:
			return [
				client_name for client_name in self._get_reseller_client_names(reseller_name)
				if len(self._get_client_domain_names_filtered(client_name)) > 0
				or client_name in self.migration_list_data.customers
			]
		else:
			return self._get_reseller_client_names(reseller_name)
		
	def _parse_hosting_plan(self, plan_node, convert_name=lambda n: n):
		self.logger.debug(u"Parse plan '%s'", plan_node.attrib['name'])

		unique_id = plan_node.attrib.get('guid')
		# Plesk 9 doesn't have GUIDs for plans and we generate unique GUID.
		if unique_id is None:
			unique_id = self.unique_plan_names(plan_node.attrib['name'])
		else:
			self.unique_plan_names(unique_id)
		return HostingPlan(
			backup=self,
			node=plan_node,
			id=unique_id,
			name=convert_name(plan_node.attrib['name']),
			is_addon=plan_node.attrib.get('is-addon', 'false') == 'true',
			properties=dict((n.attrib['name'], n.text) for n in plan_node.findall('template-item')),
			log_rotation=parse_logrotation_node(plan_node.find('logrotation')),
			php_settings=parse_php_settings_node(plan_node.find('php-settings'))
		)

	def get_plans(self):
		"""Get hosting plans (excluding add-ons) owned by admin.
		"""
		return self._get_plans(is_addon=False)

	def get_plan_by_guid(self, guid):
		plan_node = self.backup_info.find('.//domain-template[@guid="%s"]' % guid)
		if plan_node is None:
			return None
		return self._parse_hosting_plan(plan_node)

	def get_addon_plans(self):
		"""Get hosting addon plans owned by admin
		"""
		return self._get_plans(is_addon=True)

	def get_custom_plan_items(self):
		"""Get custom plan items"""
		plan_item_nodes = self.backup_info.findall('server/account-templates/plan-item')

		plan_items = []
		for node in plan_item_nodes:
			plan_items.append(CustomPlanItem(node))

		return plan_items

	def _get_plans(self, is_addon):
		is_addon_value = 'true' if is_addon else 'false'
		filtered_plan_names = self._get_admin_plan_names_filtered()
		plan_nodes = self.backup_info.findall('server/account-templates/domain-template')
		for plan_node in plan_nodes:
			if plan_node.attrib['name'] in filtered_plan_names and plan_node.attrib.get('is-addon', 'false') == is_addon_value:
				yield self._parse_hosting_plan(plan_node)

	def iter_admin_reseller_plans(self):
		"""Get reseller plans of admin user.
		"""
		plan_nodes = self.backup_info.findall('server/account-templates/reseller-template')
		for plan_node in plan_nodes:
			self.logger.debug(u"Parse reseller plan '%s'", plan_node.attrib['name'])
			unique_id = plan_node.attrib.get('guid')
			# Plesk 9 doesn't have GUIDs for plans and we generate unique GUID.
			if unique_id is None:
				unique_id = self.unique_plan_names(plan_node.attrib['name'])
			else:
				self.unique_plan_names(unique_id)
			yield ResellerPlan(
				node=plan_node,
				id=unique_id,
				name=plan_node.attrib['name'],
				properties=dict((n.attrib['name'], n.text) for n in plan_node.findall('template-item')),
				ip_pool=[parse_ip(n) for n in plan_node.findall('ip_pool/ip')],
			)

	def iter_admin_and_admin_customers_subscriptions(self):
		"""Get subscriptions owned by admin and subscriptions of customers owned by admin.

		Returns iterable with instances of Subscription.
		"""
		for subscription in self.iter_admin_subscriptions():
			yield subscription
		
		for client in self.iter_clients():
			for subscription in client.subscriptions:
				yield subscription

	def iter_admin_subscriptions(self):
		domain_names = self._get_admin_domain_names_filtered()
		return (self.get_subscription(name) for name in domain_names)

	def iter_admin_auxiliary_user_roles(self):
		return self._parse_roles_node(self.backup_info.find('admin/roles'))

	def iter_admin_auxiliary_users(self):
		return self._parse_users_node(self.backup_info.find('admin/users'), 'admin')

	def iter_resellers(self):
		for reseller_name in self._get_reseller_names_filtered():
			info = self.resellers_info[reseller_name]
			domain_names = self._get_reseller_domain_names_filtered(reseller_name)
			pinfo_nodes = info.findall('preferences/pinfo')
			password_node = info.find('properties/password')
			if password_node is not None:
				password = read_password(password_node)
			else:
				password = None
			
			yield Reseller(
				login=reseller_name, 
				password=password,
				plan_id=if_not_none(info.find('preferences/subscription/plan'), lambda p: p.attrib['plan-guid']),
				limits=dict(
					(n.attrib['name'], n.text)
					for n in info.findall('limits-and-permissions/limit')
				),
				permissions=dict(
					(n.attrib['name'], n.attrib['value'] == 'true')
					for n in info.findall('limits-and-permissions/permission')
				),
				contact=info.attrib.get('contact'),
				clients=[self.get_client(client_name) for client_name in self._get_reseller_client_names_filtered(reseller_name)],
				subscriptions=[
					self.get_subscription(domain_name) for domain_name in domain_names
				],
				personal_info=dict(
					(pinfo_node.attrib['name'], pinfo_node.text) for pinfo_node in pinfo_nodes
				),
				auxiliary_users=self._get_auxiliary_users(info),
				auxiliary_user_roles=self._get_auxiliary_user_roles(info),
				is_enabled=info.find('properties/status/enabled') is not None,
			)

	def iter_reseller_plans(self, reseller_name):
		return self._iter_reseller_addon_plans(reseller_name, is_addon=False)

	def iter_reseller_addon_plans(self, reseller_name):
		return self._iter_reseller_addon_plans(reseller_name, is_addon=True)

	def _iter_reseller_addon_plans(self, reseller_name, is_addon):
		is_addon_value = 'true' if is_addon else 'false'
		res_tree = self.resellers_info[reseller_name]
		plan_nodes = res_tree.findall('preferences/domain-template')
		filtered_plan_names = self._get_reseller_plan_names_filtered(reseller_name)
		for plan_node in plan_nodes:
			if plan_node.attrib['name'] in filtered_plan_names and plan_node.attrib.get('is-addon', 'false') == is_addon_value:
				yield self._parse_hosting_plan(plan_node)

	def iter_clients(self):
		for client_name in self._get_admin_client_names_filtered():
			yield self.get_client(client_name)

	def get_client(self, client_name):
		info = self.clients_info[client_name]
		domain_names = self._get_client_domain_names_filtered(client_name)
		pinfo_nodes = info.findall('preferences/pinfo')
		password_node = info.find('properties/password')
		if password_node is not None:
			password = read_password(password_node)
		else:
			password = None
		auxiliary_users, auxiliary_roles = self._get_auxiliary_users_and_roles(
			client_name, info
		)
		
		return Client(
			login=client_name, 
			password=password,
			contact=info.attrib.get('contact'),
			subscriptions=[
				self.get_subscription(domain_name) for domain_name in domain_names
			],
			personal_info=dict(
				(pinfo_node.attrib['name'], pinfo_node.text) for pinfo_node in pinfo_nodes
			),
			auxiliary_users=auxiliary_users,
			auxiliary_user_roles=auxiliary_roles,
			is_enabled=info.find('properties/status/enabled') is not None,
		)

	def _get_auxiliary_users_and_roles(self, client_name, client_info):
		# Get list of domain XML nodes
		domain_names = self._get_client_domain_names_filtered(client_name)
		domain_nodes = []
		for domain_name in domain_names:
			domain_nodes.append(self.domains_info[domain_name])

		# Read domain administrators
		domain_admin_users, domain_admin_roles = _get_client_auxiliary_users_and_roles_for_domain_admin(
			client_info, domain_nodes
		)

		# Read mail users
		mailuser_users, mailuser_roles = _get_client_auxiliary_users_and_roles_for_mailusers(
			client_info, domain_nodes
		)

		# Read regular users
		regular_users = self._get_auxiliary_users(client_info)
		regular_roles = self._get_auxiliary_user_roles(client_info)

		# Merge users and roles by their names
		users = unique_merge(
			key=lambda user: user.name,
			# Domains and mail users have more priority than regular client
			# users, to get correct permissions in case of backup
			# created by PMM agent for Plesk 8/9
			lists=[domain_admin_users, mailuser_users, regular_users]
		)
		roles = unique_merge(
			key=lambda role: role.name,
			lists=[domain_admin_roles, mailuser_roles, regular_roles]
		)

		return users, roles

	@classmethod
	def _get_auxiliary_users(cls, client_info):
		return list(cls._parse_users_node(client_info.find('users'), "client '%s'" % client_info.attrib['name']))

	@classmethod
	def _get_auxiliary_user_roles(cls, client_info):
		return list(cls._parse_roles_node(client_info.find('roles')))

	@classmethod
	def _parse_roles_node(cls, node):
		if node is None:
			return

		for role_node in node.findall('role'):
			if 'is-built-in' in role_node.attrib and role_node.attrib['is-built-in'] == 'false':
				yield AuxiliaryUserRole(
					name=role_node.attrib['name'],
					permissions=dict(
						(perm_node.attrib['name'], True if perm_node.attrib['value'] == 'true' else False)
						for perm_node in role_node.findall('limits-and-permissions/permission')
						if perm_node.attrib['value'] in ('true', 'false')
					),
					is_domain_admin=False
				)
	
	@classmethod
	def _parse_users_node(cls, node, parent_object_name):
		if node is None:
			return

		for user_node in node.findall('user'):
			password_node = user_node.find('properties/password')
			password = read_password(password_node)

			personal_info = dict(
				(pinfo_node.attrib['name'], pinfo_node.text) for pinfo_node in user_node.findall('preferences/pinfo')
			)

			im_map = {  # in Plesk backup IM is presented as a number, while we need a name
				'1': 'ICQ',
				'2': 'Jabber',
				'3': 'AIM',
				'4': 'Skype',
				'0': 'Other',
			}
			if 'im-type' in personal_info:
				personal_info['im-type'] = im_map[personal_info['im-type']]

			yield AuxiliaryUser(
				name=user_node.attrib['name'], 
				contact=user_node.attrib.get('contact'), 
				email=user_node.attrib.get('email'), 
				password=password, 
				roles=[perm_node.text for perm_node in user_node.findall('limits-and-permissions/role-ref')],
				personal_info=personal_info,
				subscription_name=user_node.attrib.get('subscription-name', None),
				is_active=user_node.find('properties/status/enabled') is not None,
				is_domain_admin=user_node.attrib['is-domain-admin'] != 'false',
				is_built_in=user_node.attrib.get('is-built-in') != 'false'
			)

	def get_subscription(self, name):
		if name not in self.domains_info:
			raise SubscriptionNotFoundException(name)
		domain_info = self.domains_info[name]
		ips = parse_subscription_ips(domain_info.findall('properties/ip'))
		return Subscription(
			node=domain_info,
			backup=self,
			name=name,
			plan_id=self.get_subscription_plan_guid(name),
			ip=ips.v4, ip_type=ips.v4_type, ipv6=ips.v6, ipv6_type=ips.v6_type,
			limits=dict(
				(n.attrib['name'], n.text)
				for n in domain_info.findall('limits-and-permissions/limit')
			),
			permissions=dict(
				(n.attrib['name'], n.attrib['value'] == 'true')
				for n in domain_info.findall('limits-and-permissions/permission')
			),
			is_enabled=domain_info.find('properties/status/enabled') is not None,
			hosting_type=get_hosting_type(domain_info),
			is_maintenance=get_maintenance_mode(domain_info),
		)

	def get_subscription_plan_guid(self, subscription_name):
		domain_info = self.domains_info[subscription_name]
		for plan_node in domain_info.findall('preferences/subscription/plan'):
			if plan_node.attrib.get('is-addon', 'false') == 'false':
				return plan_node.attrib['plan-guid']
		return None

	def iter_db_servers(self):
		db_server_nodes = self.backup_info.findall('server/db-servers/db-server')
		for node in db_server_nodes:
			# Line for PostgreSQL server is always present in Plesk database, even if it isn't configured,
			# but administrator login and password are missing in this case.
			# Perhaps this applies to other database types too.
			# So just skip databases without administrator login.
			db_admin_node = node.find('db-admin')
			if db_admin_node is not None:
				assert db_admin_node.find('password').attrib['type'] == 'plain'
				if node.findtext('port') == '':
					port = 0
				else:
					port = int(node.findtext('port'))
				yield DatabaseServer(
					node=node,
					host=node.findtext('host'),
					port=port,
					login=db_admin_node.attrib['name'],
					password=db_admin_node.findtext('password'),
					dbtype=node.attrib['type'],
					is_default=node.attrib.get('default', 'false') == 'true',
				)

	def iter_mailboxes(self):
		for subscription_name, subscription_node in self.domains_info.iteritems():
			if self.migration_list_data is None or subscription_name in self.migration_list_data.subscriptions:
				for domain_node in itertools.chain([subscription_node], subscription_node.findall('phosting/sites/site')):
					mailuser_nodes = domain_node.findall('mailsystem/mailusers/mailuser')
					for mailuser_node in mailuser_nodes:
						yield Mailbox.parse(mailuser_node, domain_node.attrib['name'])

	def iter_subscription_mailboxes(self, subscription_name):
		subscription_node = self.domains_info[subscription_name]
		for domain_node in itertools.chain(
			[subscription_node],
			subscription_node.findall('phosting/sites/site')
		):
			mailuser_nodes = domain_node.findall('mailsystem/mailusers/mailuser')
			for mailuser_node in mailuser_nodes:
				yield Mailbox.parse(mailuser_node, domain_node.attrib['name'])

	def iter_domain_mailboxes(self, domain_name, subscription_name):
		"""Return Mailbox objects of the given domain."""
		subscription_node = self.domains_info[subscription_name]
		for domain_node in itertools.chain(
			[subscription_node],
			subscription_node.findall('phosting/sites/site')
		):
			if domain_node.get('name') == domain_name:
				mailuser_nodes = domain_node.findall(
					'mailsystem/mailusers/mailuser'
				)
				for mailuser_node in mailuser_nodes:
					yield Mailbox.parse(
						mailuser_node, domain_node.attrib['name']
					)

	def get_subscription_site_names(self, subscription_name, include_subdomains=False):
		domain_info = self.domains_info[subscription_name]
		site_nodes = domain_info.findall('phosting/sites/site')
		return set(s.attrib['name'] for s in site_nodes if 'parent-domain-name' not in s.attrib or include_subdomains)

	def get_dns_settings(self):
		own_zones = self.backup_info.findtext('server/dns-settings/subdomain-own-zones')
		return DnsSettings(
			do_subdomains_own_zones=own_zones is not None and parse_bool(own_zones),
		)

	def iter_dns_template_records(self):
		template_node = self.backup_info.find('server/dns-settings/dns-zone')
		if template_node is not None:
			for node in template_node.findall('dnsrec'):
				yield DnsRecord.parse(node)

	def iter_sites(self, subscription_name):
		"""Iterate over all subscription sites: addon domains and subdomains.
		"""
		# 1) Subscription's own subdomains
		for subdomain in self.iter_subdomains(
			subscription_name, subscription_name
		):
			yield subdomain

		# 2) Addon domains
		for addon_domain in self.iter_addon_domains(subscription_name):
			yield addon_domain

			# 3) Subdomains of addon domains
			for subdomain in self.iter_subdomains(
				subscription_name, addon_domain.name
			):
				yield subdomain

	def iter_addon_domains(self, subscription_name):
		"""Get addon domains.
		"""
		return imap(
			AddonDomain,
			(
				i for i in self.domains_info[subscription_name].iterfind('phosting/sites/site')
				if 'parent-domain-name' not in i.attrib
			)
		)

	def iter_subdomains(self, subscription_name, domain_name):
		# Try both >= 10.4 and < 10.4 ways to get subdomains, as Plesk agent 
		# dumps subdomains in old way for old Plesks, in new way for new Plesks

		# Plesk >= 10.4 way
		for node in self.domains_info[subscription_name].iterfind('phosting/sites/site'):
			if node.attrib.get('parent-domain-name') == domain_name.encode("idna"):
				yield Subdomain(node)

		# Plesk < 10.4 way
		if subscription_name == domain_name:  # get subdomains owned by subscription itself
			for node in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain'):
				yield Subdomain8(subscription_name, node, self.get_subscription(subscription_name))
		else:  # get subdomains owned by addon domain
			for node in self.domains_info[subscription_name].iterfind(
				'phosting/sites/site[@name="%s"]/phosting/subdomains/subdomain' % (domain_name,)
			):
				yield Subdomain8(
					domain_name, node,
					[addon for addon in self.iter_addon_domains(subscription_name) if addon.name == domain_name][0]
				)

	def iter_aliases(self, subscription_name):
		domain_aliases = []
		domain_aliases.extend(
			DomainAlias(subscription_name, node)
			for node in self.domains_info[subscription_name].iterfind('preferences/domain-alias')
		)
		for addon_domain in self.iter_addon_domains(subscription_name):
			domain_aliases.extend(
				DomainAlias(addon_domain.name, node) for node in addon_domain.node.iterfind('preferences/domain-alias')
			)
		return domain_aliases

	def get_system_ips(self):
		return [parse_system_ip(n, self.is_windows) for n in self.backup_info.findall('server/properties/system-ip')]

	def get_default_ip(self):
		return self.backup_info.findtext('server/properties/default-ip')

	@property
	def is_windows(self):
		os_desc = self.backup_info.find('dump-info/os-description')
		return os_desc is not None and os_desc.attrib['type'] == 'windows'

	def replace_applications(self, applications_str):
		def find_applications(applications, domain_name):
			return applications.findall('application[domain-name="%s"]/sapp-installed' % (domain_name,))

		applications = ElementTree.fromstring(applications_str)
		for subscr in self.iter_all_subscriptions():
			subscr.replace_applications(find_applications(applications, subscr.name))
			for subdomain in self.iter_subdomains(subscr.name, subscr.name):
				subdomain.replace_applications(find_applications(applications, subdomain.name))

			for domain in self.iter_addon_domains(subscr.name):
				domain.replace_applications(find_applications(applications, domain.name))
				for subdomain in self.iter_subdomains(subscr.name, domain.name):
					subdomain.replace_applications(find_applications(applications, subdomain.name))


def replace_prefix(file_name, prefix):
	if prefix is not None:
		file_name_base = file_name[file_name.rfind('/') + 1:]
		for old_prefix in ['backup_', 'converted_', 'info_']:
			if file_name_base.startswith(old_prefix):
				file_name = "".join([
					file_name[:-len(file_name_base)],
					prefix, '_',
					file_name_base[len(old_prefix):]
				])
				break
	return file_name


class PleskBackupSource11(PleskBackupSource):
	def _index_entities(self):
		# TODO: Remove loading of all XMLs to memory when PMM team resolve
		# file naming issue. ATM it is impossible to reliably get file name
		# where domain information is stored.
		self._indexed = []
		self.logger.debug(u"Load resellers info")
		resellers_info = self._index_reseller_files()
		self.logger.debug(u"Load clients info")
		clients_info = self._index_client_files()
		self.logger.debug(u"Load domains info")
		domains_info = self._index_domain_files()

		if not utils_exists(self._indexed, lambda l: l.path == self.container.backup_info_file):
			self._indexed.append(
				obj(path=self.container.backup_info_file, obj_type='backup_info', obj_name=None, xml=self.backup_info)
			)

		return resellers_info, clients_info, domains_info

	def _index_reseller_files(self):
		resellers_info = {}
		for path, xml, node, name in self._iter_reseller_files():
			self._indexed.append(obj(path=path, obj_type='reseller', obj_name=name, xml=xml))
			self.logger.debug(u"Parsed info for reseller '%s'", name)
			resellers_info[name] = node
		return resellers_info

	def _index_client_files(self):
		clients_info = {}
		for path, xml, node, name in self._iter_client_files():
			self._indexed.append(obj(path=path, obj_type='client', obj_name=name, xml=xml))
			self.logger.debug(u"Parsed info for client '%s'", name)
			clients_info[name] = node
		return clients_info

	def _index_domain_files(self):
		domains_info = {}
		for path, xml, node, name in self._iter_domain_files():
			self._indexed.append(obj(path=path, obj_type='domain', obj_name=name, xml=xml))
			self.logger.debug(u"Parsed info for domain '%s'", name)
			domains_info[name] = node
		return domains_info

	def save(self, fileobj, archive_format=None, prefix=None):
		"""
		Save migration dump to a given file object

		:param fileobj: fileobj
		:param archive_format: 'tar', 'zip', None
		"""
		indexed_by_path = group_by_id(self._indexed, lambda l: l.path)
		new_archive = (
			ZipFileAdapter(fileobj=fileobj, mode='w')
			if 'zip' == archive_format else TarFileAdapter(fileobj=fileobj, mode='w')
		)
		with closing(new_archive):
			for member in self.container.archive.get_members():
				if member.is_file:
					if not self._is_backup_archive_member_relevant(member):
						continue

					file_name = member.name

					if member.name in indexed_by_path:
						source = create_xml_stream(indexed_by_path[member.name].xml)
						file_size = len(source.getvalue())
					else:
						source = self.container.archive.extract(member.name)
						file_size = member.size
					with closing(source):
						file_name = replace_prefix(file_name, prefix)
						new_archive.add_file(file_name, file_size, source)
			self.container.archive.close()

	def cut_irrelevant_subcriptions(self, relevant_subscription_names):
		# cut domain-info nodes from clients' xml files
		client_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'client'}
		for client_info_node in (
			self.backup_info.findall('admin/clients/client-info') +
			self.backup_info.findall('admin/resellers/reseller-info/client-info')
		):
			for domains_node in client_xml_by_name[client_info_node.attrib['name']].findall('client/domains'):
				for domain_info_node in domains_node.findall('domain-info'):
					if domain_info_node.attrib['name'] not in relevant_subscription_names:
						domains_node.remove(domain_info_node)

		# cut domain-info nodes from resellers' xml files
		reseller_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'reseller'}
		for reseller_info_node in self.backup_info.findall('admin/resellers/reseller-info'):
			reseller_xml = reseller_xml_by_name[reseller_info_node.attrib['name']]

			for domains_node in reseller_xml.findall('reseller/domains'):
				for domain_info_node in domains_node.findall('domain-info'):
					if domain_info_node.attrib['name'] not in relevant_subscription_names:
						domains_node.remove(domain_info_node)

			for client_info_node in reseller_xml.findall('reseller/clients/client-info'):
				for domain_info_node in client_info_node.findall('domain-info'):
					if domain_info_node.attrib['name'] not in relevant_subscription_names:
						client_info_node.remove(domain_info_node)

		# cut domain-info nodes from the root xml file
		for domains_parent_node in (
			self.backup_info.findall('admin/domains') +
			self.backup_info.findall('admin/clients/client-info') +
			self.backup_info.findall('admin/resellers/reseller-info') +
			self.backup_info.findall('admin/resellers/reseller-info/client-info')
		):
			for domain_node in domains_parent_node.findall('domain-info'):
				if domain_node.attrib['name'] not in relevant_subscription_names:
					domains_parent_node.remove(domain_node)

		self.domains_to_save = relevant_subscription_names

	def cut_irrelevant_clients(self, relevant_logins):
		# cut domain-info nodes from resellers' xml files
		reseller_xml_by_name = {i.obj_name: i.xml for i in self._indexed if i.obj_type == 'reseller'}
		for reseller_info_node in self.backup_info.findall('admin/resellers/reseller-info'):
			reseller_xml = reseller_xml_by_name[reseller_info_node.attrib['name']]

			for clients_node in reseller_xml.findall('reseller/clients'):
				for client_info_node in clients_node.findall('client-info'):
					if client_info_node.attrib['name'] not in relevant_logins:
						clients_node.remove(client_info_node)

		for clients_parent_node in (
			self.backup_info.findall('admin/clients') +
			self.backup_info.findall('admin/resellers/reseller-info')
		):
			for client_node in clients_parent_node.findall('client-info'):
				if client_node.attrib['name'] not in relevant_logins:
					clients_parent_node.remove(client_node)
		self.clients_to_save = relevant_logins

	def cut_irrelevant_resellers(self, relevant_logins):
		for resellers_parent_node in self.backup_info.findall('admin/resellers'):
			for reseller_node in resellers_parent_node.findall('reseller-info'):
				if reseller_node.attrib['name'] not in relevant_logins:
					resellers_parent_node.remove(reseller_node)
		self.resellers_to_save = relevant_logins

	def _get_admin_domain_names(self):
		return [
			n.attrib['name'] for n in itertools.chain(
				self.backup_info.findall('admin/domains/domain-info'),
				self.backup_info.findall('domain-info'),
				self.backup_info.findall('domain'),
			)
		]

	def _get_client_domain_names(self, client_name):
		return [n.attrib['name'] for n in self.clients_info[client_name].findall('domains/domain-info')]

	def _get_reseller_domain_names(self, reseller_name):
		return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('domains/domain-info')]

	def _get_admin_client_names(self):
		return [
			n.attrib['name'] for n in itertools.chain(
				self.backup_info.findall('admin/clients/client-info'),
				self.backup_info.findall('client')
			)
		]

	def _get_reseller_client_names(self, reseller_name):
		return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('clients/client-info')]


class PleskBackupSource10(PleskBackupSource):
	def _index_entities(self):
		resellers_info = {}
		for node in self.backup_info.findall('admin/resellers/reseller'):
			resellers_info[node.attrib['name']] = node

		clients_info = {}
		for node in \
			self.backup_info.findall('admin/clients/client') + \
			self.backup_info.findall('admin/resellers/reseller/clients/client'):

			clients_info[node.attrib['name']] = node

		domains_info = {}
		for node in \
			self.backup_info.findall('admin/domains/domain') + \
			self.backup_info.findall('admin/clients/client/domains/domain') + \
			self.backup_info.findall('admin/resellers/reseller/domains/domain') + \
			self.backup_info.findall('admin/resellers/reseller/clients/client/domains/domain'):

			domains_info[node.attrib['name']] = node

		self._indexed = []  # [ obj(path, obj_type, obj_name, xml) ]
		for path, _, node, name in self._iter_reseller_files():
			self._indexed.append(obj(path=path, obj_type='reseller', obj_name=name, xml=resellers_info[name]))
		for path, _, node, name in self._iter_client_files():
			self._indexed.append(obj(path=path, obj_type='client', obj_name=name, xml=clients_info[name]))
		for path, _, node, name in self._iter_domain_files():
			self._indexed.append(obj(path=path, obj_type='domain', obj_name=name, xml=domains_info[name]))

		return resellers_info, clients_info, domains_info

	def save(self, fileobj, archive_format=None, prefix=None):
		"""
		Save migration dump to a given file object

		:param fileobj: fileobj
		:param archive_format: 'tar', 'zip', None
		"""
		indexed_by_path = group_by_id(self._indexed, lambda i: i.path)
		archiver = ZipFileAdapter if 'zip' == archive_format else TarFileAdapter
		new_archive = archiver(fileobj=fileobj, mode='w')
		logger.debug('Saving a fileobject using %s format' % archiver.__name__)
		with closing(new_archive):
			for member in self.container.archive.get_members():
				if member.is_file:
					file_name = member.name

					if not self._is_backup_archive_member_relevant(member):
						continue

					if member.name == self.container.backup_info_file:
						logger.debug(u"Saving a backup_info_file: %s", member.name)
						source = create_xml_stream(self.backup_info)
						file_size = len(source.getvalue())
					elif member.name in indexed_by_path:
						logger.debug(u"Saving a file indexed_by_path: %s", member.name)
						node = indexed_by_path[member.name].xml
						root_node = self._wrap_into_migrationdump_node(
							node, self.container.dump_version
						)
						tree = ElementTree.ElementTree(root_node)
						source = create_xml_stream(tree)
						file_size = len(source.getvalue())
					else:
						logger.debug(u"Saving a file: %s, '%d' bytes", member.name, member.size)
						source = self.container.archive.extract(member.name)
						file_size = member.size

					with closing(source):
						file_name = replace_prefix(file_name, prefix)
						new_archive.add_file(file_name, file_size, source)
			self.container.archive.close()

	def _wrap_into_migrationdump_node(self, node, dump_version):
		return elem(
			'migration-dump',
			attrs={
				'content-included': "false",
				'agent-name': "PleskX",
				'dump-format': "panel",
				'dump-version': '.'.join(map(str, dump_version))},
			children=[
				elem('dump-info', [text_elem('description', "Created by migrator.")]),
				node])

	def cut_irrelevant_subcriptions(self, relevant_subscription_names):
		for domains_node in (
			self.backup_info.findall('admin/domains') +
			self.backup_info.findall('admin/clients/client/domains') +
			self.backup_info.findall('admin/resellers/reseller/domains') +
			self.backup_info.findall('admin/resellers/reseller/clients/client/domains')
		):
			for domain_node in domains_node.findall('domain'):
				if domain_node.attrib['name'] not in relevant_subscription_names:
					domains_node.remove(domain_node)
		self.domains_to_save = relevant_subscription_names

	def cut_irrelevant_clients(self, relevant_logins):
		for clients_node in (
			self.backup_info.findall('admin/clients') +
			self.backup_info.findall('admin/resellers/reseller/clients')
		):
			for client_node in clients_node.findall('client'):
				if client_node.attrib['name'] not in relevant_logins:
					clients_node.remove(client_node)
		self.clients_to_save = relevant_logins

	def cut_irrelevant_resellers(self, relevant_logins):
		for resellers_node in self.backup_info.findall('admin/resellers'):
			for reseller_node in resellers_node.findall('reseller'):
				if reseller_node.attrib['name'] not in relevant_logins:
					resellers_node.remove(reseller_node)
		self.resellers_to_save = relevant_logins

	def get_subscription_owner_client(self, subscription_name):
		"""Find subscription's owner. Return 'client' object."""
		for client in self.iter_all_clients():
			for s in client.subscriptions:
				if subscription_name == s.name:
					return client

	def _get_admin_domain_names(self):
		return [n.attrib['name'] for n in self.backup_info.findall('admin/domains/domain')]

	def _get_client_domain_names(self, client_name):
		return [n.attrib['name'] for n in self.clients_info[client_name].findall('domains/domain')]

	def _get_reseller_domain_names(self, reseller_name):
		return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('domains/domain')]

	def _get_admin_client_names(self):
		return [n.attrib['name'] for n in self.backup_info.findall('admin/clients/client')]

	def _get_reseller_client_names(self, reseller_name):
		return [n.attrib['name'] for n in self.resellers_info[reseller_name].findall('clients/client')]


class PleskBackupSource9(PleskBackupSource10):
	def remove_content(self):
		self.backup_info.getroot().attrib['content-included'] = 'false'
		self._cut_out_content_nodes()

	def _cut_out_content_nodes(self):
		for parent_node in self.backup_info.findall('.//content/..'):
			parent_node.remove(parent_node.find('content'))

	def iter_subdomains(self, subscription_name, domain_name):
		return (
			Subdomain8(subscription_name, i, self.get_subscription(subscription_name))
			for i in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain')
		)

	def _get_auxiliary_users_and_roles(self, client_name, client_info):
		domain_admin_users, domain_admin_roles = self._get_client_auxiliary_users_and_roles_for_domain_admin(client_info)
		mailuser_users, mailuser_roles = self._get_client_auxiliary_users_and_roles_for_mailusers(client_info)
		return (
			domain_admin_users + mailuser_users,
			domain_admin_roles + mailuser_roles
		)

	@classmethod
	def _get_client_auxiliary_users_and_roles_for_domain_admin(cls, client_node):
		"""Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
		where AuxiliaryUserRole is a special role created for domain user"""
		return _get_client_auxiliary_users_and_roles_for_domain_admin(
			client_node
		)

	@classmethod
	def _get_client_auxiliary_users_and_roles_for_mailusers(cls, client_node):
		return _get_client_auxiliary_users_and_roles_for_mailusers(
			client_node
		)

	def iter_sites(self, subscription_name):
		"""Return Subdomain8 objects, even though formally they should be Subdomain9:
		The functions of Subdomain8 should work for Plesk 9 subdomain as well,
		make a distinct Subdomain9 type when you'll need to access Plesk 9 specific subdomain's data
		"""
		return (
			Subdomain8(subscription_name, n, self.get_subscription(subscription_name))
			for n in self.domains_info[subscription_name].iterfind('phosting/subdomains/subdomain')
		)


class PleskBackupSource8(PleskBackupSourceBase):
	logger = logging.getLogger(__name__)

	def __init__(self, container, migration_list_data=None):
		self.container = container
		self.backup_info = self.container.backup_info
		# if None is passed - nothing is filtered out
		self.migration_list_data = _extract_migration_list_data(migration_list_data)

		self.clients_info, self.domains_info = self._index_entities()

	def close(self):
		self.container.close()

	def cut_irrelevant_subcriptions(self, relevant_subscription_names):
		for client_node in self.backup_info.findall('client'):
			for domain_node in client_node.findall('domain'):
				if domain_node.attrib['name'] not in relevant_subscription_names:
					client_node.remove(domain_node)

	def cut_irrelevant_clients(self, relevant_logins):
		for client_node in self.backup_info.findall('client'):
			if client_node.attrib['name'] not in relevant_logins:
				self.backup_info.getroot().remove(client_node)

	def cut_irrelevant_resellers(self, relevant_logins):
		# There are no resellers in Plesk 8.x, nothing to cut
		pass

	@property
	def backup_info_file(self):
		return self.container.backup_info_file

	def _index_entities(self):
		clients_info = {}
		for node in self.backup_info.findall('client'):
			clients_info[node.attrib['name']] = node

		domains_info = {}
		for node in self.backup_info.findall('client/domain'):
			domains_info[node.attrib['name']] = node

		return clients_info, domains_info

	def _get_client_domain_names_filtered(self, client_name):
		if self.migration_list_data is not None:
			return set(self._get_client_domain_names(client_name)) & self.migration_list_data.subscriptions 
		else:
			return self._get_client_domain_names(client_name)

	def _get_client_domain_names(self, client_name):
		return [n.attrib['name'] for n in self.clients_info[client_name].findall('domain')]

	def _get_admin_client_names_filtered(self):
		if self.migration_list_data is not None:
			return [
				client_name for client_name in self._get_admin_client_names() 
				if len(self._get_client_domain_names_filtered(client_name)) > 0
				or client_name in self.migration_list_data.customers
			]
		else:
			return self._get_admin_client_names()

	def _get_admin_client_names(self):
		return self.clients_info.keys()

	def get_plans(self):
		"""Get hosting plans (excluding add-ons) owned by admin."""
		return []

	def get_addon_plans(self):
		"""Get hosting addon plans owned by admin"""
		return []

	def get_custom_plan_items(self):
		"""Get custom plan items"""
		return []
	
	def iter_admin_reseller_plans(self):
		"""Get reseller plans of admin user.
		"""
		return iter([])

	def iter_admin_and_admin_customers_subscriptions(self):
		"""Get subscriptions owned by admin and subscriptions of customers owned by admin.

		Returns iterable with instances of Subscription.
		"""
		for client in self.iter_clients():
			for subscription in client.subscriptions:
				yield subscription

	def iter_admin_subscriptions(self):
		return iter([])

	def iter_admin_auxiliary_user_roles(self):
		return iter([])

	def iter_admin_auxiliary_users(self):
		return iter([])

	def iter_resellers(self):
		return iter([])

	def iter_reseller_plans(self, reseller_name):
		raise NotImplementedError(u"Plesk 8 doesn't support resellers")

	def iter_reseller_addon_plans(self, reseller_name):
		raise NotImplementedError(u"Plesk 8 doesn't support resellers")

	def iter_clients(self):
		for client_name in self._get_admin_client_names_filtered():
			yield self.get_client(client_name)

	def iter_all_clients(self):
		return self.iter_clients()

	def get_client(self, client_name):
		info = self.clients_info[client_name]
		domain_names = self._get_client_domain_names_filtered(client_name)
		pinfo_nodes = info.findall('pinfo')
		password_node = info.find('password')
		password = read_password(password_node)
		
		domain_admin_users, domain_admin_roles = self._get_client_auxiliary_users_and_roles_for_domain_admin(info)
		mailuser_users, mailuser_roles = self._get_client_auxiliary_users_and_roles_for_mailusers(info)
		return Client(
			login=client_name, 
			password=password,
			contact=info.attrib.get('contact'),
			subscriptions=[
				self.get_subscription(domain_name) for domain_name in domain_names
			],
			personal_info=dict(
				(pinfo_node.attrib['name'], pinfo_node.text) for pinfo_node in pinfo_nodes
			),
			auxiliary_users=domain_admin_users + mailuser_users,
			auxiliary_user_roles=domain_admin_roles + mailuser_roles,
			is_enabled=info.find('status/enabled') is not None,
		)

	@classmethod
	def _get_client_auxiliary_users_and_roles_for_domain_admin(cls, client_node):
		"""Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
		where AuxiliaryUserRole is a special role created for domain user"""
		return _get_client_auxiliary_users_and_roles_for_domain_admin(
			client_node
		)

	@classmethod
	def _get_client_auxiliary_users_and_roles_for_mailusers(cls, client_node):
		return _get_client_auxiliary_users_and_roles_for_mailusers(
			client_node
		)

	def get_subscription(self, name):
		if name not in self.domains_info:
			raise SubscriptionNotFoundException(name)
		domain_info = self.domains_info[name]
		ips = parse_subscription_ips(domain_info.findall('ip'))
		return Subscription8(
			node=domain_info,
			backup=self,
			name=name,
			plan_id=None,
			ip=ips.v4, ip_type=ips.v4_type, ipv6=ips.v6, ipv6_type=ips.v6_type,
			limits=dict(
				(n.attrib['name'], n.text)
				for n in domain_info.findall('limit')
			),
			permissions=dict(
				(n.attrib['name'], n.attrib['value'] == 'true')
				for n in domain_info.findall('permission')
			),
			is_enabled=domain_info.find('status/enabled') is not None,
			hosting_type=get_hosting_type(domain_info),
			is_maintenance=get_maintenance_mode(domain_info)
		)

	def iter_db_servers(self):
		db_server_nodes = self.backup_info.findall('server/db-server')
		for node in db_server_nodes:
			# Line for PostgreSQL server is always present in Plesk database, even if it isn't configured,
			# but administrator login and password are missing in this case.
			# Perhaps this applies to other database types too.
			# So just skip databases without administrator login.
			db_admin_node = node.find('admin')
			if db_admin_node is not None:
				assert db_admin_node.find('password').attrib['type'] == 'plain'
				yield DatabaseServer(
					node=node,
					host=node.findtext('host'),
					port=int(node.findtext('port')),
					login=db_admin_node.attrib['name'],
					password=db_admin_node.findtext('password'),
					dbtype=node.attrib['type'],
					is_default=node.attrib.get('default', 'false') == 'true',
				)

	def iter_mailboxes(self):
		for subscription_name, subscription_node in self.domains_info.iteritems():
			if self.migration_list_data is None or subscription_name in self.migration_list_data.subscriptions:
				mailuser_nodes = subscription_node.findall('mailsystem/mailuser')
				for mailuser_node in mailuser_nodes:
					yield Mailbox8.parse(mailuser_node, subscription_name)

	def iter_subscription_mailboxes(self, subscription_name):
		subscription_node = self.domains_info[subscription_name]
		mailuser_nodes = subscription_node.findall('mailsystem/mailuser')
		for mailuser_node in mailuser_nodes:
			yield Mailbox8.parse(mailuser_node, subscription_name)

	def get_subscription_site_names(self, subscription_name, include_subdomains=False):
		domain_info = self.domains_info[subscription_name]
		if include_subdomains:
			return {'%s.%s' % (subscription_name, n.attrib['name']) for n in domain_info.findall('phosting/subdomain')}
		else:
			return set()

	def get_dns_settings(self):
		return DnsSettings(
			do_subdomains_own_zones=False,
		)

	def iter_dns_template_records(self):
		template_node = self.backup_info.find('server/dns-settings/dns-zone')
		for node in template_node.findall('dnsrec'):
			yield DnsRecord.parse(node)

	def iter_sites(self, subscription_name):
		"""Iterate over all subscription sites: addon domains and subdomains.
		"""
		return (
			Subdomain8(subscription_name, n, self.get_subscription(subscription_name))
			for n in self.domains_info[subscription_name].iterfind('phosting/subdomain')
		)

	def iter_addon_domains(self, subscription_name):
		"""Get addon domains.
		"""
		return []

	def iter_subdomains(self, subscription_name, domain_name):
		domain_info = self.domains_info[subscription_name]
		return (
			Subdomain8(subscription_name, i, self.get_subscription(subscription_name))
			for i in domain_info.iterfind('phosting/subdomain')
		)

	def iter_aliases(self, subscription_name):
		return (
			DomainAlias(subscription_name, node)
			for node in self.domains_info[subscription_name].iterfind('domain-alias')
		)

	def get_system_ips(self):
		return [parse_system_ip(n, self.is_windows) for n in self.backup_info.findall('server/system-ip')]

	def get_default_ip(self):
		return self.backup_info.findtext('server/default-ip')

	@property
	def is_windows(self):
		return False

	def save(self, fileobj, archive_format=None, prefix=None):
		with closing(create_xml_stream(self.backup_info)) as s:
			dump_content = s.getvalue()
		dump_msg = self.container.archive.files[self.container.backup_info_file]
		dump_msg.set_payload(dump_content)
		dump_msg.replace_header('Content-Length', len(dump_content))

		with closing(gzip.GzipFile(fileobj=fileobj)) as gzipped:
			gzipped.write(self.container.archive.msg.as_string())


def _convert_plesk8_auxiliary_user_permissions(permissions):
	"""Convert Plesk 8 auxiliary user permissions to Plesk 10.x format"""

	permissions_map = {
		"manage_maillists": "mailListsManagement",  # Mailing lists management
		"manage_anonftp": "anonymousFtpManagement",  # Anonymous FTP management
		"manage_crontab": "scheduledTasksManagement",  # Scheduler management
		"manage_dns": "dnsManagement",  # DNS zone management
		"manage_log": "logRotationManagement",  # Log rotation management
		"manage_spamfilter": "spamfilterManagement",  # Spam filter management
		"manage_virusfilter": "antivirusManagement",  # Antivirus management
		"allow_ftp_backups": "backupRestoreManagement",  # Allow backing up and restoring data and us: ftp/local
		"manage_phosting": "webSitesAndDomainsManagement",  # Allow manage hosting: create/update/delete sites
		"manage_webapps": "applicationsManagement",  # Allow manage aps applications
		"manage_webstat": "browseStats",  # Allow to view statistics
		# Skipped permissions:
		# - select_db_server
		# - stdgui
		# - multiple-sessions
		# - manage_not_chroot_shell
		# - manage_quota
		# - manage_sh_access
		# - manage_subdomains
		# - manage_domain_aliases
		# - manage_ftp_password
		# - manage_dashboard 
		# - dashboard
		# - allow_local_backups
	}

	result = {
		# default permissions
		'filesManagement': True,
		'mailManagement': True,
		'userManagement': False,
		'ftpAccountsManagement': False,
		'databasesManagement': True,
		'javaApplicationsManagement': False,
		'sitebuilderManagement': False,

	}
	for permission_name, permission_value in permissions.iteritems():
		if permission_name in permissions_map:
			result[permissions_map[permission_name]] = permission_value
	return result


def _get_client_auxiliary_users_and_roles_for_domain_admin(client_node, domain_nodes=[]):
	"""Return auxiliary users and roles for domain admin on Plesk 8/9
	
	Returns generator of tuples (AuxiliaryUser, AuxiliaryUserRole)
	where AuxiliaryUserRole is a special role created for domain user
	"""

	roles = []
	users = []

	# Difference between Plesk 8 and Plesk 9 is location of domain node
	for domain_node in itertools.chain(
		multiple_findall(client_node, ['domain', 'domains/domain']),
		domain_nodes
	):
		user_nodes = domain_node.findall('domainuser')
		if len(user_nodes) == 1:
			user_node = user_nodes[0]
			if user_node.attrib['status'] == 'true':
				personal_info = dict(
					(pinfo_node.attrib['name'], pinfo_node.text)
					for pinfo_node in user_node.findall('pinfo') if pinfo_node.text is not None
				)
				# create special role for domain user and put all permissions in it
				role_name = u"Domain Administrator (%s)" % (domain_node.attrib['name'],)
				password_node = user_node.find('password')
				password = read_password(password_node)

				users.append(AuxiliaryUser(
					name=domain_node.attrib['name'],  # auxiliary user name is the same as domain name
					contact=user_node.attrib.get('contact'),  # contact could be absent in PfU 8/9
					email=personal_info.get('email'),  # email could be absend in PfU 8/9
					password=password,
					roles=[role_name],
					personal_info=personal_info,
					subscription_name=domain_node.attrib['name'],
					is_active=user_node.attrib['status'] == 'true',
					is_domain_admin=True,
					is_built_in=False,
				))
				roles.append(AuxiliaryUserRole(
					name=role_name,
					permissions=_convert_plesk8_auxiliary_user_permissions(dict(
						# Difference between Plesk 8 and Plesk 9 is name of attribute
						(perm_node.attrib['name'], True if multiple_attrib_get(perm_node, ['allowed', 'value']) == 'true' else False)
						for perm_node in user_node.findall('permission')
						if multiple_attrib_get(perm_node, ['allowed', 'value']) in ('true', 'false')
					)),
					is_domain_admin=True
				))
		elif len(user_nodes) > 1:
			# there is only one auxiliary user per domain, in PfU 8 it is called domain administrator
			raise Exception(
				u"Expected to have one or no domainuser nodes in Plesk backup XML for client '%s'" % (
					client_node.attrib['name'],
				)
			)

	return users, roles


def _get_client_auxiliary_users_and_roles_for_mailusers(client_node, domain_nodes=[]):
	"""Return auxiliary users and roles for mail users on Plesk 8/9
	
	Returns list with 2 elements:
	- the first one is a list of AuxiliaryUser objects - users created for each mail user
	- the second one is a list of AuxiliaryUserRole objects - roles specially created for auxiliary users
	"""

	roles = dict()
	users = []
	# Difference between Plesk 8 and Plesk 9 is location of domain node
	for domain_node in itertools.chain(
		multiple_findall(client_node, ['domain', 'domains/domain']),
		domain_nodes
	):
		# Difference between Plesk 8 and Plesk 9 is location of mailuser node
		for mail_user_node in multiple_findall(domain_node, ['mailsystem/mailuser', 'mailsystem/mailusers/mailuser']):
			permissions = frozenset([
				permission_node.attrib['name'] 
				for permission_node in multiple_findall(
					mail_user_node, ['mailuser-permission', 'limits-and-permissions/mailuser-permission']
				)
				# Difference between Plesk 8 and Plesk 9 is name of attribute
				if multiple_attrib_get(permission_node, ['allowed', 'value']) == 'true'
			])
			
			if 'cp-access' in permissions:
				permission_names = {'manage-spamfilter': 'spamfilter', 'manage-virusfilter': 'virusfilter'}
				plesk_permission_names = {'manage-spamfilter': 'spamfilterManagement', 'manage-virusfilter': 'antivirusManagement'}

				# list is used to save order, so role naming will be the same as if role was created by Plesk restore
				role_permissions = tuple(
					permission for permission in ['manage-spamfilter', 'manage-virusfilter']
					if permission in permissions
				)

				role_name = u'Mail User (%s)' % (
					"; ".join([permission_names[permission] for permission in role_permissions])
				)

				if role_permissions not in roles:  # add role according to permissions
					roles[role_permissions] = AuxiliaryUserRole(
						name=role_name,
						permissions=dict((plesk_permission_names[permission], True) for permission in role_permissions),
						is_domain_admin=False,
					)
				name = u'%s@%s' % (mail_user_node.attrib['name'], domain_node.attrib['name'])

				# Difference between Plesk 8 and Plesk 9 is location of password node
				password_node = multiple_find(mail_user_node, ['password', 'properties/password'])
				if password_node is not None:
					password = read_password(password_node)
					if password.text != '':
						users.append(AuxiliaryUser(
							name=name,
							contact=name,
							email=name,
							password=password,
							roles=[role_name],
							personal_info={},
							subscription_name=domain_node.attrib['name'],
							is_active=True,
							is_domain_admin=False,
							is_built_in=False,
						))
					else:
						# there is no sense to create auxiliary user for mail user with
						#  empty password (which is the same as no password) as before migration it:
						# - was not able to login
						# - was not presented as a separate entity
						pass
				else:
					# there is no sense to create auxiliary user for mail user with no password as before migration it:
					# - was not able to login
					# - was not presented as a separate entity
					pass 

	return users, roles.values()


def _extract_migration_list_data(migration_list_data):
	if migration_list_data is not None:
		return obj(
			subscriptions=set(migration_list_data.subscriptions_mapping.keys()),
			customers=migration_list_data.customers_mapping.keys(),
			resellers=migration_list_data.resellers,
			plans=migration_list_data.plans
		)
	else:
		return None


class SubscriptionNotFoundException(Exception):
	def __init__(self, name):
		super(Exception, self).__init__(
			"Failed to find subscription '%s' in backup file" % (
				name
			)
		)
