import logging
import itertools
from collections import defaultdict

from parallels.utils import exists
from parallels.utils import group_by
from parallels.utils import format_list
from parallels.utils.ip import is_ipv4
from parallels.utils.ip import is_ipv6
from parallels.common.checking import Issue
from parallels.common.checking import Problem
from parallels.common.logging_context import log_context
from parallels.common.actions.base.subscription_action import SubscriptionAction
from parallels.common.converter.dns import Rec, render_dns_record_template, \
	extract_custom_records, render_dns_zone_template, pretty_record_str       
from parallels.plesk_api.operator.dns import DnsRecord

logger = logging.getLogger(__name__)


class DNS(SubscriptionAction):
	"""Convert DNS resource records of a subscription being migrated to PPA."""

	def get_description(self):
		return "Convert DNS records"

	def get_failure_message(self, global_context, subscription):
		return "Failed to convert DNS records for subscription '%s'" % (subscription.name)

	def filter_subscription(self, global_context, subscription):
		"""
		:type global_context: parallels.common.global_context.GlobalMigrationContext
		:type subscription: parallels.common.migrated_subscription.MigratedSubscription
		"""
		return True

	def run(self, global_context, subscription):
		"""
		:type global_context: parallels.common.global_context.GlobalMigrationContext
		:type subscription: parallels.common.migrated_subscription.MigratedSubscription
		"""
		source_ips = self._get_all_source_ips(global_context, subscription)
		source_dns_zone_template = self._get_source_dns_template(
											global_context, subscription)
		target_dns_zone_template = global_context.import_api.get_dns_template()
		converter = SubscriptionDnsConverter(
			subscription, global_context, source_ips, source_dns_zone_template,
			target_dns_zone_template)
		converter.process_subscription()

	def _get_source_dns_template(self, global_context, subscription):
		"""By default, get DNS template from the migration dump.

		:type global_context: parallels.common.global_context.GlobalMigrationContext
		:type subscription: parallels.common.migrated_subscription.MigratedSubscription
		"""
		return list(
			subscription.full_converted_backup.iter_dns_template_records()
		)

	def _get_all_source_ips(self, global_context, subscription):
		"""Get all IPs of all source servers. 

		To be overriden in Expand migrator

		:type global_context: parallels.common.global_context.GlobalMigrationContext
		:type subscription: parallels.common.migrated_subscription.MigratedSubscription
		"""
		ips = set()
		for settings in global_context.source_plesks.itervalues():
			backup = global_context.load_raw_plesk_backup(settings)
			for ip in backup.get_system_ips():
				ips.add(ip.address)
		return ips


class SubscriptionDnsConverter(object):
	"""Convert DNS resource records of a subscription being migrated to PPA.
	
	DNS records are updated as follows:
	- DNS records, that point to source server IP addresses, after
	  migration, should point to the target server IPs.
	- Resource records, that refer to IP addresses not controlled by source
	  panel, should remain intact.
	- Resource records, that point to the services controlled by source control
	  panel, should be updated or removed.
	"""
	def __init__(
			self, subscription, global_context, source_ip_addresses,
			source_dns_zone_template, target_dns_zone_template):
		self.subscription = subscription
		self.global_context = global_context
		self.plesk_ips = source_ip_addresses
		self.source_dns_zone_template = source_dns_zone_template
		self.target_dns_zone_template = target_dns_zone_template

	def process_subscription(self):
		"""Change DNS zones and IP addresses to the values suitable for the target panel."""
		logger.debug(u"Source DNS zone template: " % self.source_dns_zone_template)
		main_domain = [self.subscription.converted_backup]
		addon_domains = self.migration_dump.iter_addon_domains(self.subscription.name)
		for domain in itertools.chain(main_domain, addon_domains):
			with log_context(domain.name):
				for subdomain in self._get_subdomains_iter(domain.name):
					self._process_subdomain(domain, subdomain)
				self._process_domain(domain)

		self._process_domain_aliases()

	def _process_subdomain(self, domain, subdomain):
		# Applicable only for Plesk >= 10.4, before all subdomains did not have own DNS zones
		if subdomain.name[0] == '*' or subdomain.dns_zone is None:
			return

		should_overwrite_subdomain_zone = (
				not subdomain.dns_zone.enabled
				and not self.do_subdomains_own_zones)

		if should_overwrite_subdomain_zone:
			# Subdomains are in parent domains's zone
			# Remove all garbage records dumped by Plesk backup
			subdomain_records = self._get_subdomain_records_from_domain_zone(domain, subdomain)
			self._move_records_to_subdomain_zone(domain, subdomain, subdomain_records)

		self._convert_dns_zone_records(
			subdomain.dns_zone, self.plesk_ips, self._get_source_dns_template(), 
			self._add_www_template_record(self._get_target_dns_zone_template(), subdomain), 
			self.get_source_ip_addresses(), self.get_target_ip_addresses())

		if should_overwrite_subdomain_zone:
			self._cleanup_alias_records(domain, subdomain, subdomain_records)

	def _process_domain(self, domain):
		if domain.dns_zone is None:
			return

		source_template = (
			self._get_source_dns_template()
			+ self._get_subdomain_template_records(
				self.subscription.name, domain.name))
		target_template = (
			self._add_wildcard_subdomains_template_records(
				self._add_www_template_record(
					self._get_target_dns_zone_template(), domain),
				self.subscription.name, domain)
				| set(
					self._get_subdomain_template_records(
						self.subscription.name, domain.name)))
		self._convert_dns_zone_records(
			domain.dns_zone, self.plesk_ips,
			source_template, target_template,
			self.get_source_ip_addresses(), self.get_target_ip_addresses())
		
	def _process_domain_aliases(self):
		parent_domain_by_name = {}
		for domain in itertools.chain(
				[self.subscription.converted_backup],
				self.migration_dump.iter_addon_domains(self.subscription.name)):
			parent_domain_by_name[domain.name] = domain
			
		for domain in self.migration_dump.iter_aliases(self.subscription.name):
			with log_context(domain.name):
				if domain.dns_zone is not None:
					source_template = (
						self._get_source_dns_template()
						+ self._get_subdomain_template_records(
							self.subscription.name, domain.parent_domain_name,
							domain.name, force=True))
					target_template = (
						self._add_www_template_record(
							self._get_target_dns_zone_template(),
							parent_domain_by_name[domain.parent_domain_name])
						| set(
							self._get_subdomain_template_records(
								self.subscription.name, domain.parent_domain_name,
								domain.name, force=True)))
					self._convert_dns_zone_records(
						domain.dns_zone, self.plesk_ips,
						source_template, target_template,
						self.get_source_ip_addresses(), self.get_target_ip_addresses())

	def _cleanup_alias_records(self, domain, subdomain, subdomain_records):
		"""Convert subdomain's records inside domain aliases."""
		for alias in self.migration_dump.iter_aliases(self.subscription.name):
			if alias.parent_domain_name == domain.name and alias.dns_zone is not None:
				self._remove_records_from_alias_zone(
						alias, domain.name, subdomain_records)
				self._add_subdomain_records_to_alias_zone(
						subdomain, alias, domain.name, subdomain_records)

	def _remove_records_from_alias_zone(self, alias, domain_name, domain_existing_records):
		"""Remove subdomain records from alias DNS zone"""
		for rec in domain_existing_records : # remove all records that look like subdomain's
			for alias_rec in alias.dns_zone.iter_dns_records():
				alias_idna_record = self._get_idna_record(alias_rec)
				subdomain_idna_record = self._replace_domain_name(rec, domain_name, alias.name)
				if alias_idna_record == subdomain_idna_record:
					logger.debug(
						"Remove a record from '%s' alias ('%s' domain) zone: '%s'",
						alias.name, domain_name, pretty_record_str(alias_rec))
					alias.dns_zone.remove_dns_record(alias_rec)

	def _add_subdomain_records_to_alias_zone(
			self, subdomain, alias, domain_name, subdomain_records):
		"""Add alias records from subdomain DNS zones.

		Add records from the new subdomain's zone, replacing domain name with
		alias name. Add record only if similar record existed for subdomain
		before. Do not add all records generated by DNS template, like
		'webmail.<subdomain>', 'mail.<subdomain>', etc
		"""
		for rec in subdomain.dns_zone.iter_dns_records():
			if exists(
					subdomain_records,
					lambda r: r.rec_type == rec.rec_type and r.src.encode('idna') == rec.src.encode('idna')): 
				replaced_record = self._replace_domain_name(rec, domain_name, alias.name)
				logger.debug(
						"Add '%s' subdomain's record to '%s' alias zone: '%s'",
						subdomain.name, alias.name,
						pretty_record_str(replaced_record))
				alias.dns_zone.add_dns_record(replaced_record)

	def _get_idna_record(self, record):
		return Rec(
			record.rec_type, 
			record.src.encode('idna'),
			record.dst.encode('idna'),
			record.opt
		)

	def _replace_domain_name(self, record, old_name, new_name):
		return Rec(
			record.rec_type, 
			self._replace_parent_domain_name(record.src, old_name, new_name),
			self._replace_parent_domain_name(record.dst, old_name, new_name),
			record.opt
		)

	def _replace_parent_domain_name(self, domain_name, old_suffix, new_suffix):
		"""Replace parent domain name of a subdomain."""
		idna_domain_name = domain_name.encode('idna')
		if idna_domain_name.endswith("%s." % old_suffix.encode('idna')):
			return (idna_domain_name[:-(len(old_suffix.encode('idna'))+1)]
				+ new_suffix.encode('idna') + '.')
		else:
			return idna_domain_name

	@property
	def migration_dump(self):
		return self.subscription.full_converted_backup

	@property
	def do_subdomains_own_zones(self):
		return self.migration_dump.get_dns_settings().do_subdomains_own_zones

	def _move_records_to_subdomain_zone(self, domain, subdomain, records):
		"""Overwrite subdomain zone with the records from the parent domain zone."""
		for rec in list(subdomain.dns_zone.iter_dns_records()):
			subdomain.dns_zone.remove_dns_record(rec)
		# Copy all records from parent domain's zone to subdomain's zone
		for rec in records:
			domain.dns_zone.remove_dns_record(rec)
			subdomain.dns_zone.add_dns_record(rec)
		# Enable zone as it is disabled in backup (which, in terms of
		# backup-restore means that record are stored in parent domain's zone)
		subdomain.dns_zone.enable()

	def _get_subdomain_records_from_domain_zone(self, domain, subdomain):
		"""Find records to be moved to subdomain zone."""
		if (domain.dns_zone is None 
				or subdomain.name[0] == '*'
				or subdomain.dns_zone is None or subdomain.dns_zone.enabled
				or self.do_subdomains_own_zones):
			return []

		records = []
		subdomain_idna_name = '%s.' % subdomain.name.encode('idna')
		subdomain_idna_suffix = '.%s' % subdomain_idna_name
		for record in domain.dns_zone.iter_dns_records():
			if (
					record.src.encode('idna') == subdomain_idna_name
					or record.src.encode('idna').endswith(subdomain_idna_suffix)):
				records.append(record)
		return records

	def _add_www_template_record(self, zone_template, domain):
		"""Add 'www.example.com' CNAME record to the template.

		In case there is a 'www.<domain>' record in source template, but
		there is no 'www.<domain>' record in target template, www records
		are removed from DNS zones. So if you have a www domain alias
		(which is a special "checkbox", not a usual domain alias) it won't
		work after migration.  To workaround the issue we add
		'www.<domain>' record to target template in case domain has www
		alias and there is no 'www.<domain>' record in target template
		"""
		result_zone_template = set(zone_template)

		if domain.www_alias_enabled and not exists(
				zone_template, lambda r: r.src == 'www.<domain>.'):
			result_zone_template.add(Rec('CNAME', 'www.<domain>.', '<domain>.', ''))
			
		return result_zone_template

	def _add_wildcard_subdomains_template_records(
			self, zone_template, subscription_name, domain):
		result_zone_template = set(zone_template)

		all_ips = self.get_source_ip_addresses()
		for subdomain in self._get_subdomains_iter(domain.name):
			if subdomain.name.startswith('*') and domain.dns_zone is not None:
				
				subdomain_name = "%s." % (subdomain.name,)
				template_records = {
					'ip.web': Rec('A', subdomain_name, '<ip.web>', ''), 
					'ipv6.web': Rec('AAAA', subdomain_name, '<ipv6.web>', '') 
				}

				web_ips = { name: all_ips[name] for name in template_records.keys()}
				for domain_record in domain.dns_zone.iter_dns_records():
					for name, template_record in template_records.iteritems():
						if domain_record.src == subdomain_name and (
								domain_record.dst == web_ips[name]):
							result_zone_template.add(template_record)

		return result_zone_template

	def get_source_ip_addresses(self):
		web_ipv4 = self.subscription.converted_backup.ip
		web_ipv6 = self.subscription.converted_backup.ipv6
		mail_ipv4 = self.subscription.source_mail_ip
		mail_ipv6 = self.subscription.source_mail_ipv6

		if self.migration_dump.container.dump_version[0] in (8, 9):
			if (
				# 1) Workaround for Plesk 8 (Linux) and Plesk 9 (Windows and
				# Linux): if subscription has no hosting, it has no IP
				# addresses in backup. But DNS records were rendered with
				# some of Plesk server's IP address.  So if we leave IP
				# addresses empty, almost all records that contain IP address
				# are considered custom, and transfered as is. However we
				# expect to re-instantiate such records with new PPA service
				# nodes IP addresses.

				# To workaround the problem we try to detect which server's IP
				# is an IP of the subscription without hosting.  We consider
				# that IP address of the Plesk server that matches maximum
				# number of records of domain's DNS zone when Plesk server's
				# DNS template is rendered - is IP of the subscription.

				# Also in previous version of the workaround we tried to
				# detected with Plesk API request which returns DNS server of
				# the subscription. But it has not worked on customer's
				# installation that (according to Plesk API) has DNS on another
				# node.
				web_ipv4 is None

				# 2) Another special case: while migrating PfW 8, we convert
				# backup to PfW 9 format with PMM.  During this conversion PMM
				# sets some IP address from client's pool as domain's IP
				# address.  However sometimes this address does not match
				# address in DNS records. So we detect domain's IP address by
				# DNS records. 
				or (
					self.migration_dump.is_windows
					and self.subscription.converted_backup.hosting_type == 'none')):
				logger.debug(u"Seems that domain '%s' has disabled web hosting, which makes Plesk not to put domain IP in backup. Run workaround - determine IP address by DNS records", self.subscription.name)

				dns_zone_recs = []
				if self.subscription.converted_backup.dns_zone is not None:
					dns_zone_recs = [Rec(rec_type=r.rec_type, src=r.src, dst=r.dst, opt=r.opt) for r in self.subscription.converted_backup.dns_zone.iter_dns_records()]
				
				def matching_records_count(ip, dns_zone_recs):
					subscr_vars = {'ip': ip, 'domain': self.subscription.name}
					instantiated_recs = render_dns_zone_template(self._get_source_dns_template(), subscr_vars).keys()
					return len(set(dns_zone_recs) & set(instantiated_recs))

				# There were no IPv6 in PfU 8 and 9, so we process only IPv4
				web_ipv4 = max(
					[(ip.address, matching_records_count(ip.address, dns_zone_recs)) for ip in self.migration_dump.get_system_ips()],
					key=lambda x: x[1]
				)[0]

				# Note that there is no need to set up mail IP separately as:
				# - In case of non-centralized mail in Plesks 8 and 9 we have only <ip> variable in DNS templates, 
				# there is no <ip.mail> and <ip.webmail> variables
				# - In case of centralized mail in Expand based on Plesks 8 and 9 IP address of centralized mail server
				# is "hardcoded" in DNS template from source Plesk server backup
			
		source_subscription_ips = {
			'ip': web_ipv4,
			'ipv6': web_ipv6,
			'ip.dns': self.migration_dump.get_default_ip(),
			'ipv6.dns': None, # in current Plesk implementation ipv6.dns variable is always absent
			'ip.web': web_ipv4,
			'ipv6.web': web_ipv6,
			'ip.webmail': mail_ipv4,
			'ipv6.webmail': mail_ipv6,
			'ip.mail': mail_ipv4,
			'ipv6.mail': mail_ipv6,
		}
		return source_subscription_ips

	def _get_source_dns_template(self):
		return self.source_dns_zone_template

	def _get_target_dns_zone_template(self):
			return self.target_dns_zone_template

	def get_target_ip_addresses(self):
		return {
			'ip': self.subscription.target_public_web_ipv4,
			'ipv6': self.subscription.target_public_web_ipv6,
			'ip.web': self.subscription.target_public_web_ipv4,
			'ipv6.web': self.subscription.target_public_web_ipv6,
			# all NS-related records are created by POA, not by Plesk, so we do not instantiate such records from template
			'ip.dns': None,
			'ipv6.dns': None, 
			# We will use value from config due to we can not detect where is webmail for this subscription
			'ip.webmail': self.global_context.webmail_ipv4,
			'ipv6.webmail': None, # XXX IPv6 address should be detected too
			'ip.mail': self.subscription.target_public_mail_ipv4,
			'ipv6.mail': self.subscription.target_public_mail_ipv6
		}

	def _subdomain_zone_is_enabled(self, subdomain):
		# XXX where is it used?
		return (
			subdomain.dns_zone is not None
			and (
				subdomain.dns_zone.enabled
				or self.do_subdomains_own_zones))

	# Domain's zone contains some records for subdomains, this function collects them to add to DNS template.
	# For domain aliases process all parent domain's subdomains.
	def _get_subdomain_template_records(self, subscription_name, domain_name, alias_name=None, force=False):
		"""
		force - in case of domain alias we should always include subdomain records in template, as subdomain records may be
		in DNS zone of alias, and presense of subdomain zone or "do subdomains own zones" flags should be ignored
		"""
		if alias_name == None:
			alias_name = domain_name
		subdomain_records = []
		for subdomain in self._get_subdomains_iter(domain_name):
			if (subdomain.dns_zone is None or (not self.do_subdomains_own_zones and not subdomain.dns_zone.enabled) or force):
				subdomain_records.extend([
					Rec('A', u'%s.%s.' % (subdomain.short_name.encode('idna'), alias_name.encode('idna')), '<ip.web>', ''),
					Rec('AAAA', u'%s.%s.' % (subdomain.short_name.encode('idna'), alias_name.encode('idna')), '<ipv6.web>', ''),
				])
		return subdomain_records

	def _get_subdomains_iter(self, domain_name):
		return self.migration_dump.iter_subdomains(self.subscription.name, domain_name)

	def report(self, issue):
		self.global_context.safe.add_issue_subscription(self.subscription.name, issue)

	def _convert_dns_zone_records(
			self, dns_zone, source_server_ips, source_template,
			target_dns_zone_template, source_subscription_ips, new_subscr_vars):
		"""Convert DNS records of given DNS zone as follows:
		   1) virtually split records in two sets: standard records and custom records
		   a record is standard if it equals to a record instantiated from DNS template
		   2) remove all standard records from zone
		   3) generate new standard records, using new (target) DNS template, and add them to zone
		   4) process custom records
		   4.1) warn if custom records' data points to one of source Plesk's IPs
		   4.2) remove all AXFR records as they break zone restoration
		"""
		logger.debug(u"Convert DNS records for '%s' zone", dns_zone.domain_name)

		dns_zone.remove_marker_records()

		class RecExt(Rec):
			PRIORITY_TARGET = 1
			PRIORITY_SOURCE = 2

			def __new__(cls, rec_type, src, dst, opt, priority):
				self = super(RecExt, cls).__new__(cls, rec_type, src, dst, opt)
				
				# priority during conflict resolution: source records have more priority than target records
				# see conflict resolution below in the _convert_dns_zone_records function to understand what's this
				if priority is None:
					self.priority = self.PRIORITY_SOURCE
				else:
					self.priority = priority 

				return self

			@classmethod
			def from_rec(cls, rec, priority):
				return cls(rec_type=rec.rec_type, src=rec.src, dst=rec.dst, opt=rec.opt, priority=priority)

		class DnsZoneAdapter(object):
			def __init__(self, dnz_zone):
				self.dns_zone = dns_zone

				# Convert `DnsRecord`s to `Rec`s.
				self.xml_recs = dict(
					(self._create_rec(r, RecExt.PRIORITY_SOURCE), r)
					for r in dns_zone.iter_dns_records()
				)

			def remove_dns_record(self, rec):
				self.dns_zone.remove_dns_record(self.xml_recs[rec])
				del self.xml_recs[rec]

			def add_dns_record(self, rec, priority):
				rec_ext = RecExt.from_rec(rec, priority)
				if rec_ext not in self.xml_recs:
					self.xml_recs[rec_ext] = dns_zone.add_dns_record(rec_ext)
				else:
					# Make resulting DNS records unique: skip duplicates.
					# There are two cases when records can be duplicate:
					# (usual) two PTR records with the same IP address and same domain (instantiated from template's PTR records
					# for web and mail, in case when web and mail hosted on the same server)
					# (potential) custom records may refer to the new IP address, be valid in source Plesk,
					# but duplicate after we generate standard records by template using the new IP address
					logger.debug(u"Duplicate DNS record %r will not be added again" % (rec_ext,))

			def get_records(self):
				return self.xml_recs.keys()

			@staticmethod
			def _create_rec(xml_rec, priority):
				return RecExt(rec_type=xml_rec.rec_type, src=xml_rec.src, dst=xml_rec.dst, opt=xml_rec.opt, priority=priority)

		dns_zone_adapter = DnsZoneAdapter(dns_zone)

		old_vars = dict(source_subscription_ips, domain = dns_zone.domain_name.encode('idna'))

		standard_records, custom_records = extract_custom_records(
			source_template,
			old_vars,
			dns_zone_adapter.get_records()
		)

		# Standard records that were removed from source DNS zone
		removed_records = set(render_dns_zone_template(source_template, old_vars)) - set(dns_zone_adapter.get_records())

		new_vars = dict(new_subscr_vars, domain = dns_zone.domain_name.encode('idna'))

		# Remove all standard DNS records
		for rec, tmpl_rec in standard_records.iteritems():
			logger.debug(u"The standard DNS record %r will be removed", rec)
			dns_zone_adapter.remove_dns_record(rec)

		# Consider the following workaround for a problem with NS records while reading the code below
		#
		# Example: 
		# on source server you have a DNS template item "NS <domain> ns.providerdns.tld" where ns.providerdns.tld is a centralized DNS server
		# corresponding record for this template item exists in domain's zone
		# also on source server you have an old "A ns.<domain> <ip>" record which does not correspond to any template item
		# and it does not make any sense in real life - it is not used.
		# however during migration we consider this A record custom and transferred it as is, leaving the old IP address
		# also by default in PPA there are template items "NS <domain> ns.<domain>" "A ns.<domain> <dns.ip>"
		# so finally we have the following zone (let's have domain.tld as <domain>, 10.58.1.1 as <dns.ip> - PPA DNS IP, 192.168.1.1 as <ip> - source server IP):
		#   NS domain.tld ns.domain.tld
		#   A ns.domain.tld 10.58.1.1
		#   A ns.domain.tld 192.168.1.1
		# so NS will point to correct IP only 50% of the time
		# to avoid such problems, we remove all such "custom" A records that point from new DNS to some old IP
		#
		# In this set we will put dst of all NS records generated by target template
		target_ns_record_names = set()

		# Add standard DNS records generated by target DNS template
		for tmpl_rec in target_dns_zone_template:
			new_rec = render_dns_record_template(tmpl_rec, new_vars)
			if new_rec is not None:	# skip records generated by template with undefined parameters
						# (undefined parameters: ipv6.dns, all ipv6.* parameters for subscription without IPv6)
				if render_dns_record_template(tmpl_rec, old_vars) not in removed_records:
					logger.debug(
						u"DNS record %r is generated by template record (src='%s', dst='%s')",
						new_rec, tmpl_rec.src, tmpl_rec.dst
					)
					dns_zone_adapter.add_dns_record(new_rec, RecExt.PRIORITY_TARGET)
					if new_rec.rec_type == 'NS':
						target_ns_record_names.add(new_rec.dst)
				else:
					# Skip records that were removed from source DNS zone. 
					# Records to skip are detected in the following way (for each item of new template, regardless of new variables): 
					# if record instantiated with the new template and old variables matches some record 
					# that was removed from source DNS zone (removed from source DNS zone means it could be instantiated with old 
					# template and old variables, but does not actually exists in source DNS zone) - corresponding new template item should be skipped
					pass

		def drec(rec_type, src, dst, opt=''):
			return DnsRecord(id=None, rec_type=rec_type, src=src, dst=dst, opt=opt)

		def source_template_records_or(src, default_template_records):
			template_selected_records = [r for r in target_dns_zone_template if r.rec_type in ('A', 'CNAME', 'AAAA') and r.src == src]
			if len(template_selected_records) > 0:
				return template_selected_records
			else:
				return default_template_records

		# Each leaf element of the following structure is a pair (old record template, new records templates).
		# If DNS record matching old record template is found it is replaced by records instantiated by new record templates
		# If new records templates is None - new record templates is considered equal to [old record template,]
		custom_known_records_template = {
			'web': [
				(drec(rec_type='A', src='www.<domain>.', dst='<ip.web>'), None),
				(drec(rec_type='A', src='ftp.<domain>.', dst='<ip.web>'), None),
				(drec(rec_type='A', src='sitebuilder.<domain>.', dst='<ip.web>'), None),
				(drec(rec_type='AAAA', src='www.<domain>.', dst='<ipv6.web>'), None),
				(drec(rec_type='AAAA', src='ftp.<domain>.', dst='<ipv6.web>'), None),
				(drec(rec_type='AAAA', src='sitebuilder.<domain>.', dst='<ipv6.web>'), None),
				# There are no CNAME records (comparing to 'mail' and 'webmail' services below) here.
				# CNAME records will be left as is and still point to "<domain>.", which in its turn points to web IP - and that is correct
			],
			'mail': [
				(drec(rec_type='A', src='mx.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='A', src='mail.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='A', src='pop.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='A', src='pop3.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='A', src='smtp.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='A', src='imap.<domain>.', dst='<ip.mail>'), None),
				(drec(rec_type='AAAA', src='mx.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='AAAA', src='mail.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='AAAA', src='pop.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='AAAA', src='pop3.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='AAAA', src='smtp.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='AAAA', src='imap.<domain>.', dst='<ipv6.mail>'), None),
				(drec(rec_type='CNAME', src='mx.<domain>.', dst='<domain>.'), 
					source_template_records_or('mx.<domain>.', [
						drec(rec_type='A', src='mx.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='mx.<domain>.', dst='<ipv6.mail>'),
					])
				),
				(drec(rec_type='CNAME', src='mail.<domain>.', dst='<domain>.'), 
					source_template_records_or('mail.<domain>.', [
						drec(rec_type='A', src='mail.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='mail.<domain>.', dst='<ipv6.mail>'),
					])
				),
				(drec(rec_type='CNAME', src='pop.<domain>.', dst='<domain>.'), 
					source_template_records_or('pop.<domain>.', [
						drec(rec_type='A', src='pop.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='pop.<domain>.', dst='<ipv6.mail>'),
					])
				),
				(drec(rec_type='CNAME', src='pop3.<domain>.', dst='<domain>.'), 
					source_template_records_or('pop3.<domain>.', [
						drec(rec_type='A', src='pop3.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='pop3.<domain>.', dst='<ipv6.mail>'),
					])
				),
				(drec(rec_type='CNAME', src='smtp.<domain>.', dst='<domain>.'), 
					source_template_records_or('smtp.<domain>.', [
						drec(rec_type='A', src='smtp.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='smtp.<domain>.', dst='<ipv6.mail>'),
					])
				),
				(drec(rec_type='CNAME', src='imap.<domain>.', dst='<domain>.'), 
					source_template_records_or('imap.<domain>.', [
						drec(rec_type='A', src='imap.<domain>.', dst='<ip.mail>'),
						drec(rec_type='AAAA', src='imap.<domain>.', dst='<ipv6.mail>'),
					])
				),
			],
			'webmail': [
				(drec(rec_type='A', src='webmail.<domain>.', dst='<ip.webmail>'), None),
				(drec(rec_type='AAAA', src='webmail.<domain>.', dst='<ipv6.webmail>'), None),
				(drec(rec_type='CNAME', src='webmail.<domain>.', dst='<domain>.'), 
					source_template_records_or('webmail.<domain>.', [
						drec(rec_type='A', src='webmail.<domain>.', dst='<ip.webmail>'),
						drec(rec_type='AAAA', src='webmail.<domain>.', dst='<ipv6.webmail>')
					])
				),
			]
		}

		def render_possible_custom_known_records(possible_custom_known_records, old_vars, new_vars):
			"""
			Returns the following structure: {
				service #1 name: {
					old record #1: [new record #1-1, new record #1-2],
					old record #2: [ ... ],
					...
				},
				service #2 name: { ... },
				...
			}

			Service is 'web', 'mail', 'webmail'.
			Old record is considered to be replaced with new records
			"""
			possible_custom_known_records = defaultdict(dict)
			for service_name, service_record_templates in custom_known_records_template.iteritems():
				for service_record_old_template, service_record_new_templates in service_record_templates:
					if service_record_new_templates is None:
						service_record_new_templates = [service_record_old_template]

					old_rec = render_dns_record_template(service_record_old_template, old_vars)
					if old_rec is not None:
						possible_custom_known_records[service_name][old_rec] = []
						for service_record_new_template in service_record_new_templates:
							new_rec = render_dns_record_template(service_record_new_template, new_vars)
							if new_rec is not None:
								possible_custom_known_records[service_name][old_rec].append(new_rec)

			return possible_custom_known_records

		possible_custom_known_records = render_possible_custom_known_records(custom_known_records_template, old_vars, new_vars)

		possible_custom_known_records_on_other_ips = defaultdict(lambda: defaultdict(dict))
		for ip in source_server_ips:
			if is_ipv4(ip):
				ip_vars = {'domain': old_vars['domain'], 'ip': ip, 'ip.dns': ip, 'ip.web': ip, 'ip.webmail': ip, 'ip.mail': ip}
			elif is_ipv6(ip):
				ip_vars = {'domain': old_vars['domain'], 'ipv6': ip, 'ipv6.dns': ip, 'ipv6.web': ip, 'ipv6.webmail': ip, 'ipv6.mail': ip}
			else:
				ip_vars = {}

			possible_custom_known_records_on_other_ips[ip] = render_possible_custom_known_records(custom_known_records_template, ip_vars, new_vars)

		def replace_rec(old_rec, new_recs, problem_replaced_id, problem_replaced_msg, problem_replaced_solution, problem_removed_id, problem_removed_msg, problem_removed_solution):
			dns_zone_adapter.remove_dns_record(rec)
			if len(new_recs) > 0:
				for new_rec in new_recs:
					dns_zone_adapter.add_dns_record(new_rec, RecExt.PRIORITY_SOURCE)
				self.report(Issue(
					Problem(
						problem_replaced_id, Problem.WARNING,
						problem_replaced_msg.format(
							old_record=pretty_record_str(rec), 
							service_name=service_name, 
							new_records=format_list(map(pretty_record_str, new_recs)),
						) 
					),
					problem_replaced_solution
				))
			else:
				self.report(Issue(
					Problem(
						problem_removed_id, Problem.WARNING,
						problem_removed_msg.format(
							old_record=pretty_record_str(rec), 
							service_name=service_name,
						) 
					),
					problem_removed_solution
				))

		for rec in custom_records:
			# Warn if custom record contains old host IP address, because we don't
			# know whether it corresponds to Web or some other service.
			if \
				(rec.rec_type in ['A', 'AAAA'] and rec.dst in source_server_ips) \
				or (rec.rec_type == 'PTR' and rec.src in source_server_ips):

				if rec.src in target_ns_record_names:
					self.report(Issue(
						Problem(
							'custom_record_can_break_dns', Problem.WARNING,
							u"Custom record (%s) refers to one of host IP addresses and could break operability of DNS. It was not migrated" % (rec,), 
						),
						u"Most probably it is not a problem, but it is recommended to review DNS zone records after migration"
					))
					dns_zone_adapter.remove_dns_record(rec)
				else:
					has_fixed = False

					for service_name, service_possible_custom_known_records in possible_custom_known_records.iteritems():
						if rec in service_possible_custom_known_records:
							replace_rec(
								old_rec=rec, new_recs=service_possible_custom_known_records[rec],
								problem_replaced_id='custom_record_points_to_host_ip_fixed',
								problem_replaced_msg=u"Custom record '{old_record}' refers to one of host IP addresses. The record was changed to point to subscription's {service_name} IP and migrated as {new_records}",
								problem_replaced_solution=u"Most probably it is not a problem, but it is recommended to review DNS zone records after migration",
								problem_removed_id='custom_record_points_to_host_ip_removed',
								problem_removed_msg=u"Custom record '{old_record}' refers to one of host IP addresses. The record was removed as corresponding service does not exist on target subscription",
								problem_removed_solution=u"It is recommended to review DNS zone records after migration"
							)
							has_fixed = True

					if not has_fixed:
						for ip in source_server_ips:
							for service_name, service_record_templates in possible_custom_known_records_on_other_ips[ip].iteritems():
								if rec in service_record_templates:
									replace_rec(
										old_rec=rec, new_recs=service_record_templates[rec],
										problem_replaced_id='custom_record_points_to_non_service_source_ip_changed',
										problem_replaced_msg=u"Custom record '{old_record}' refers to one of source Plesk IP addresses, but the IP does not match source subscriptions's service IPs. " + \
													u"Most probably this record is related to {service_name} service. The record was changed to point to subscription's {service_name} IP and migrated as {new_records}",
										problem_replaced_solution=u"It is recommended to review DNS zone records after migration",
										problem_removed_id='custom_record_points_to_non_service_source_ip_removed',
										problem_removed_msg=u"Custom record '{old_record}' refers to one of source Plesk IP addresses, but the IP does not match source subscriptions's service IPs. " + \
													u"Most probably this record is related to {service_name} service. The record was removed as service does not exist on target subscription",
										problem_removed_solution=u"It is recommended to review DNS zone records after migration",
									)
									has_fixed = True
									break
							if has_fixed:
								break

					if not has_fixed:
							self.report(Issue(
								Problem(
									'custom_record_points_to_host_ip', Problem.WARNING,
									(
										u"Custom record '%s' refers to one of source Plesk IP addresses, but the IP does not match source subscriptions's service IPs. " + 
										u"The record was migrated as is" 
									) % (
										pretty_record_str(rec),
									)
								),
								u"It is strongly recommended to review DNS zone records after migration"
							))

			elif rec.rec_type == 'CNAME':
				for service_name, service_possible_custom_known_records in possible_custom_known_records.iteritems():
					if rec in service_possible_custom_known_records:
						replace_rec(
							old_rec=rec, new_recs=service_possible_custom_known_records[rec],
							problem_replaced_id='custom_cname_record_points_to_domain_fixed',
							problem_replaced_msg=u"Custom CNAME record '{old_record}' refers to domain name. This can make {service_name} service to work incorrectly. The record was changed and migrated as {new_records}",
							problem_replaced_solution=u"It is recommended to review DNS zone records after migration",
							problem_removed_id='custom_cname_record_points_to_domain_removed',
							problem_removed_msg=u"Custom CNAME record '{old_record}' refers to domain name. It is considered that the record is related to {service_name} service. The record was removed as the service does not exist on target subscription",
							problem_removed_solution=u"It is recommended to review DNS zone records after migration",
						)

			# Workaround for bug 108156: AXFR records break zones restoration
			elif rec.rec_type == 'AXFR':
				dns_zone_adapter.remove_dns_record(rec)
				if rec.dst not in source_server_ips or rec.opt != '':
					self.report(Issue(
						Problem(
							'axfr_record_not_transferred', Problem.WARNING,
							u"This domain has AXFR record (%r) allowing external DNS server(s) to transfer its zone. After transferring this domain to PPA, an external DNS server will still be able to transfer zone from Plesk server, but the zone transferred will not contain subdomains' records. The subdomains' records will be held in subdomains' own zones, and must be transferred with these zones. Allowing transfer of subdomain zones from Plesk server is not implemented in migration." % (rec,)
						),
						u"Configure those external DNS server(s) to transfer the subdomains' zones from PPA DNS server(s)."
					))

		# Resolve conflicts. 
		#
		# Example of a conflict:
		# You have a "ftp.<domain>" A record pointing to some external server
		# Also you have a "ftp.<domain>" A record instantiated by target template
		# So after migration you will have 2 "ftp.<domain>" A records, which is not correct
		# The same is applicable to CNAME vs A record conflict which makes Plesk restore not to restore the whole DNS zone
		# as Plesk considers such records conflicting and does not allow to have both CNAME and A record with the same src in a zone.
		#
		# Source records have always more priority than records instantiated by target DNS template
		for _, records in group_by([r for r in dns_zone_adapter.get_records() if r.rec_type in ('A', 'AAAA', 'CNAME')], lambda r: r.src).iteritems():
			records_by_priority = group_by(records, lambda r: r.priority)
			if len(records_by_priority.get(RecExt.PRIORITY_SOURCE, [])) > 0 and len(records_by_priority.get(RecExt.PRIORITY_TARGET, [])) > 0:
				# if there are PRIORITY_SOURCE records - then remove all PRIORITY_TARGET record
				for r in records_by_priority[RecExt.PRIORITY_TARGET]:
					dns_zone_adapter.remove_dns_record(r)
					self.report(Issue(
						Problem(
							'dns_records_conflict_resolved', Problem.WARNING,
							(
								u"DNS record(s) instantiated by PPA template (%s) conflict with record(s) from source server (%s). " + 
								u"Records from source server were migrated, records instantiated by PPA template are not added to DNS zone" 
							) % (
								format_list(map(pretty_record_str, records_by_priority[RecExt.PRIORITY_TARGET])), 
								format_list(map(pretty_record_str, records_by_priority[RecExt.PRIORITY_SOURCE])), 
							)
						),
						u"It is strongly recommended to review DNS zone records after migration"
					))

		def encode_name(s):
			"""Encode name in IDNA lower case for correct comparison"""
			try:
				return s.encode('idna').lower()
			except UnicodeError:
				logger.debug("Failed to encode name '%s' in IDNA", exc_info=True)
				# fallback to initial string, not to fail whole migration 
				# if something is wrong when encoding
				return s

		a_records_src = set([encode_name(rec.src) for rec in dns_zone_adapter.get_records() if rec.rec_type in (['A', 'AAAA'])])

		for rec in dns_zone_adapter.get_records():
			if rec.rec_type == 'CNAME' and rec.src == "%s." % (dns_zone.domain_name,):
				dns_zone_adapter.remove_dns_record(rec)
				self.report(Issue(
					Problem(
						'dns_records_cname', Problem.WARNING,
						(
							u"CNAME DNS record '%s' won't be migrated to PPA. " +
							u"DNS zone in PPA has an NS record with such source, but according to RFC 1912, CNAME record cannot coexist with other records with the same source" 
						) % (
							pretty_record_str(rec)
						)
					),
					u"It is strongly recommended to review DNS zone records after migration. In general case changing CNAME to A record would solve the problem."
				))

			# BIND on CentOS 6 don't even start if you add invalid NS record to the zone, which:
			# - points to some domain inside the same DNS zone
			# - does not have corresponding A/AAAA record (so the record can not be resolved to IP address)
			# Old versions of BIND allowed such records, so it is possible to meet them on old source DNS servers.
			# So, remove such records.
			#
			# NS records that point to other zones are considered correct.
			is_dst_from_the_same_zone = any([
				encode_name(rec.dst).endswith('.%s.' % (encode_name(dns_zone.domain_name),)), # something like 'aaa.example.com.' if domain is 'example.com'
				encode_name(rec.dst) == encode_name('%s.' % (dns_zone.domain_name,)) # exact match - 'example.com.'
			])
			if rec.rec_type == 'NS' and is_dst_from_the_same_zone and encode_name(rec.dst) not in a_records_src:
				# Silently remove the bad record - no sense to notify customer and spam him with unnecessary warnings
				# as the record did not worked even on source server
				logger.debug("Remove bad NS record '%s' that does not have corresponding A record" % pretty_record_str(rec))
				dns_zone_adapter.remove_dns_record(rec)

